< prev index next >
src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/arraycopy/ArrayCopySnippets.java
Print this page
*** 26,40 ****
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.KLASS_SUPER_CHECK_OFFSET_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayClassElementOffset;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayIndexScale;
- import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperElementTypePrimitiveInPlace;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadHub;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.superCheckOffsetOffset;
! import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
import java.lang.reflect.Method;
import java.util.EnumMap;
--- 26,41 ----
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.KLASS_SUPER_CHECK_OFFSET_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayClassElementOffset;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayIndexScale;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadHub;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.superCheckOffsetOffset;
! import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
! import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.LIKELY_PROBABILITY;
! import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY;
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
import java.lang.reflect.Method;
import java.util.EnumMap;
*** 45,58 ****
import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
import org.graalvm.compiler.debug.DebugHandlersFactory;
import org.graalvm.compiler.debug.GraalError;
import org.graalvm.compiler.graph.Node;
import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
- import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
import org.graalvm.compiler.hotspot.word.KlassPointer;
import org.graalvm.compiler.nodes.CallTargetNode;
- import org.graalvm.compiler.nodes.ConstantNode;
import org.graalvm.compiler.nodes.DeoptimizeNode;
import org.graalvm.compiler.nodes.Invoke;
import org.graalvm.compiler.nodes.InvokeNode;
import org.graalvm.compiler.nodes.NamedLocationIdentity;
import org.graalvm.compiler.nodes.PiNode;
--- 46,57 ----
*** 63,74 ****
--- 62,75 ----
import org.graalvm.compiler.nodes.java.ArrayLengthNode;
import org.graalvm.compiler.nodes.spi.LoweringTool;
import org.graalvm.compiler.nodes.type.StampTool;
import org.graalvm.compiler.nodes.util.GraphUtil;
import org.graalvm.compiler.options.OptionValues;
+ import org.graalvm.compiler.replacements.ReplacementsUtil;
import org.graalvm.compiler.replacements.SnippetCounter;
import org.graalvm.compiler.replacements.SnippetCounter.Group;
+ import org.graalvm.compiler.replacements.SnippetIntegerHistogram;
import org.graalvm.compiler.replacements.SnippetTemplate;
import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
import org.graalvm.compiler.replacements.Snippets;
import org.graalvm.compiler.replacements.nodes.BasicArrayCopyNode;
*** 77,626 ****
import org.graalvm.util.UnmodifiableEconomicMap;
import org.graalvm.word.LocationIdentity;
import org.graalvm.word.WordFactory;
import jdk.vm.ci.code.TargetDescription;
- import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
import jdk.vm.ci.meta.DeoptimizationAction;
import jdk.vm.ci.meta.DeoptimizationReason;
- import jdk.vm.ci.meta.JavaConstant;
import jdk.vm.ci.meta.JavaKind;
import jdk.vm.ci.meta.ResolvedJavaMethod;
import jdk.vm.ci.meta.ResolvedJavaType;
public class ArrayCopySnippets implements Snippets {
! private static int checkArrayType(KlassPointer nonNullHub) {
! int layoutHelper = readLayoutHelper(nonNullHub);
! if (probability(SLOW_PATH_PROBABILITY, layoutHelper >= 0)) {
! DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
! }
! return layoutHelper;
! }
!
! private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length, Counters counters) {
! if (probability(SLOW_PATH_PROBABILITY, srcPos < 0)) {
! counters.checkAIOOBECounter.inc();
! DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
! }
! if (probability(SLOW_PATH_PROBABILITY, destPos < 0)) {
! counters.checkAIOOBECounter.inc();
! DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
! }
! if (probability(SLOW_PATH_PROBABILITY, length < 0)) {
! counters.checkAIOOBECounter.inc();
! DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
! }
! if (probability(SLOW_PATH_PROBABILITY, srcPos > ArrayLengthNode.arrayLength(src) - length)) {
! counters.checkAIOOBECounter.inc();
! DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
! }
! if (probability(SLOW_PATH_PROBABILITY, destPos > ArrayLengthNode.arrayLength(dest) - length)) {
! counters.checkAIOOBECounter.inc();
! DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
! }
! counters.checkSuccessCounter.inc();
}
@Snippet
! public static void arraycopyZeroLengthIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
Object nonNullSrc = GraalDirectives.guardingNonNull(src);
Object nonNullDest = GraalDirectives.guardingNonNull(dest);
! KlassPointer srcHub = loadHub(nonNullSrc);
! KlassPointer destHub = loadHub(nonNullDest);
! checkArrayType(srcHub);
! checkArrayType(destHub);
checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
counters.zeroLengthStaticCounter.inc();
}
@Snippet
! public static void arraycopyExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind, @ConstantParameter SnippetCounter counter,
! @ConstantParameter SnippetCounter copiedCounter, @ConstantParameter Counters counters) {
Object nonNullSrc = GraalDirectives.guardingNonNull(src);
Object nonNullDest = GraalDirectives.guardingNonNull(dest);
checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
! counter.inc();
! copiedCounter.add(length);
ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind);
- if (length == 0) {
- counters.zeroLengthDynamicCounter.inc();
- } else {
- counters.nonZeroLengthDynamicCounter.inc();
- counters.nonZeroLengthDynamicCopiedCounter.add(length);
- }
}
- /**
- * This intrinsic is useful for the case where we know something statically about one of the
- * inputs but not the other.
- */
@Snippet
! public static void arraycopyPredictedExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind,
! @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter, @ConstantParameter Counters counters) {
Object nonNullSrc = GraalDirectives.guardingNonNull(src);
Object nonNullDest = GraalDirectives.guardingNonNull(dest);
! KlassPointer srcHub = loadHub(nonNullSrc);
! KlassPointer destHub = loadHub(nonNullDest);
! if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) {
! DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
! }
checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
! counter.inc();
! copiedCounter.add(length);
! ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind);
! if (length == 0) {
! counters.zeroLengthDynamicCounter.inc();
! } else {
! counters.nonZeroLengthDynamicCounter.inc();
! counters.nonZeroLengthDynamicCopiedCounter.add(length);
! }
! }
! @Snippet
! public static void arraycopyPredictedObjectWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, KlassPointer objectArrayKlass,
! @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter, @ConstantParameter Counters counters) {
! if (length > 0) {
! KlassPointer srcHub = loadHub(PiNode.asNonNullObject(nonNullSrc));
! KlassPointer destHub = loadHub(PiNode.asNonNullObject(nonNullDest));
! if (probability(FAST_PATH_PROBABILITY, srcHub == destHub || destHub == objectArrayKlass)) {
! counter.inc();
! copiedCounter.add(length);
! counters.predictedObjectArrayCopyFastPathCounter.inc();
! counters.predictedObjectArrayCopyFastPathCopiedCounter.add(length);
! ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length);
! } else {
! counters.predictedObjectArrayCopySlowPathCounter.inc();
! counters.predictedObjectArrayCopySlowPathCopiedCounter.add(length);
! System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length);
! }
! }
}
- /**
- * This is the basic template for the full arraycopy checks, including a check that the
- * underlying type is really an array type.
- */
@Snippet
! public static void arraycopySlowPathIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, KlassPointer predictedKlass, @ConstantParameter JavaKind elementKind,
! @ConstantParameter SnippetInfo slowPath, @ConstantParameter Object slowPathArgument, @ConstantParameter Counters counters) {
Object nonNullSrc = GraalDirectives.guardingNonNull(src);
Object nonNullDest = GraalDirectives.guardingNonNull(dest);
! KlassPointer srcHub = loadHub(nonNullSrc);
! KlassPointer destHub = loadHub(nonNullDest);
! checkArrayType(srcHub);
! checkArrayType(destHub);
checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
! if (length == 0) {
! counters.zeroLengthDynamicCounter.inc();
! } else {
! counters.nonZeroLengthDynamicCounter.inc();
! counters.nonZeroLengthDynamicCopiedCounter.add(length);
! }
! ArrayCopySlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, predictedKlass, elementKind, slowPath, slowPathArgument);
}
- /**
- * Snippet for unrolled arraycopy.
- */
@Snippet
! public static void arraycopyUnrolledIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter int unrolledLength, @ConstantParameter JavaKind elementKind,
! @ConstantParameter Counters counters) {
Object nonNullSrc = GraalDirectives.guardingNonNull(src);
Object nonNullDest = GraalDirectives.guardingNonNull(dest);
checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
! if (length == 0) {
! counters.zeroLengthDynamicCounter.inc();
! } else {
! counters.nonZeroLengthDynamicCounter.inc();
! counters.nonZeroLengthDynamicCopiedCounter.add(length);
! }
! ArrayCopyUnrollNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, unrolledLength, elementKind);
}
@Snippet
! public static void checkcastArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, @ConstantParameter Counters counters) {
! if (length > 0) {
! KlassPointer destKlass = loadHub(nonNullDest);
KlassPointer srcKlass = loadHub(nonNullSrc);
! if (probability(SLOW_PATH_PROBABILITY, srcKlass == destKlass)) {
// no storecheck required.
counters.objectCheckcastSameTypeCounter.inc();
counters.objectCheckcastSameTypeCopiedCounter.add(length);
ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length);
} else {
KlassPointer destElemKlass = destKlass.readKlassPointer(arrayClassElementOffset(INJECTED_VMCONFIG), OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION);
Word superCheckOffset = WordFactory.signed(destElemKlass.readInt(superCheckOffsetOffset(INJECTED_VMCONFIG), KLASS_SUPER_CHECK_OFFSET_LOCATION));
! counters.objectCheckcastCounter.inc();
! counters.objectCheckcastCopiedCounter.add(length);
int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false);
! if (copiedElements != 0) {
/*
! * the checkcast stub doesn't throw the ArrayStoreException, but returns the
! * number of copied elements (xor'd with -1).
*/
copiedElements ^= -1;
System.arraycopy(nonNullSrc, srcPos + copiedElements, nonNullDest, destPos + copiedElements, length - copiedElements);
}
}
}
}
! @Snippet
! public static void arraycopyGeneric(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
! Object nonNullSrc = GraalDirectives.guardingNonNull(src);
! Object nonNullDest = GraalDirectives.guardingNonNull(dest);
! KlassPointer srcHub = loadHub(nonNullSrc);
! KlassPointer destHub = loadHub(nonNullDest);
! if (probability(FAST_PATH_PROBABILITY, srcHub.equal(destHub)) && probability(FAST_PATH_PROBABILITY, nonNullSrc != nonNullDest)) {
! int layoutHelper = checkArrayType(srcHub);
! final boolean isObjectArray = ((layoutHelper & layoutHelperElementTypePrimitiveInPlace(INJECTED_VMCONFIG)) == 0);
! checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
! if (probability(FAST_PATH_PROBABILITY, isObjectArray)) {
! counters.genericObjectExactCallCounter.inc();
! counters.genericObjectExactCallCopiedCounter.add(length);
! ArrayCopyCallNode.disjointArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, JavaKind.Object);
! } else {
! counters.genericPrimitiveCallCounter.inc();
! counters.genericPrimitiveCallCopiedCounter.add(length);
! UnsafeArrayCopyNode.arraycopyPrimitive(nonNullSrc, srcPos, nonNullDest, destPos, length, layoutHelper);
}
- } else {
- counters.systemArraycopyCounter.inc();
- counters.systemArraycopyCopiedCounter.add(length);
- System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length);
}
}
! @Fold
! static LocationIdentity getArrayLocation(JavaKind kind) {
! return NamedLocationIdentity.getArrayLocation(kind);
}
! @Snippet
! public static void arraycopyUnrolledWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, @ConstantParameter int length, @ConstantParameter JavaKind elementKind) {
! final int scale = arrayIndexScale(elementKind);
! int arrayBaseOffset = arrayBaseOffset(elementKind);
! LocationIdentity arrayLocation = getArrayLocation(elementKind);
! if (nonNullSrc == nonNullDest && srcPos < destPos) { // bad aliased case
! long start = (long) (length - 1) * scale;
! long i = start;
! ExplodeLoopNode.explodeLoop();
! for (int iteration = 0; iteration < length; iteration++) {
! if (i >= 0) {
! Object a = RawLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation);
! RawStoreNode.storeObject(nonNullDest, arrayBaseOffset + i + (long) destPos * scale, a, elementKind, arrayLocation, false);
! i -= scale;
}
}
} else {
! long end = (long) length * scale;
! long i = 0;
! ExplodeLoopNode.explodeLoop();
! for (int iteration = 0; iteration < length; iteration++) {
! if (i < end) {
! Object a = RawLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation);
! RawStoreNode.storeObject(nonNullDest, arrayBaseOffset + i + (long) destPos * scale, a, elementKind, arrayLocation, false);
! i += scale;
}
}
}
}
static class Counters {
final SnippetCounter checkSuccessCounter;
final SnippetCounter checkAIOOBECounter;
- final SnippetCounter objectCheckcastCounter;
- final SnippetCounter objectCheckcastSameTypeCounter;
- final SnippetCounter predictedObjectArrayCopySlowPathCounter;
- final SnippetCounter predictedObjectArrayCopyFastPathCounter;
-
- final SnippetCounter genericPrimitiveCallCounter;
- final SnippetCounter genericObjectExactCallCounter;
- final SnippetCounter systemArraycopyCounter;
-
final SnippetCounter zeroLengthStaticCounter;
! final SnippetCounter zeroLengthDynamicCounter;
! final SnippetCounter nonZeroLengthDynamicCounter;
! final SnippetCounter nonZeroLengthDynamicCopiedCounter;
! final SnippetCounter genericPrimitiveCallCopiedCounter;
! final SnippetCounter genericObjectExactCallCopiedCounter;
final SnippetCounter systemArraycopyCopiedCounter;
! final SnippetCounter objectCheckcastCopiedCounter;
final SnippetCounter objectCheckcastSameTypeCopiedCounter;
! final SnippetCounter predictedObjectArrayCopySlowPathCopiedCounter;
! final SnippetCounter predictedObjectArrayCopyFastPathCopiedCounter;
final EnumMap<JavaKind, SnippetCounter> arraycopyCallCounters = new EnumMap<>(JavaKind.class);
- final EnumMap<JavaKind, SnippetCounter> arraycopyCounters = new EnumMap<>(JavaKind.class);
-
final EnumMap<JavaKind, SnippetCounter> arraycopyCallCopiedCounters = new EnumMap<>(JavaKind.class);
- final EnumMap<JavaKind, SnippetCounter> arraycopyCopiedCounters = new EnumMap<>(JavaKind.class);
Counters(SnippetCounter.Group.Factory factory) {
final Group checkCounters = factory.createSnippetCounterGroup("System.arraycopy checkInputs");
! final Group counters = factory.createSnippetCounterGroup("System.arraycopy");
! final Group copiedCounters = factory.createSnippetCounterGroup("System.arraycopy copied elements");
! final Group lengthCounters = factory.createSnippetCounterGroup("System.arraycopy 0-length checks");
checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess");
checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE");
! objectCheckcastCounter = new SnippetCounter(counters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays");
! objectCheckcastSameTypeCounter = new SnippetCounter(counters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays");
! predictedObjectArrayCopySlowPathCounter = new SnippetCounter(counters, "Object[]{slow-path}", "used System.arraycopy slow path for predicted Object[] arrays");
! predictedObjectArrayCopyFastPathCounter = new SnippetCounter(counters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays");
! genericPrimitiveCallCounter = new SnippetCounter(counters, "genericPrimitive", "generic arraycopy snippet for primitive arrays");
! genericObjectExactCallCounter = new SnippetCounter(counters, "genericObjectExact", "generic arraycopy snippet for special object arrays");
! systemArraycopyCounter = new SnippetCounter(counters, "genericObject", "call to System.arraycopy");
!
! zeroLengthStaticCounter = new SnippetCounter(lengthCounters, "0-lengthcopy static", "arraycopy where the length is statically 0");
! zeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "0-lengthcopy dynamically", "arraycopy where the length is dynamically 0");
! nonZeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero");
!
! nonZeroLengthDynamicCopiedCounter = new SnippetCounter(copiedCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero");
! genericPrimitiveCallCopiedCounter = new SnippetCounter(copiedCounters, "genericPrimitive", "generic arraycopy snippet for primitive arrays");
! genericObjectExactCallCopiedCounter = new SnippetCounter(copiedCounters, "genericObjectExact", "generic arraycopy snippet for special object arrays");
! systemArraycopyCopiedCounter = new SnippetCounter(copiedCounters, "genericObject", "call to System.arraycopy");
!
! objectCheckcastCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays");
! objectCheckcastSameTypeCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays");
! predictedObjectArrayCopySlowPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{slow-path}",
! "used System.arraycopy slow path for predicted Object[] arrays");
! predictedObjectArrayCopyFastPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays");
! createArraycopyCounter(JavaKind.Byte, counters, copiedCounters);
! createArraycopyCounter(JavaKind.Boolean, counters, copiedCounters);
! createArraycopyCounter(JavaKind.Char, counters, copiedCounters);
! createArraycopyCounter(JavaKind.Short, counters, copiedCounters);
! createArraycopyCounter(JavaKind.Int, counters, copiedCounters);
! createArraycopyCounter(JavaKind.Long, counters, copiedCounters);
! createArraycopyCounter(JavaKind.Float, counters, copiedCounters);
! createArraycopyCounter(JavaKind.Double, counters, copiedCounters);
! createArraycopyCounter(JavaKind.Object, counters, copiedCounters);
}
void createArraycopyCounter(JavaKind kind, Group counters, Group copiedCounters) {
! arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays"));
! arraycopyCounters.put(kind, new SnippetCounter(counters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays"));
!
! arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays"));
! arraycopyCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays"));
}
}
public static class Templates extends SnippetTemplate.AbstractTemplates {
public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target) {
super(options, factories, providers, providers.getSnippetReflection(), target);
this.counters = new Counters(factory);
}
- private ResolvedJavaMethod originalArraycopy() throws GraalError {
- if (originalArraycopy == null) {
- Method method;
- try {
- method = System.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class);
- } catch (NoSuchMethodException | SecurityException e) {
- throw new GraalError(e);
- }
- originalArraycopy = providers.getMetaAccess().lookupJavaMethod(method);
- }
- return originalArraycopy;
- }
-
- private ResolvedJavaMethod originalArraycopy;
-
- private final SnippetInfo checkcastArraycopyWorkSnippet = snippet("checkcastArraycopyWork");
- private final SnippetInfo arraycopyGenericSnippet = snippet("arraycopyGeneric");
-
- private final SnippetInfo arraycopySlowPathIntrinsicSnippet = snippet("arraycopySlowPathIntrinsic");
- private final SnippetInfo arraycopyUnrolledIntrinsicSnippet = snippet("arraycopyUnrolledIntrinsic");
- private final SnippetInfo arraycopyExactIntrinsicSnippet = snippet("arraycopyExactIntrinsic");
- private final SnippetInfo arraycopyZeroLengthIntrinsicSnippet = snippet("arraycopyZeroLengthIntrinsic");
- private final SnippetInfo arraycopyPredictedExactIntrinsicSnippet = snippet("arraycopyPredictedExactIntrinsic");
- private final SnippetInfo arraycopyPredictedObjectWorkSnippet = snippet("arraycopyPredictedObjectWork");
-
- private final SnippetInfo arraycopyUnrolledWorkSnippet = snippet("arraycopyUnrolledWork");
-
- private final Counters counters;
-
protected SnippetInfo snippet(String methodName) {
SnippetInfo info = snippet(ArrayCopySnippets.class, methodName, LocationIdentity.any());
info.setOriginalMethod(originalArraycopy());
return info;
}
- public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy) {
- return selectComponentKind(arraycopy, true);
- }
-
- public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy, boolean exact) {
- ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp());
- ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp());
-
- if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) {
- if (!exact) {
- JavaKind component = getComponentKind(srcType);
- if (component != null) {
- return component;
- }
- return getComponentKind(destType);
- }
- return null;
- }
- if (exact) {
- if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) {
- return null;
- }
- if (!arraycopy.isExact()) {
- return null;
- }
- }
- return srcType.getComponentType().getJavaKind();
- }
-
- private static JavaKind getComponentKind(ResolvedJavaType type) {
- if (type != null && type.isArray()) {
- return type.getComponentType().getJavaKind();
- }
- return null;
- }
-
- private static boolean shouldUnroll(ValueNode length) {
- return length.isConstant() && length.asJavaConstant().asInt() <= 8 && length.asJavaConstant().asInt() != 0;
- }
-
public void lower(ArrayCopyNode arraycopy, LoweringTool tool) {
! JavaKind componentKind = selectComponentKind(arraycopy);
! SnippetInfo snippetInfo = null;
! SnippetInfo slowPathSnippetInfo = null;
! Object slowPathArgument = null;
- if (arraycopy.getLength().isConstant() && arraycopy.getLength().asJavaConstant().asLong() == 0) {
- snippetInfo = arraycopyZeroLengthIntrinsicSnippet;
- } else if (arraycopy.isExact()) {
- snippetInfo = arraycopyExactIntrinsicSnippet;
- if (shouldUnroll(arraycopy.getLength())) {
- snippetInfo = arraycopyUnrolledIntrinsicSnippet;
- }
- } else {
- if (componentKind == JavaKind.Object) {
ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp());
ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp());
ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType();
ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType();
! if (srcComponentType != null && destComponentType != null && !srcComponentType.isPrimitive() && !destComponentType.isPrimitive()) {
! snippetInfo = arraycopySlowPathIntrinsicSnippet;
! slowPathSnippetInfo = checkcastArraycopyWorkSnippet;
! slowPathArgument = LocationIdentity.any();
! /*
! * Because this snippet has to use Sysytem.arraycopy as a slow path, we must
! * pretend to kill any() so clear the componentKind.
! */
! componentKind = null;
! }
! }
! if (componentKind == null && snippetInfo == null) {
! JavaKind predictedKind = selectComponentKind(arraycopy, false);
! if (predictedKind != null) {
! /*
! * At least one array is of a known type requiring no store checks, so
! * assume the other is of the same type. Generally this is working around
! * deficiencies in our propagation of type information.
! */
! componentKind = predictedKind;
! if (predictedKind == JavaKind.Object) {
! snippetInfo = arraycopySlowPathIntrinsicSnippet;
! slowPathSnippetInfo = arraycopyPredictedObjectWorkSnippet;
! slowPathArgument = predictedKind;
! componentKind = null;
! } else {
! snippetInfo = arraycopyPredictedExactIntrinsicSnippet;
! }
}
}
- if (snippetInfo == null) {
- snippetInfo = arraycopyGenericSnippet;
}
}
Arguments args = new Arguments(snippetInfo, arraycopy.graph().getGuardsStage(), tool.getLoweringStage());
args.add("src", arraycopy.getSource());
args.add("srcPos", arraycopy.getSourcePosition());
args.add("dest", arraycopy.getDestination());
args.add("destPos", arraycopy.getDestinationPosition());
args.add("length", arraycopy.getLength());
! if (snippetInfo == arraycopyUnrolledIntrinsicSnippet) {
args.addConst("unrolledLength", arraycopy.getLength().asJavaConstant().asInt());
- args.addConst("elementKind", componentKind != null ? componentKind : JavaKind.Illegal);
- } else if (snippetInfo == arraycopySlowPathIntrinsicSnippet) {
- ValueNode predictedKlass = null;
- if (slowPathArgument == arraycopyPredictedObjectWorkSnippet) {
- HotSpotResolvedObjectType arrayClass = (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(Object[].class);
- predictedKlass = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayClass.klass(), tool.getMetaAccess(), arraycopy.graph());
- } else {
- predictedKlass = ConstantNode.forConstant(KlassPointerStamp.klassAlwaysNull(), JavaConstant.NULL_POINTER, tool.getMetaAccess(), arraycopy.graph());
}
! args.add("predictedKlass", predictedKlass);
! args.addConst("elementKind", componentKind != null ? componentKind : JavaKind.Illegal);
! args.addConst("slowPath", slowPathSnippetInfo);
! assert slowPathArgument != null;
! args.addConst("slowPathArgument", slowPathArgument);
! } else if (snippetInfo == arraycopyExactIntrinsicSnippet || snippetInfo == arraycopyPredictedExactIntrinsicSnippet) {
! assert componentKind != null;
! args.addConst("elementKind", componentKind);
! args.addConst("counter", counters.arraycopyCallCounters.get(componentKind));
! args.addConst("copiedCounter", counters.arraycopyCallCopiedCounters.get(componentKind));
}
args.addConst("counters", counters);
instantiate(args, arraycopy);
}
! public void lower(ArrayCopySlowPathNode arraycopy, LoweringTool tool) {
StructuredGraph graph = arraycopy.graph();
if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
! // Can't be lowered yet
return;
}
SnippetInfo snippetInfo = arraycopy.getSnippet();
Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage());
! args.add("nonNullSrc", arraycopy.getSource());
args.add("srcPos", arraycopy.getSourcePosition());
! args.add("nonNullDest", arraycopy.getDestination());
args.add("destPos", arraycopy.getDestinationPosition());
- if (snippetInfo == arraycopyUnrolledWorkSnippet) {
- args.addConst("length", ((Integer) arraycopy.getArgument()).intValue());
- args.addConst("elementKind", arraycopy.getElementKind());
- } else {
args.add("length", arraycopy.getLength());
- }
- if (snippetInfo == arraycopyPredictedObjectWorkSnippet) {
- args.add("objectArrayKlass", arraycopy.getPredictedKlass());
- args.addConst("counter", counters.arraycopyCallCounters.get(JavaKind.Object));
- args.addConst("copiedCounter", counters.arraycopyCallCopiedCounters.get(JavaKind.Object));
args.addConst("counters", counters);
- }
instantiate(args, arraycopy);
}
! public void lower(ArrayCopyUnrollNode arraycopy, LoweringTool tool) {
! StructuredGraph graph = arraycopy.graph();
! if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
! // Can't be lowered yet
! return;
}
! SnippetInfo snippetInfo = arraycopyUnrolledWorkSnippet;
! Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage());
! args.add("nonNullSrc", arraycopy.getSource());
! args.add("srcPos", arraycopy.getSourcePosition());
! args.add("nonNullDest", arraycopy.getDestination());
! args.add("destPos", arraycopy.getDestinationPosition());
! args.addConst("length", arraycopy.getUnrollLength());
! args.addConst("elementKind", arraycopy.getElementKind());
! template(graph.getDebug(), args).instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args);
}
/**
* Instantiate the snippet template and fix up the FrameState of any Invokes of
* System.arraycopy and propagate the captured bci in the ArrayCopySlowPathNode.
--- 78,519 ----
import org.graalvm.util.UnmodifiableEconomicMap;
import org.graalvm.word.LocationIdentity;
import org.graalvm.word.WordFactory;
import jdk.vm.ci.code.TargetDescription;
import jdk.vm.ci.meta.DeoptimizationAction;
import jdk.vm.ci.meta.DeoptimizationReason;
import jdk.vm.ci.meta.JavaKind;
import jdk.vm.ci.meta.ResolvedJavaMethod;
import jdk.vm.ci.meta.ResolvedJavaType;
public class ArrayCopySnippets implements Snippets {
! private enum ArrayCopyTypeCheck {
! UNDEFINED_ARRAY_TYPE_CHECK,
! // we know that both objects are arrays and have the same type
! NO_ARRAY_TYPE_CHECK,
! // can be used when we know that one of the objects is a primitive array
! HUB_BASED_ARRAY_TYPE_CHECK,
! // must be used when we don't have sufficient information to use one of the others
! LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK
}
@Snippet
! public static void arraycopyZeroLengthSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
! @ConstantParameter Counters counters) {
Object nonNullSrc = GraalDirectives.guardingNonNull(src);
Object nonNullDest = GraalDirectives.guardingNonNull(dest);
! checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
counters.zeroLengthStaticCounter.inc();
}
@Snippet
! public static void arraycopyExactSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
! @ConstantParameter JavaKind elementKind, @ConstantParameter SnippetCounter elementKindCounter, @ConstantParameter SnippetCounter elementKindCopiedCounter,
! @ConstantParameter Counters counters) {
Object nonNullSrc = GraalDirectives.guardingNonNull(src);
Object nonNullDest = GraalDirectives.guardingNonNull(dest);
+ checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
! incrementLengthCounter(length, counters);
!
! elementKindCounter.inc();
! elementKindCopiedCounter.add(length);
ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind);
}
@Snippet
! public static void arraycopyUnrolledSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
! @ConstantParameter JavaKind elementKind, @ConstantParameter int unrolledLength, @ConstantParameter Counters counters) {
Object nonNullSrc = GraalDirectives.guardingNonNull(src);
Object nonNullDest = GraalDirectives.guardingNonNull(dest);
! checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
! incrementLengthCounter(length, counters);
! unrolledArraycopyWork(nonNullSrc, srcPos, nonNullDest, destPos, unrolledLength, elementKind);
}
@Snippet
! public static void arraycopyCheckcastSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
! @ConstantParameter Counters counters, @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind) {
Object nonNullSrc = GraalDirectives.guardingNonNull(src);
Object nonNullDest = GraalDirectives.guardingNonNull(dest);
! checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
! incrementLengthCounter(length, counters);
!
! ArrayCopyWithSlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind);
}
@Snippet
! public static void arraycopyGenericSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, @ConstantParameter Counters counters,
! @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind) {
Object nonNullSrc = GraalDirectives.guardingNonNull(src);
Object nonNullDest = GraalDirectives.guardingNonNull(dest);
+ checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
! incrementLengthCounter(length, counters);
!
! ArrayCopyWithSlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind);
}
@Snippet
! public static void arraycopyNativeSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
! // all checks are done in the native method, so no need to emit additional checks here
! incrementLengthCounter(length, counters);
! counters.systemArraycopyCounter.inc();
! counters.systemArraycopyCopiedCounter.add(length);
!
! System.arraycopy(src, srcPos, dest, destPos, length);
! }
!
! @Fold
! static LocationIdentity getArrayLocation(JavaKind kind) {
! return NamedLocationIdentity.getArrayLocation(kind);
! }
!
! private static void unrolledArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, JavaKind elementKind) {
! int scale = arrayIndexScale(elementKind);
! int arrayBaseOffset = arrayBaseOffset(elementKind);
! LocationIdentity arrayLocation = getArrayLocation(elementKind);
!
! long sourceOffset = arrayBaseOffset + (long) srcPos * scale;
! long destOffset = arrayBaseOffset + (long) destPos * scale;
! long position = 0;
! long delta = scale;
! if (probability(NOT_FREQUENT_PROBABILITY, nonNullSrc == nonNullDest && srcPos < destPos)) {
! // bad aliased case so we need to copy the array from back to front
! position = (long) (length - 1) * scale;
! delta = -delta;
! }
!
! // the length was already checked before - we can emit unconditional instructions
! ExplodeLoopNode.explodeLoop();
! for (int iteration = 0; iteration < length; iteration++) {
! Object value = RawLoadNode.load(nonNullSrc, sourceOffset + position, elementKind, arrayLocation);
! RawStoreNode.storeObject(nonNullDest, destOffset + position, value, elementKind, arrayLocation, false);
! position += delta;
! }
! }
!
! @Snippet(allowPartialIntrinsicArgumentMismatch = true)
! public static void checkcastArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
! if (probability(FREQUENT_PROBABILITY, length > 0)) {
! Object nonNullSrc = PiNode.asNonNullObject(src);
! Object nonNullDest = PiNode.asNonNullObject(dest);
KlassPointer srcKlass = loadHub(nonNullSrc);
! KlassPointer destKlass = loadHub(nonNullDest);
! if (probability(LIKELY_PROBABILITY, srcKlass == destKlass)) {
// no storecheck required.
counters.objectCheckcastSameTypeCounter.inc();
counters.objectCheckcastSameTypeCopiedCounter.add(length);
ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length);
} else {
KlassPointer destElemKlass = destKlass.readKlassPointer(arrayClassElementOffset(INJECTED_VMCONFIG), OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION);
Word superCheckOffset = WordFactory.signed(destElemKlass.readInt(superCheckOffsetOffset(INJECTED_VMCONFIG), KLASS_SUPER_CHECK_OFFSET_LOCATION));
!
! counters.objectCheckcastDifferentTypeCounter.inc();
! counters.objectCheckcastDifferentTypeCopiedCounter.add(length);
!
int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false);
! if (probability(SLOW_PATH_PROBABILITY, copiedElements != 0)) {
/*
! * the stub doesn't throw the ArrayStoreException, but returns the number of
! * copied elements (xor'd with -1).
*/
copiedElements ^= -1;
System.arraycopy(nonNullSrc, srcPos + copiedElements, nonNullDest, destPos + copiedElements, length - copiedElements);
}
}
}
}
! @Snippet(allowPartialIntrinsicArgumentMismatch = true)
! public static void genericArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
! if (probability(FREQUENT_PROBABILITY, length > 0)) {
! counters.genericArraycopyDifferentTypeCounter.inc();
! counters.genericArraycopyDifferentTypeCopiedCounter.add(length);
! int copiedElements = GenericArrayCopyCallNode.genericArraycopy(src, srcPos, dest, destPos, length);
! if (probability(SLOW_PATH_PROBABILITY, copiedElements != 0)) {
! /*
! * the stub doesn't throw the ArrayStoreException, but returns the number of copied
! * elements (xor'd with -1).
! */
! copiedElements ^= -1;
! System.arraycopy(src, srcPos + copiedElements, dest, destPos + copiedElements, length - copiedElements);
}
}
}
! private static void incrementLengthCounter(int length, Counters counters) {
! counters.lengthHistogram.inc(length);
}
! private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length, Counters counters) {
! if (probability(SLOW_PATH_PROBABILITY, srcPos < 0) ||
! probability(SLOW_PATH_PROBABILITY, destPos < 0) ||
! probability(SLOW_PATH_PROBABILITY, length < 0) ||
! probability(SLOW_PATH_PROBABILITY, srcPos > ArrayLengthNode.arrayLength(src) - length) ||
! probability(SLOW_PATH_PROBABILITY, destPos > ArrayLengthNode.arrayLength(dest) - length)) {
! counters.checkAIOOBECounter.inc();
! DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
}
+ counters.checkSuccessCounter.inc();
}
+
+ private static void checkArrayTypes(Object nonNullSrc, Object nonNullDest, ArrayCopyTypeCheck arrayTypeCheck) {
+ if (arrayTypeCheck == ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK) {
+ // nothing to do
+ } else if (arrayTypeCheck == ArrayCopyTypeCheck.HUB_BASED_ARRAY_TYPE_CHECK) {
+ KlassPointer srcHub = loadHub(nonNullSrc);
+ KlassPointer destHub = loadHub(nonNullDest);
+ if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) {
+ DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
+ }
+ } else if (arrayTypeCheck == ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK) {
+ KlassPointer srcHub = loadHub(nonNullSrc);
+ KlassPointer destHub = loadHub(nonNullDest);
+ checkArrayType(srcHub);
+ checkArrayType(destHub);
} else {
! ReplacementsUtil.staticAssert(false, "unknown array type check");
}
}
+
+ private static int checkArrayType(KlassPointer nonNullHub) {
+ int layoutHelper = readLayoutHelper(nonNullHub);
+ if (probability(SLOW_PATH_PROBABILITY, layoutHelper >= 0)) {
+ DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
}
+ return layoutHelper;
}
static class Counters {
final SnippetCounter checkSuccessCounter;
final SnippetCounter checkAIOOBECounter;
final SnippetCounter zeroLengthStaticCounter;
! final SnippetIntegerHistogram lengthHistogram;
! final SnippetCounter systemArraycopyCounter;
final SnippetCounter systemArraycopyCopiedCounter;
! final SnippetCounter genericArraycopyDifferentTypeCopiedCounter;
! final SnippetCounter genericArraycopyDifferentTypeCounter;
!
final SnippetCounter objectCheckcastSameTypeCopiedCounter;
! final SnippetCounter objectCheckcastSameTypeCounter;
! final SnippetCounter objectCheckcastDifferentTypeCopiedCounter;
! final SnippetCounter objectCheckcastDifferentTypeCounter;
final EnumMap<JavaKind, SnippetCounter> arraycopyCallCounters = new EnumMap<>(JavaKind.class);
final EnumMap<JavaKind, SnippetCounter> arraycopyCallCopiedCounters = new EnumMap<>(JavaKind.class);
Counters(SnippetCounter.Group.Factory factory) {
final Group checkCounters = factory.createSnippetCounterGroup("System.arraycopy checkInputs");
! final Group callCounters = factory.createSnippetCounterGroup("System.arraycopy calls");
! final Group copiedElementsCounters = factory.createSnippetCounterGroup("System.arraycopy copied elements");
! final Group lengthCounters = factory.createSnippetCounterGroup("System.arraycopy with 0-length");
checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess");
checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE");
! zeroLengthStaticCounter = new SnippetCounter(lengthCounters, "0-length copy static", "calls where the length is statically 0");
! lengthHistogram = new SnippetIntegerHistogram(lengthCounters, 2, "length", "length");
!
! systemArraycopyCounter = new SnippetCounter(callCounters, "native System.arraycopy", "JNI-based System.arraycopy call");
! systemArraycopyCopiedCounter = new SnippetCounter(copiedElementsCounters, "native System.arraycopy", "JNI-based System.arraycopy call");
!
! genericArraycopyDifferentTypeCounter = new SnippetCounter(callCounters, "generic[] stub", "generic arraycopy stub");
! genericArraycopyDifferentTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "generic[] stub", "generic arraycopy stub");
!
! objectCheckcastSameTypeCounter = new SnippetCounter(callCounters, "checkcast object[] (same-type)", "checkcast object[] stub but src.klass == dest.klass Object[] arrays");
! objectCheckcastSameTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "checkcast object[] (same-type)", "checkcast object[] stub but src.klass == dest.klass Object[] arrays");
! objectCheckcastDifferentTypeCounter = new SnippetCounter(callCounters, "checkcast object[] (store-check)", "checkcast object[] stub with store check");
! objectCheckcastDifferentTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "checkcast object[] (store-check)", "checkcast object[] stub with store check");
!
! createArraycopyCounter(JavaKind.Byte, callCounters, copiedElementsCounters);
! createArraycopyCounter(JavaKind.Boolean, callCounters, copiedElementsCounters);
! createArraycopyCounter(JavaKind.Char, callCounters, copiedElementsCounters);
! createArraycopyCounter(JavaKind.Short, callCounters, copiedElementsCounters);
! createArraycopyCounter(JavaKind.Int, callCounters, copiedElementsCounters);
! createArraycopyCounter(JavaKind.Long, callCounters, copiedElementsCounters);
! createArraycopyCounter(JavaKind.Float, callCounters, copiedElementsCounters);
! createArraycopyCounter(JavaKind.Double, callCounters, copiedElementsCounters);
! createArraycopyCounter(JavaKind.Object, callCounters, copiedElementsCounters);
}
void createArraycopyCounter(JavaKind kind, Group counters, Group copiedCounters) {
! arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[] stub", "arraycopy call for " + kind + "[] arrays"));
! arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[] stub", "arraycopy call for " + kind + "[] arrays"));
}
}
public static class Templates extends SnippetTemplate.AbstractTemplates {
+ private final SnippetInfo arraycopyGenericSnippet = snippet("arraycopyGenericSnippet");
+ private final SnippetInfo arraycopyUnrolledSnippet = snippet("arraycopyUnrolledSnippet");
+ private final SnippetInfo arraycopyExactSnippet = snippet("arraycopyExactSnippet");
+ private final SnippetInfo arraycopyZeroLengthSnippet = snippet("arraycopyZeroLengthSnippet");
+ private final SnippetInfo arraycopyCheckcastSnippet = snippet("arraycopyCheckcastSnippet");
+ private final SnippetInfo arraycopyNativeSnippet = snippet("arraycopyNativeSnippet");
+
+ private final SnippetInfo checkcastArraycopyWithSlowPathWork = snippet("checkcastArraycopyWithSlowPathWork");
+ private final SnippetInfo genericArraycopyWithSlowPathWork = snippet("genericArraycopyWithSlowPathWork");
+
+ private ResolvedJavaMethod originalArraycopy;
+ private final Counters counters;
public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target) {
super(options, factories, providers, providers.getSnippetReflection(), target);
this.counters = new Counters(factory);
}
protected SnippetInfo snippet(String methodName) {
SnippetInfo info = snippet(ArrayCopySnippets.class, methodName, LocationIdentity.any());
info.setOriginalMethod(originalArraycopy());
return info;
}
public void lower(ArrayCopyNode arraycopy, LoweringTool tool) {
! JavaKind elementKind = selectComponentKind(arraycopy);
! SnippetInfo snippetInfo;
! ArrayCopyTypeCheck arrayTypeCheck;
ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp());
ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp());
+ if (!canBeArray(srcType) || !canBeArray(destType)) {
+ // at least one of the objects is definitely not an array - use the native call
+ // right away as the copying will fail anyways
+ snippetInfo = arraycopyNativeSnippet;
+ arrayTypeCheck = ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK;
+ } else {
ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType();
ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType();
!
! if (arraycopy.isExact()) {
! // there is a sufficient type match - we don't need any additional type checks
! snippetInfo = arraycopyExactSnippet;
! arrayTypeCheck = ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK;
! } else if (srcComponentType == null && destComponentType == null) {
! // we don't know anything about the types - use the generic copying
! snippetInfo = arraycopyGenericSnippet;
! arrayTypeCheck = ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK;
! } else if (srcComponentType != null && destComponentType != null) {
! if (!srcComponentType.isPrimitive() && !destComponentType.isPrimitive()) {
! // it depends on the array content if the copy succeeds - we need
! // a type check for every store
! snippetInfo = arraycopyCheckcastSnippet;
! arrayTypeCheck = ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK;
! } else {
! // one object is an object array, the other one is a primitive array.
! // this copy will always fail - use the native call right away
! assert !srcComponentType.equals(destComponentType) : "must be handled by arraycopy.isExact()";
! snippetInfo = arraycopyNativeSnippet;
! arrayTypeCheck = ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK;
! }
! } else {
! ResolvedJavaType nonNullComponentType = srcComponentType != null ? srcComponentType : destComponentType;
! if (nonNullComponentType.isPrimitive()) {
! // one involved object is a primitive array - we can safely assume that we
! // are copying primitive arrays
! snippetInfo = arraycopyExactSnippet;
! arrayTypeCheck = ArrayCopyTypeCheck.HUB_BASED_ARRAY_TYPE_CHECK;
! elementKind = nonNullComponentType.getJavaKind();
! } else {
! // one involved object is an object array - we can safely assume that we are
! // copying object arrays that might require a store check
! snippetInfo = arraycopyCheckcastSnippet;
! arrayTypeCheck = ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK;
}
}
}
+
+ // a few special cases that are easier to handle when all other variables already have a
+ // value
+ if (arraycopy.getLength().isConstant() && arraycopy.getLength().asJavaConstant().asLong() == 0) {
+ snippetInfo = arraycopyZeroLengthSnippet;
+ } else if (snippetInfo == arraycopyExactSnippet && shouldUnroll(arraycopy.getLength())) {
+ snippetInfo = arraycopyUnrolledSnippet;
}
+
+ // create the snippet
Arguments args = new Arguments(snippetInfo, arraycopy.graph().getGuardsStage(), tool.getLoweringStage());
args.add("src", arraycopy.getSource());
args.add("srcPos", arraycopy.getSourcePosition());
args.add("dest", arraycopy.getDestination());
args.add("destPos", arraycopy.getDestinationPosition());
args.add("length", arraycopy.getLength());
! if (snippetInfo != arraycopyNativeSnippet) {
! assert arrayTypeCheck != ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK;
! args.addConst("arrayTypeCheck", arrayTypeCheck);
! }
! if (snippetInfo == arraycopyUnrolledSnippet) {
! args.addConst("elementKind", elementKind != null ? elementKind : JavaKind.Illegal);
args.addConst("unrolledLength", arraycopy.getLength().asJavaConstant().asInt());
}
! if (snippetInfo == arraycopyExactSnippet) {
! assert elementKind != null;
! args.addConst("elementKind", elementKind);
! args.addConst("elementKindCounter", counters.arraycopyCallCounters.get(elementKind));
! args.addConst("elementKindCopiedCounter", counters.arraycopyCallCopiedCounters.get(elementKind));
}
args.addConst("counters", counters);
+ if (snippetInfo == arraycopyCheckcastSnippet) {
+ args.addConst("workSnippet", checkcastArraycopyWithSlowPathWork);
+ args.addConst("elementKind", JavaKind.Illegal);
+ }
+ if (snippetInfo == arraycopyGenericSnippet) {
+ args.addConst("workSnippet", genericArraycopyWithSlowPathWork);
+ args.addConst("elementKind", JavaKind.Illegal);
+ }
+
instantiate(args, arraycopy);
}
! public void lower(ArrayCopyWithSlowPathNode arraycopy, LoweringTool tool) {
StructuredGraph graph = arraycopy.graph();
if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
! // if an arraycopy contains a slow path, we can't lower it right away
return;
}
+
SnippetInfo snippetInfo = arraycopy.getSnippet();
Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage());
! args.add("src", arraycopy.getSource());
args.add("srcPos", arraycopy.getSourcePosition());
! args.add("dest", arraycopy.getDestination());
args.add("destPos", arraycopy.getDestinationPosition());
args.add("length", arraycopy.getLength());
args.addConst("counters", counters);
instantiate(args, arraycopy);
}
! private static boolean canBeArray(ResolvedJavaType type) {
! return type == null || type.isJavaLangObject() || type.isArray();
}
!
! public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy) {
! ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp());
! ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp());
!
! if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) {
! return null;
! }
! if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) {
! return null;
! }
! if (!arraycopy.isExact()) {
! return null;
! }
! return srcType.getComponentType().getJavaKind();
! }
!
! private static boolean shouldUnroll(ValueNode length) {
! return length.isConstant() && length.asJavaConstant().asInt() <= 8 && length.asJavaConstant().asInt() != 0;
}
/**
* Instantiate the snippet template and fix up the FrameState of any Invokes of
* System.arraycopy and propagate the captured bci in the ArrayCopySlowPathNode.
*** 648,663 ****
} else {
assert arraycopy.stateAfter() != null : arraycopy;
newInvoke.setStateAfter(arraycopy.stateAfter());
}
graph.replaceFixedWithFixed((InvokeNode) invoke.asNode(), newInvoke);
! } else if (originalNode instanceof ArrayCopySlowPathNode) {
! ArrayCopySlowPathNode slowPath = (ArrayCopySlowPathNode) replacements.get(originalNode);
assert arraycopy.stateAfter() != null : arraycopy;
assert slowPath.stateAfter() == arraycopy.stateAfter();
slowPath.setBci(arraycopy.getBci());
}
}
GraphUtil.killCFG(arraycopy);
}
}
}
--- 541,569 ----
} else {
assert arraycopy.stateAfter() != null : arraycopy;
newInvoke.setStateAfter(arraycopy.stateAfter());
}
graph.replaceFixedWithFixed((InvokeNode) invoke.asNode(), newInvoke);
! } else if (originalNode instanceof ArrayCopyWithSlowPathNode) {
! ArrayCopyWithSlowPathNode slowPath = (ArrayCopyWithSlowPathNode) replacements.get(originalNode);
assert arraycopy.stateAfter() != null : arraycopy;
assert slowPath.stateAfter() == arraycopy.stateAfter();
slowPath.setBci(arraycopy.getBci());
}
}
GraphUtil.killCFG(arraycopy);
}
+
+ private ResolvedJavaMethod originalArraycopy() throws GraalError {
+ if (originalArraycopy == null) {
+ Method method;
+ try {
+ method = System.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class);
+ } catch (NoSuchMethodException | SecurityException e) {
+ throw new GraalError(e);
+ }
+ originalArraycopy = providers.getMetaAccess().lookupJavaMethod(method);
+ }
+ return originalArraycopy;
+ }
}
}
< prev index next >