< prev index next >

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/arraycopy/ArrayCopySnippets.java

Print this page

        

@@ -26,15 +26,16 @@
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.KLASS_SUPER_CHECK_OFFSET_LOCATION;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayClassElementOffset;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayIndexScale;
-import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperElementTypePrimitiveInPlace;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadHub;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.superCheckOffsetOffset;
-import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
+import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
+import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.LIKELY_PROBABILITY;
+import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY;
 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
 
 import java.lang.reflect.Method;
 import java.util.EnumMap;

@@ -45,14 +46,12 @@
 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
 import org.graalvm.compiler.debug.DebugHandlersFactory;
 import org.graalvm.compiler.debug.GraalError;
 import org.graalvm.compiler.graph.Node;
 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
-import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
 import org.graalvm.compiler.hotspot.word.KlassPointer;
 import org.graalvm.compiler.nodes.CallTargetNode;
-import org.graalvm.compiler.nodes.ConstantNode;
 import org.graalvm.compiler.nodes.DeoptimizeNode;
 import org.graalvm.compiler.nodes.Invoke;
 import org.graalvm.compiler.nodes.InvokeNode;
 import org.graalvm.compiler.nodes.NamedLocationIdentity;
 import org.graalvm.compiler.nodes.PiNode;

@@ -63,12 +62,14 @@
 import org.graalvm.compiler.nodes.java.ArrayLengthNode;
 import org.graalvm.compiler.nodes.spi.LoweringTool;
 import org.graalvm.compiler.nodes.type.StampTool;
 import org.graalvm.compiler.nodes.util.GraphUtil;
 import org.graalvm.compiler.options.OptionValues;
+import org.graalvm.compiler.replacements.ReplacementsUtil;
 import org.graalvm.compiler.replacements.SnippetCounter;
 import org.graalvm.compiler.replacements.SnippetCounter.Group;
+import org.graalvm.compiler.replacements.SnippetIntegerHistogram;
 import org.graalvm.compiler.replacements.SnippetTemplate;
 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
 import org.graalvm.compiler.replacements.Snippets;
 import org.graalvm.compiler.replacements.nodes.BasicArrayCopyNode;

@@ -77,550 +78,442 @@
 import org.graalvm.util.UnmodifiableEconomicMap;
 import org.graalvm.word.LocationIdentity;
 import org.graalvm.word.WordFactory;
 
 import jdk.vm.ci.code.TargetDescription;
-import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
 import jdk.vm.ci.meta.DeoptimizationAction;
 import jdk.vm.ci.meta.DeoptimizationReason;
-import jdk.vm.ci.meta.JavaConstant;
 import jdk.vm.ci.meta.JavaKind;
 import jdk.vm.ci.meta.ResolvedJavaMethod;
 import jdk.vm.ci.meta.ResolvedJavaType;
 
 public class ArrayCopySnippets implements Snippets {
 
-    private static int checkArrayType(KlassPointer nonNullHub) {
-        int layoutHelper = readLayoutHelper(nonNullHub);
-        if (probability(SLOW_PATH_PROBABILITY, layoutHelper >= 0)) {
-            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
-        }
-        return layoutHelper;
-    }
-
-    private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length, Counters counters) {
-        if (probability(SLOW_PATH_PROBABILITY, srcPos < 0)) {
-            counters.checkAIOOBECounter.inc();
-            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
-        }
-        if (probability(SLOW_PATH_PROBABILITY, destPos < 0)) {
-            counters.checkAIOOBECounter.inc();
-            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
-        }
-        if (probability(SLOW_PATH_PROBABILITY, length < 0)) {
-            counters.checkAIOOBECounter.inc();
-            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
-        }
-        if (probability(SLOW_PATH_PROBABILITY, srcPos > ArrayLengthNode.arrayLength(src) - length)) {
-            counters.checkAIOOBECounter.inc();
-            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
-        }
-        if (probability(SLOW_PATH_PROBABILITY, destPos > ArrayLengthNode.arrayLength(dest) - length)) {
-            counters.checkAIOOBECounter.inc();
-            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
-        }
-        counters.checkSuccessCounter.inc();
+    private enum ArrayCopyTypeCheck {
+        UNDEFINED_ARRAY_TYPE_CHECK,
+        // we know that both objects are arrays and have the same type
+        NO_ARRAY_TYPE_CHECK,
+        // can be used when we know that one of the objects is a primitive array
+        HUB_BASED_ARRAY_TYPE_CHECK,
+        // must be used when we don't have sufficient information to use one of the others
+        LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK
     }
 
     @Snippet
-    public static void arraycopyZeroLengthIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
+    public static void arraycopyZeroLengthSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
+                    @ConstantParameter Counters counters) {
         Object nonNullSrc = GraalDirectives.guardingNonNull(src);
         Object nonNullDest = GraalDirectives.guardingNonNull(dest);
-        KlassPointer srcHub = loadHub(nonNullSrc);
-        KlassPointer destHub = loadHub(nonNullDest);
-        checkArrayType(srcHub);
-        checkArrayType(destHub);
+        checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
         checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
         counters.zeroLengthStaticCounter.inc();
     }
 
     @Snippet
-    public static void arraycopyExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind, @ConstantParameter SnippetCounter counter,
-                    @ConstantParameter SnippetCounter copiedCounter, @ConstantParameter Counters counters) {
+    public static void arraycopyExactSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
+                    @ConstantParameter JavaKind elementKind, @ConstantParameter SnippetCounter elementKindCounter, @ConstantParameter SnippetCounter elementKindCopiedCounter,
+                    @ConstantParameter Counters counters) {
         Object nonNullSrc = GraalDirectives.guardingNonNull(src);
         Object nonNullDest = GraalDirectives.guardingNonNull(dest);
+        checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
         checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
-        counter.inc();
-        copiedCounter.add(length);
+        incrementLengthCounter(length, counters);
+
+        elementKindCounter.inc();
+        elementKindCopiedCounter.add(length);
         ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind);
-        if (length == 0) {
-            counters.zeroLengthDynamicCounter.inc();
-        } else {
-            counters.nonZeroLengthDynamicCounter.inc();
-            counters.nonZeroLengthDynamicCopiedCounter.add(length);
-        }
     }
 
-    /**
-     * This intrinsic is useful for the case where we know something statically about one of the
-     * inputs but not the other.
-     */
     @Snippet
-    public static void arraycopyPredictedExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind,
-                    @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter, @ConstantParameter Counters counters) {
+    public static void arraycopyUnrolledSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
+                    @ConstantParameter JavaKind elementKind, @ConstantParameter int unrolledLength, @ConstantParameter Counters counters) {
         Object nonNullSrc = GraalDirectives.guardingNonNull(src);
         Object nonNullDest = GraalDirectives.guardingNonNull(dest);
-        KlassPointer srcHub = loadHub(nonNullSrc);
-        KlassPointer destHub = loadHub(nonNullDest);
-        if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) {
-            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
-        }
+        checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
         checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
-        counter.inc();
-        copiedCounter.add(length);
-        ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind);
-        if (length == 0) {
-            counters.zeroLengthDynamicCounter.inc();
-        } else {
-            counters.nonZeroLengthDynamicCounter.inc();
-            counters.nonZeroLengthDynamicCopiedCounter.add(length);
-        }
-    }
+        incrementLengthCounter(length, counters);
 
-    @Snippet
-    public static void arraycopyPredictedObjectWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, KlassPointer objectArrayKlass,
-                    @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter, @ConstantParameter Counters counters) {
-        if (length > 0) {
-            KlassPointer srcHub = loadHub(PiNode.asNonNullObject(nonNullSrc));
-            KlassPointer destHub = loadHub(PiNode.asNonNullObject(nonNullDest));
-            if (probability(FAST_PATH_PROBABILITY, srcHub == destHub || destHub == objectArrayKlass)) {
-                counter.inc();
-                copiedCounter.add(length);
-                counters.predictedObjectArrayCopyFastPathCounter.inc();
-                counters.predictedObjectArrayCopyFastPathCopiedCounter.add(length);
-                ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length);
-            } else {
-                counters.predictedObjectArrayCopySlowPathCounter.inc();
-                counters.predictedObjectArrayCopySlowPathCopiedCounter.add(length);
-                System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length);
-            }
-        }
+        unrolledArraycopyWork(nonNullSrc, srcPos, nonNullDest, destPos, unrolledLength, elementKind);
     }
 
-    /**
-     * This is the basic template for the full arraycopy checks, including a check that the
-     * underlying type is really an array type.
-     */
     @Snippet
-    public static void arraycopySlowPathIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, KlassPointer predictedKlass, @ConstantParameter JavaKind elementKind,
-                    @ConstantParameter SnippetInfo slowPath, @ConstantParameter Object slowPathArgument, @ConstantParameter Counters counters) {
+    public static void arraycopyCheckcastSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
+                    @ConstantParameter Counters counters, @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind) {
         Object nonNullSrc = GraalDirectives.guardingNonNull(src);
         Object nonNullDest = GraalDirectives.guardingNonNull(dest);
-        KlassPointer srcHub = loadHub(nonNullSrc);
-        KlassPointer destHub = loadHub(nonNullDest);
-        checkArrayType(srcHub);
-        checkArrayType(destHub);
+        checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
         checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
-        if (length == 0) {
-            counters.zeroLengthDynamicCounter.inc();
-        } else {
-            counters.nonZeroLengthDynamicCounter.inc();
-            counters.nonZeroLengthDynamicCopiedCounter.add(length);
-        }
-        ArrayCopySlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, predictedKlass, elementKind, slowPath, slowPathArgument);
+        incrementLengthCounter(length, counters);
+
+        ArrayCopyWithSlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind);
     }
 
-    /**
-     * Snippet for unrolled arraycopy.
-     */
     @Snippet
-    public static void arraycopyUnrolledIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter int unrolledLength, @ConstantParameter JavaKind elementKind,
-                    @ConstantParameter Counters counters) {
+    public static void arraycopyGenericSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, @ConstantParameter Counters counters,
+                    @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind) {
         Object nonNullSrc = GraalDirectives.guardingNonNull(src);
         Object nonNullDest = GraalDirectives.guardingNonNull(dest);
+        checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
         checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
-        if (length == 0) {
-            counters.zeroLengthDynamicCounter.inc();
-        } else {
-            counters.nonZeroLengthDynamicCounter.inc();
-            counters.nonZeroLengthDynamicCopiedCounter.add(length);
-        }
-        ArrayCopyUnrollNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, unrolledLength, elementKind);
+        incrementLengthCounter(length, counters);
+
+        ArrayCopyWithSlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind);
     }
 
     @Snippet
-    public static void checkcastArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, @ConstantParameter Counters counters) {
-        if (length > 0) {
-            KlassPointer destKlass = loadHub(nonNullDest);
+    public static void arraycopyNativeSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
+        // all checks are done in the native method, so no need to emit additional checks here
+        incrementLengthCounter(length, counters);
+        counters.systemArraycopyCounter.inc();
+        counters.systemArraycopyCopiedCounter.add(length);
+
+        System.arraycopy(src, srcPos, dest, destPos, length);
+    }
+
+    @Fold
+    static LocationIdentity getArrayLocation(JavaKind kind) {
+        return NamedLocationIdentity.getArrayLocation(kind);
+    }
+
+    private static void unrolledArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, JavaKind elementKind) {
+        int scale = arrayIndexScale(elementKind);
+        int arrayBaseOffset = arrayBaseOffset(elementKind);
+        LocationIdentity arrayLocation = getArrayLocation(elementKind);
+
+        long sourceOffset = arrayBaseOffset + (long) srcPos * scale;
+        long destOffset = arrayBaseOffset + (long) destPos * scale;
+        long position = 0;
+        long delta = scale;
+        if (probability(NOT_FREQUENT_PROBABILITY, nonNullSrc == nonNullDest && srcPos < destPos)) {
+            // bad aliased case so we need to copy the array from back to front
+            position = (long) (length - 1) * scale;
+            delta = -delta;
+        }
+
+        // the length was already checked before - we can emit unconditional instructions
+        ExplodeLoopNode.explodeLoop();
+        for (int iteration = 0; iteration < length; iteration++) {
+            Object value = RawLoadNode.load(nonNullSrc, sourceOffset + position, elementKind, arrayLocation);
+            RawStoreNode.storeObject(nonNullDest, destOffset + position, value, elementKind, arrayLocation, false);
+            position += delta;
+        }
+    }
+
+    @Snippet(allowPartialIntrinsicArgumentMismatch = true)
+    public static void checkcastArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
+        if (probability(FREQUENT_PROBABILITY, length > 0)) {
+            Object nonNullSrc = PiNode.asNonNullObject(src);
+            Object nonNullDest = PiNode.asNonNullObject(dest);
             KlassPointer srcKlass = loadHub(nonNullSrc);
-            if (probability(SLOW_PATH_PROBABILITY, srcKlass == destKlass)) {
+            KlassPointer destKlass = loadHub(nonNullDest);
+            if (probability(LIKELY_PROBABILITY, srcKlass == destKlass)) {
                 // no storecheck required.
                 counters.objectCheckcastSameTypeCounter.inc();
                 counters.objectCheckcastSameTypeCopiedCounter.add(length);
                 ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length);
             } else {
                 KlassPointer destElemKlass = destKlass.readKlassPointer(arrayClassElementOffset(INJECTED_VMCONFIG), OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION);
                 Word superCheckOffset = WordFactory.signed(destElemKlass.readInt(superCheckOffsetOffset(INJECTED_VMCONFIG), KLASS_SUPER_CHECK_OFFSET_LOCATION));
-                counters.objectCheckcastCounter.inc();
-                counters.objectCheckcastCopiedCounter.add(length);
+
+                counters.objectCheckcastDifferentTypeCounter.inc();
+                counters.objectCheckcastDifferentTypeCopiedCounter.add(length);
+
                 int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false);
-                if (copiedElements != 0) {
+                if (probability(SLOW_PATH_PROBABILITY, copiedElements != 0)) {
                     /*
-                     * the checkcast stub doesn't throw the ArrayStoreException, but returns the
-                     * number of copied elements (xor'd with -1).
+                     * the stub doesn't throw the ArrayStoreException, but returns the number of
+                     * copied elements (xor'd with -1).
                      */
                     copiedElements ^= -1;
                     System.arraycopy(nonNullSrc, srcPos + copiedElements, nonNullDest, destPos + copiedElements, length - copiedElements);
                 }
             }
         }
     }
 
-    @Snippet
-    public static void arraycopyGeneric(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
-        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
-        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
-        KlassPointer srcHub = loadHub(nonNullSrc);
-        KlassPointer destHub = loadHub(nonNullDest);
-        if (probability(FAST_PATH_PROBABILITY, srcHub.equal(destHub)) && probability(FAST_PATH_PROBABILITY, nonNullSrc != nonNullDest)) {
-            int layoutHelper = checkArrayType(srcHub);
-            final boolean isObjectArray = ((layoutHelper & layoutHelperElementTypePrimitiveInPlace(INJECTED_VMCONFIG)) == 0);
-            checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
-            if (probability(FAST_PATH_PROBABILITY, isObjectArray)) {
-                counters.genericObjectExactCallCounter.inc();
-                counters.genericObjectExactCallCopiedCounter.add(length);
-                ArrayCopyCallNode.disjointArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, JavaKind.Object);
-            } else {
-                counters.genericPrimitiveCallCounter.inc();
-                counters.genericPrimitiveCallCopiedCounter.add(length);
-                UnsafeArrayCopyNode.arraycopyPrimitive(nonNullSrc, srcPos, nonNullDest, destPos, length, layoutHelper);
+    @Snippet(allowPartialIntrinsicArgumentMismatch = true)
+    public static void genericArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
+        if (probability(FREQUENT_PROBABILITY, length > 0)) {
+            counters.genericArraycopyDifferentTypeCounter.inc();
+            counters.genericArraycopyDifferentTypeCopiedCounter.add(length);
+            int copiedElements = GenericArrayCopyCallNode.genericArraycopy(src, srcPos, dest, destPos, length);
+            if (probability(SLOW_PATH_PROBABILITY, copiedElements != 0)) {
+                /*
+                 * the stub doesn't throw the ArrayStoreException, but returns the number of copied
+                 * elements (xor'd with -1).
+                 */
+                copiedElements ^= -1;
+                System.arraycopy(src, srcPos + copiedElements, dest, destPos + copiedElements, length - copiedElements);
             }
-        } else {
-            counters.systemArraycopyCounter.inc();
-            counters.systemArraycopyCopiedCounter.add(length);
-            System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length);
         }
     }
 
-    @Fold
-    static LocationIdentity getArrayLocation(JavaKind kind) {
-        return NamedLocationIdentity.getArrayLocation(kind);
+    private static void incrementLengthCounter(int length, Counters counters) {
+        counters.lengthHistogram.inc(length);
     }
 
-    @Snippet
-    public static void arraycopyUnrolledWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, @ConstantParameter int length, @ConstantParameter JavaKind elementKind) {
-        final int scale = arrayIndexScale(elementKind);
-        int arrayBaseOffset = arrayBaseOffset(elementKind);
-        LocationIdentity arrayLocation = getArrayLocation(elementKind);
-        if (nonNullSrc == nonNullDest && srcPos < destPos) { // bad aliased case
-            long start = (long) (length - 1) * scale;
-            long i = start;
-            ExplodeLoopNode.explodeLoop();
-            for (int iteration = 0; iteration < length; iteration++) {
-                if (i >= 0) {
-                    Object a = RawLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation);
-                    RawStoreNode.storeObject(nonNullDest, arrayBaseOffset + i + (long) destPos * scale, a, elementKind, arrayLocation, false);
-                    i -= scale;
+    private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length, Counters counters) {
+        if (probability(SLOW_PATH_PROBABILITY, srcPos < 0) ||
+                        probability(SLOW_PATH_PROBABILITY, destPos < 0) ||
+                        probability(SLOW_PATH_PROBABILITY, length < 0) ||
+                        probability(SLOW_PATH_PROBABILITY, srcPos > ArrayLengthNode.arrayLength(src) - length) ||
+                        probability(SLOW_PATH_PROBABILITY, destPos > ArrayLengthNode.arrayLength(dest) - length)) {
+            counters.checkAIOOBECounter.inc();
+            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
                 }
+        counters.checkSuccessCounter.inc();
             }
+
+    private static void checkArrayTypes(Object nonNullSrc, Object nonNullDest, ArrayCopyTypeCheck arrayTypeCheck) {
+        if (arrayTypeCheck == ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK) {
+            // nothing to do
+        } else if (arrayTypeCheck == ArrayCopyTypeCheck.HUB_BASED_ARRAY_TYPE_CHECK) {
+            KlassPointer srcHub = loadHub(nonNullSrc);
+            KlassPointer destHub = loadHub(nonNullDest);
+            if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) {
+                DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
+            }
+        } else if (arrayTypeCheck == ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK) {
+            KlassPointer srcHub = loadHub(nonNullSrc);
+            KlassPointer destHub = loadHub(nonNullDest);
+            checkArrayType(srcHub);
+            checkArrayType(destHub);
         } else {
-            long end = (long) length * scale;
-            long i = 0;
-            ExplodeLoopNode.explodeLoop();
-            for (int iteration = 0; iteration < length; iteration++) {
-                if (i < end) {
-                    Object a = RawLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation);
-                    RawStoreNode.storeObject(nonNullDest, arrayBaseOffset + i + (long) destPos * scale, a, elementKind, arrayLocation, false);
-                    i += scale;
+            ReplacementsUtil.staticAssert(false, "unknown array type check");
                 }
             }
+
+    private static int checkArrayType(KlassPointer nonNullHub) {
+        int layoutHelper = readLayoutHelper(nonNullHub);
+        if (probability(SLOW_PATH_PROBABILITY, layoutHelper >= 0)) {
+            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
         }
+        return layoutHelper;
     }
 
     static class Counters {
         final SnippetCounter checkSuccessCounter;
         final SnippetCounter checkAIOOBECounter;
 
-        final SnippetCounter objectCheckcastCounter;
-        final SnippetCounter objectCheckcastSameTypeCounter;
-        final SnippetCounter predictedObjectArrayCopySlowPathCounter;
-        final SnippetCounter predictedObjectArrayCopyFastPathCounter;
-
-        final SnippetCounter genericPrimitiveCallCounter;
-        final SnippetCounter genericObjectExactCallCounter;
-        final SnippetCounter systemArraycopyCounter;
-
         final SnippetCounter zeroLengthStaticCounter;
-        final SnippetCounter zeroLengthDynamicCounter;
-        final SnippetCounter nonZeroLengthDynamicCounter;
+        final SnippetIntegerHistogram lengthHistogram;
 
-        final SnippetCounter nonZeroLengthDynamicCopiedCounter;
-        final SnippetCounter genericPrimitiveCallCopiedCounter;
-        final SnippetCounter genericObjectExactCallCopiedCounter;
+        final SnippetCounter systemArraycopyCounter;
         final SnippetCounter systemArraycopyCopiedCounter;
 
-        final SnippetCounter objectCheckcastCopiedCounter;
+        final SnippetCounter genericArraycopyDifferentTypeCopiedCounter;
+        final SnippetCounter genericArraycopyDifferentTypeCounter;
+
         final SnippetCounter objectCheckcastSameTypeCopiedCounter;
-        final SnippetCounter predictedObjectArrayCopySlowPathCopiedCounter;
-        final SnippetCounter predictedObjectArrayCopyFastPathCopiedCounter;
+        final SnippetCounter objectCheckcastSameTypeCounter;
+        final SnippetCounter objectCheckcastDifferentTypeCopiedCounter;
+        final SnippetCounter objectCheckcastDifferentTypeCounter;
 
         final EnumMap<JavaKind, SnippetCounter> arraycopyCallCounters = new EnumMap<>(JavaKind.class);
-        final EnumMap<JavaKind, SnippetCounter> arraycopyCounters = new EnumMap<>(JavaKind.class);
-
         final EnumMap<JavaKind, SnippetCounter> arraycopyCallCopiedCounters = new EnumMap<>(JavaKind.class);
-        final EnumMap<JavaKind, SnippetCounter> arraycopyCopiedCounters = new EnumMap<>(JavaKind.class);
 
         Counters(SnippetCounter.Group.Factory factory) {
             final Group checkCounters = factory.createSnippetCounterGroup("System.arraycopy checkInputs");
-            final Group counters = factory.createSnippetCounterGroup("System.arraycopy");
-            final Group copiedCounters = factory.createSnippetCounterGroup("System.arraycopy copied elements");
-            final Group lengthCounters = factory.createSnippetCounterGroup("System.arraycopy 0-length checks");
+            final Group callCounters = factory.createSnippetCounterGroup("System.arraycopy calls");
+            final Group copiedElementsCounters = factory.createSnippetCounterGroup("System.arraycopy copied elements");
+            final Group lengthCounters = factory.createSnippetCounterGroup("System.arraycopy with 0-length");
 
             checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess");
             checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE");
 
-            objectCheckcastCounter = new SnippetCounter(counters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays");
-            objectCheckcastSameTypeCounter = new SnippetCounter(counters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays");
-            predictedObjectArrayCopySlowPathCounter = new SnippetCounter(counters, "Object[]{slow-path}", "used System.arraycopy slow path for predicted Object[] arrays");
-            predictedObjectArrayCopyFastPathCounter = new SnippetCounter(counters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays");
-            genericPrimitiveCallCounter = new SnippetCounter(counters, "genericPrimitive", "generic arraycopy snippet for primitive arrays");
-            genericObjectExactCallCounter = new SnippetCounter(counters, "genericObjectExact", "generic arraycopy snippet for special object arrays");
-            systemArraycopyCounter = new SnippetCounter(counters, "genericObject", "call to System.arraycopy");
-
-            zeroLengthStaticCounter = new SnippetCounter(lengthCounters, "0-lengthcopy static", "arraycopy where the length is statically 0");
-            zeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "0-lengthcopy dynamically", "arraycopy where the length is dynamically 0");
-            nonZeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero");
-
-            nonZeroLengthDynamicCopiedCounter = new SnippetCounter(copiedCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero");
-            genericPrimitiveCallCopiedCounter = new SnippetCounter(copiedCounters, "genericPrimitive", "generic arraycopy snippet for primitive arrays");
-            genericObjectExactCallCopiedCounter = new SnippetCounter(copiedCounters, "genericObjectExact", "generic arraycopy snippet for special object arrays");
-            systemArraycopyCopiedCounter = new SnippetCounter(copiedCounters, "genericObject", "call to System.arraycopy");
-
-            objectCheckcastCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays");
-            objectCheckcastSameTypeCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays");
-            predictedObjectArrayCopySlowPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{slow-path}",
-                            "used System.arraycopy slow path for predicted Object[] arrays");
-            predictedObjectArrayCopyFastPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays");
-            createArraycopyCounter(JavaKind.Byte, counters, copiedCounters);
-            createArraycopyCounter(JavaKind.Boolean, counters, copiedCounters);
-            createArraycopyCounter(JavaKind.Char, counters, copiedCounters);
-            createArraycopyCounter(JavaKind.Short, counters, copiedCounters);
-            createArraycopyCounter(JavaKind.Int, counters, copiedCounters);
-            createArraycopyCounter(JavaKind.Long, counters, copiedCounters);
-            createArraycopyCounter(JavaKind.Float, counters, copiedCounters);
-            createArraycopyCounter(JavaKind.Double, counters, copiedCounters);
-            createArraycopyCounter(JavaKind.Object, counters, copiedCounters);
+            zeroLengthStaticCounter = new SnippetCounter(lengthCounters, "0-length copy static", "calls where the length is statically 0");
+            lengthHistogram = new SnippetIntegerHistogram(lengthCounters, 2, "length", "length");
+
+            systemArraycopyCounter = new SnippetCounter(callCounters, "native System.arraycopy", "JNI-based System.arraycopy call");
+            systemArraycopyCopiedCounter = new SnippetCounter(copiedElementsCounters, "native System.arraycopy", "JNI-based System.arraycopy call");
+
+            genericArraycopyDifferentTypeCounter = new SnippetCounter(callCounters, "generic[] stub", "generic arraycopy stub");
+            genericArraycopyDifferentTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "generic[] stub", "generic arraycopy stub");
+
+            objectCheckcastSameTypeCounter = new SnippetCounter(callCounters, "checkcast object[] (same-type)", "checkcast object[] stub but src.klass == dest.klass Object[] arrays");
+            objectCheckcastSameTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "checkcast object[] (same-type)", "checkcast object[] stub but src.klass == dest.klass Object[] arrays");
+            objectCheckcastDifferentTypeCounter = new SnippetCounter(callCounters, "checkcast object[] (store-check)", "checkcast object[] stub with store check");
+            objectCheckcastDifferentTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "checkcast object[] (store-check)", "checkcast object[] stub with store check");
+
+            createArraycopyCounter(JavaKind.Byte, callCounters, copiedElementsCounters);
+            createArraycopyCounter(JavaKind.Boolean, callCounters, copiedElementsCounters);
+            createArraycopyCounter(JavaKind.Char, callCounters, copiedElementsCounters);
+            createArraycopyCounter(JavaKind.Short, callCounters, copiedElementsCounters);
+            createArraycopyCounter(JavaKind.Int, callCounters, copiedElementsCounters);
+            createArraycopyCounter(JavaKind.Long, callCounters, copiedElementsCounters);
+            createArraycopyCounter(JavaKind.Float, callCounters, copiedElementsCounters);
+            createArraycopyCounter(JavaKind.Double, callCounters, copiedElementsCounters);
+            createArraycopyCounter(JavaKind.Object, callCounters, copiedElementsCounters);
         }
 
         void createArraycopyCounter(JavaKind kind, Group counters, Group copiedCounters) {
-            arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays"));
-            arraycopyCounters.put(kind, new SnippetCounter(counters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays"));
-
-            arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays"));
-            arraycopyCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays"));
+            arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[] stub", "arraycopy call for " + kind + "[] arrays"));
+            arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[] stub", "arraycopy call for " + kind + "[] arrays"));
         }
     }
 
     public static class Templates extends SnippetTemplate.AbstractTemplates {
+        private final SnippetInfo arraycopyGenericSnippet = snippet("arraycopyGenericSnippet");
+        private final SnippetInfo arraycopyUnrolledSnippet = snippet("arraycopyUnrolledSnippet");
+        private final SnippetInfo arraycopyExactSnippet = snippet("arraycopyExactSnippet");
+        private final SnippetInfo arraycopyZeroLengthSnippet = snippet("arraycopyZeroLengthSnippet");
+        private final SnippetInfo arraycopyCheckcastSnippet = snippet("arraycopyCheckcastSnippet");
+        private final SnippetInfo arraycopyNativeSnippet = snippet("arraycopyNativeSnippet");
+
+        private final SnippetInfo checkcastArraycopyWithSlowPathWork = snippet("checkcastArraycopyWithSlowPathWork");
+        private final SnippetInfo genericArraycopyWithSlowPathWork = snippet("genericArraycopyWithSlowPathWork");
+
+        private ResolvedJavaMethod originalArraycopy;
+        private final Counters counters;
 
         public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target) {
             super(options, factories, providers, providers.getSnippetReflection(), target);
             this.counters = new Counters(factory);
         }
 
-        private ResolvedJavaMethod originalArraycopy() throws GraalError {
-            if (originalArraycopy == null) {
-                Method method;
-                try {
-                    method = System.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class);
-                } catch (NoSuchMethodException | SecurityException e) {
-                    throw new GraalError(e);
-                }
-                originalArraycopy = providers.getMetaAccess().lookupJavaMethod(method);
-            }
-            return originalArraycopy;
-        }
-
-        private ResolvedJavaMethod originalArraycopy;
-
-        private final SnippetInfo checkcastArraycopyWorkSnippet = snippet("checkcastArraycopyWork");
-        private final SnippetInfo arraycopyGenericSnippet = snippet("arraycopyGeneric");
-
-        private final SnippetInfo arraycopySlowPathIntrinsicSnippet = snippet("arraycopySlowPathIntrinsic");
-        private final SnippetInfo arraycopyUnrolledIntrinsicSnippet = snippet("arraycopyUnrolledIntrinsic");
-        private final SnippetInfo arraycopyExactIntrinsicSnippet = snippet("arraycopyExactIntrinsic");
-        private final SnippetInfo arraycopyZeroLengthIntrinsicSnippet = snippet("arraycopyZeroLengthIntrinsic");
-        private final SnippetInfo arraycopyPredictedExactIntrinsicSnippet = snippet("arraycopyPredictedExactIntrinsic");
-        private final SnippetInfo arraycopyPredictedObjectWorkSnippet = snippet("arraycopyPredictedObjectWork");
-
-        private final SnippetInfo arraycopyUnrolledWorkSnippet = snippet("arraycopyUnrolledWork");
-
-        private final Counters counters;
-
         protected SnippetInfo snippet(String methodName) {
             SnippetInfo info = snippet(ArrayCopySnippets.class, methodName, LocationIdentity.any());
             info.setOriginalMethod(originalArraycopy());
             return info;
         }
 
-        public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy) {
-            return selectComponentKind(arraycopy, true);
-        }
-
-        public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy, boolean exact) {
-            ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp());
-            ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp());
-
-            if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) {
-                if (!exact) {
-                    JavaKind component = getComponentKind(srcType);
-                    if (component != null) {
-                        return component;
-                    }
-                    return getComponentKind(destType);
-                }
-                return null;
-            }
-            if (exact) {
-                if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) {
-                    return null;
-                }
-                if (!arraycopy.isExact()) {
-                    return null;
-                }
-            }
-            return srcType.getComponentType().getJavaKind();
-        }
-
-        private static JavaKind getComponentKind(ResolvedJavaType type) {
-            if (type != null && type.isArray()) {
-                return type.getComponentType().getJavaKind();
-            }
-            return null;
-        }
-
-        private static boolean shouldUnroll(ValueNode length) {
-            return length.isConstant() && length.asJavaConstant().asInt() <= 8 && length.asJavaConstant().asInt() != 0;
-        }
-
         public void lower(ArrayCopyNode arraycopy, LoweringTool tool) {
-            JavaKind componentKind = selectComponentKind(arraycopy);
-            SnippetInfo snippetInfo = null;
-            SnippetInfo slowPathSnippetInfo = null;
-            Object slowPathArgument = null;
+            JavaKind elementKind = selectComponentKind(arraycopy);
+            SnippetInfo snippetInfo;
+            ArrayCopyTypeCheck arrayTypeCheck;
 
-            if (arraycopy.getLength().isConstant() && arraycopy.getLength().asJavaConstant().asLong() == 0) {
-                snippetInfo = arraycopyZeroLengthIntrinsicSnippet;
-            } else if (arraycopy.isExact()) {
-                snippetInfo = arraycopyExactIntrinsicSnippet;
-                if (shouldUnroll(arraycopy.getLength())) {
-                    snippetInfo = arraycopyUnrolledIntrinsicSnippet;
-                }
-            } else {
-                if (componentKind == JavaKind.Object) {
                     ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp());
                     ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp());
+            if (!canBeArray(srcType) || !canBeArray(destType)) {
+                // at least one of the objects is definitely not an array - use the native call
+                // right away as the copying will fail anyways
+                snippetInfo = arraycopyNativeSnippet;
+                arrayTypeCheck = ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK;
+            } else {
                     ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType();
                     ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType();
-                    if (srcComponentType != null && destComponentType != null && !srcComponentType.isPrimitive() && !destComponentType.isPrimitive()) {
-                        snippetInfo = arraycopySlowPathIntrinsicSnippet;
-                        slowPathSnippetInfo = checkcastArraycopyWorkSnippet;
-                        slowPathArgument = LocationIdentity.any();
-                        /*
-                         * Because this snippet has to use Sysytem.arraycopy as a slow path, we must
-                         * pretend to kill any() so clear the componentKind.
-                         */
-                        componentKind = null;
-                    }
-                }
-                if (componentKind == null && snippetInfo == null) {
-                    JavaKind predictedKind = selectComponentKind(arraycopy, false);
-                    if (predictedKind != null) {
-                        /*
-                         * At least one array is of a known type requiring no store checks, so
-                         * assume the other is of the same type. Generally this is working around
-                         * deficiencies in our propagation of type information.
-                         */
-                        componentKind = predictedKind;
-                        if (predictedKind == JavaKind.Object) {
-                            snippetInfo = arraycopySlowPathIntrinsicSnippet;
-                            slowPathSnippetInfo = arraycopyPredictedObjectWorkSnippet;
-                            slowPathArgument = predictedKind;
-                            componentKind = null;
-                        } else {
-                            snippetInfo = arraycopyPredictedExactIntrinsicSnippet;
-                        }
+
+                if (arraycopy.isExact()) {
+                    // there is a sufficient type match - we don't need any additional type checks
+                    snippetInfo = arraycopyExactSnippet;
+                    arrayTypeCheck = ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK;
+                } else if (srcComponentType == null && destComponentType == null) {
+                    // we don't know anything about the types - use the generic copying
+                    snippetInfo = arraycopyGenericSnippet;
+                    arrayTypeCheck = ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK;
+                } else if (srcComponentType != null && destComponentType != null) {
+                    if (!srcComponentType.isPrimitive() && !destComponentType.isPrimitive()) {
+                        // it depends on the array content if the copy succeeds - we need
+                        // a type check for every store
+                        snippetInfo = arraycopyCheckcastSnippet;
+                        arrayTypeCheck = ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK;
+                    } else {
+                        // one object is an object array, the other one is a primitive array.
+                        // this copy will always fail - use the native call right away
+                        assert !srcComponentType.equals(destComponentType) : "must be handled by arraycopy.isExact()";
+                        snippetInfo = arraycopyNativeSnippet;
+                        arrayTypeCheck = ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK;
+                    }
+                } else {
+                    ResolvedJavaType nonNullComponentType = srcComponentType != null ? srcComponentType : destComponentType;
+                    if (nonNullComponentType.isPrimitive()) {
+                        // one involved object is a primitive array - we can safely assume that we
+                        // are copying primitive arrays
+                        snippetInfo = arraycopyExactSnippet;
+                        arrayTypeCheck = ArrayCopyTypeCheck.HUB_BASED_ARRAY_TYPE_CHECK;
+                        elementKind = nonNullComponentType.getJavaKind();
+                    } else {
+                        // one involved object is an object array - we can safely assume that we are
+                        // copying object arrays that might require a store check
+                        snippetInfo = arraycopyCheckcastSnippet;
+                        arrayTypeCheck = ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK;
                     }
                 }
-                if (snippetInfo == null) {
-                    snippetInfo = arraycopyGenericSnippet;
                 }
+
+            // a few special cases that are easier to handle when all other variables already have a
+            // value
+            if (arraycopy.getLength().isConstant() && arraycopy.getLength().asJavaConstant().asLong() == 0) {
+                snippetInfo = arraycopyZeroLengthSnippet;
+            } else if (snippetInfo == arraycopyExactSnippet && shouldUnroll(arraycopy.getLength())) {
+                snippetInfo = arraycopyUnrolledSnippet;
             }
+
+            // create the snippet
             Arguments args = new Arguments(snippetInfo, arraycopy.graph().getGuardsStage(), tool.getLoweringStage());
             args.add("src", arraycopy.getSource());
             args.add("srcPos", arraycopy.getSourcePosition());
             args.add("dest", arraycopy.getDestination());
             args.add("destPos", arraycopy.getDestinationPosition());
             args.add("length", arraycopy.getLength());
-            if (snippetInfo == arraycopyUnrolledIntrinsicSnippet) {
+            if (snippetInfo != arraycopyNativeSnippet) {
+                assert arrayTypeCheck != ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK;
+                args.addConst("arrayTypeCheck", arrayTypeCheck);
+            }
+            if (snippetInfo == arraycopyUnrolledSnippet) {
+                args.addConst("elementKind", elementKind != null ? elementKind : JavaKind.Illegal);
                 args.addConst("unrolledLength", arraycopy.getLength().asJavaConstant().asInt());
-                args.addConst("elementKind", componentKind != null ? componentKind : JavaKind.Illegal);
-            } else if (snippetInfo == arraycopySlowPathIntrinsicSnippet) {
-                ValueNode predictedKlass = null;
-                if (slowPathArgument == arraycopyPredictedObjectWorkSnippet) {
-                    HotSpotResolvedObjectType arrayClass = (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(Object[].class);
-                    predictedKlass = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayClass.klass(), tool.getMetaAccess(), arraycopy.graph());
-                } else {
-                    predictedKlass = ConstantNode.forConstant(KlassPointerStamp.klassAlwaysNull(), JavaConstant.NULL_POINTER, tool.getMetaAccess(), arraycopy.graph());
                 }
-                args.add("predictedKlass", predictedKlass);
-                args.addConst("elementKind", componentKind != null ? componentKind : JavaKind.Illegal);
-                args.addConst("slowPath", slowPathSnippetInfo);
-                assert slowPathArgument != null;
-                args.addConst("slowPathArgument", slowPathArgument);
-            } else if (snippetInfo == arraycopyExactIntrinsicSnippet || snippetInfo == arraycopyPredictedExactIntrinsicSnippet) {
-                assert componentKind != null;
-                args.addConst("elementKind", componentKind);
-                args.addConst("counter", counters.arraycopyCallCounters.get(componentKind));
-                args.addConst("copiedCounter", counters.arraycopyCallCopiedCounters.get(componentKind));
+            if (snippetInfo == arraycopyExactSnippet) {
+                assert elementKind != null;
+                args.addConst("elementKind", elementKind);
+                args.addConst("elementKindCounter", counters.arraycopyCallCounters.get(elementKind));
+                args.addConst("elementKindCopiedCounter", counters.arraycopyCallCopiedCounters.get(elementKind));
             }
             args.addConst("counters", counters);
+            if (snippetInfo == arraycopyCheckcastSnippet) {
+                args.addConst("workSnippet", checkcastArraycopyWithSlowPathWork);
+                args.addConst("elementKind", JavaKind.Illegal);
+            }
+            if (snippetInfo == arraycopyGenericSnippet) {
+                args.addConst("workSnippet", genericArraycopyWithSlowPathWork);
+                args.addConst("elementKind", JavaKind.Illegal);
+            }
+
             instantiate(args, arraycopy);
         }
 
-        public void lower(ArrayCopySlowPathNode arraycopy, LoweringTool tool) {
+        public void lower(ArrayCopyWithSlowPathNode arraycopy, LoweringTool tool) {
             StructuredGraph graph = arraycopy.graph();
             if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
-                // Can't be lowered yet
+                // if an arraycopy contains a slow path, we can't lower it right away
                 return;
             }
+
             SnippetInfo snippetInfo = arraycopy.getSnippet();
             Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage());
-            args.add("nonNullSrc", arraycopy.getSource());
+            args.add("src", arraycopy.getSource());
             args.add("srcPos", arraycopy.getSourcePosition());
-            args.add("nonNullDest", arraycopy.getDestination());
+            args.add("dest", arraycopy.getDestination());
             args.add("destPos", arraycopy.getDestinationPosition());
-            if (snippetInfo == arraycopyUnrolledWorkSnippet) {
-                args.addConst("length", ((Integer) arraycopy.getArgument()).intValue());
-                args.addConst("elementKind", arraycopy.getElementKind());
-            } else {
                 args.add("length", arraycopy.getLength());
-            }
-            if (snippetInfo == arraycopyPredictedObjectWorkSnippet) {
-                args.add("objectArrayKlass", arraycopy.getPredictedKlass());
-                args.addConst("counter", counters.arraycopyCallCounters.get(JavaKind.Object));
-                args.addConst("copiedCounter", counters.arraycopyCallCopiedCounters.get(JavaKind.Object));
                 args.addConst("counters", counters);
-            }
             instantiate(args, arraycopy);
         }
 
-        public void lower(ArrayCopyUnrollNode arraycopy, LoweringTool tool) {
-            StructuredGraph graph = arraycopy.graph();
-            if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
-                // Can't be lowered yet
-                return;
+        private static boolean canBeArray(ResolvedJavaType type) {
+            return type == null || type.isJavaLangObject() || type.isArray();
             }
-            SnippetInfo snippetInfo = arraycopyUnrolledWorkSnippet;
-            Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage());
-            args.add("nonNullSrc", arraycopy.getSource());
-            args.add("srcPos", arraycopy.getSourcePosition());
-            args.add("nonNullDest", arraycopy.getDestination());
-            args.add("destPos", arraycopy.getDestinationPosition());
-            args.addConst("length", arraycopy.getUnrollLength());
-            args.addConst("elementKind", arraycopy.getElementKind());
-            template(graph.getDebug(), args).instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args);
+
+        public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy) {
+            ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp());
+            ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp());
+
+            if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) {
+                return null;
+            }
+            if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) {
+                return null;
+            }
+            if (!arraycopy.isExact()) {
+                return null;
+            }
+            return srcType.getComponentType().getJavaKind();
+        }
+
+        private static boolean shouldUnroll(ValueNode length) {
+            return length.isConstant() && length.asJavaConstant().asInt() <= 8 && length.asJavaConstant().asInt() != 0;
         }
 
         /**
          * Instantiate the snippet template and fix up the FrameState of any Invokes of
          * System.arraycopy and propagate the captured bci in the ArrayCopySlowPathNode.

@@ -648,16 +541,29 @@
                     } else {
                         assert arraycopy.stateAfter() != null : arraycopy;
                         newInvoke.setStateAfter(arraycopy.stateAfter());
                     }
                     graph.replaceFixedWithFixed((InvokeNode) invoke.asNode(), newInvoke);
-                } else if (originalNode instanceof ArrayCopySlowPathNode) {
-                    ArrayCopySlowPathNode slowPath = (ArrayCopySlowPathNode) replacements.get(originalNode);
+                } else if (originalNode instanceof ArrayCopyWithSlowPathNode) {
+                    ArrayCopyWithSlowPathNode slowPath = (ArrayCopyWithSlowPathNode) replacements.get(originalNode);
                     assert arraycopy.stateAfter() != null : arraycopy;
                     assert slowPath.stateAfter() == arraycopy.stateAfter();
                     slowPath.setBci(arraycopy.getBci());
                 }
             }
             GraphUtil.killCFG(arraycopy);
         }
+
+        private ResolvedJavaMethod originalArraycopy() throws GraalError {
+            if (originalArraycopy == null) {
+                Method method;
+                try {
+                    method = System.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class);
+                } catch (NoSuchMethodException | SecurityException e) {
+                    throw new GraalError(e);
+                }
+                originalArraycopy = providers.getMetaAccess().lookupJavaMethod(method);
+            }
+            return originalArraycopy;
+        }
     }
 }
< prev index next >