--- old/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/arraycopy/ArrayCopySnippets.java 2017-11-03 23:56:46.671457175 -0700 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/arraycopy/ArrayCopySnippets.java 2017-11-03 23:56:46.339442318 -0700 @@ -28,11 +28,12 @@ import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayClassElementOffset; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayIndexScale; -import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperElementTypePrimitiveInPlace; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadHub; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper; import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.superCheckOffsetOffset; -import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY; +import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY; +import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.LIKELY_PROBABILITY; +import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY; import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; @@ -47,10 +48,8 @@ import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.hotspot.meta.HotSpotProviders; -import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp; import org.graalvm.compiler.hotspot.word.KlassPointer; import org.graalvm.compiler.nodes.CallTargetNode; -import org.graalvm.compiler.nodes.ConstantNode; import org.graalvm.compiler.nodes.DeoptimizeNode; import org.graalvm.compiler.nodes.Invoke; import org.graalvm.compiler.nodes.InvokeNode; @@ -65,8 +64,10 @@ import org.graalvm.compiler.nodes.type.StampTool; import org.graalvm.compiler.nodes.util.GraphUtil; import org.graalvm.compiler.options.OptionValues; +import org.graalvm.compiler.replacements.ReplacementsUtil; import org.graalvm.compiler.replacements.SnippetCounter; import org.graalvm.compiler.replacements.SnippetCounter.Group; +import org.graalvm.compiler.replacements.SnippetIntegerHistogram; import org.graalvm.compiler.replacements.SnippetTemplate; import org.graalvm.compiler.replacements.SnippetTemplate.Arguments; import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo; @@ -79,170 +80,132 @@ import org.graalvm.word.WordFactory; import jdk.vm.ci.code.TargetDescription; -import jdk.vm.ci.hotspot.HotSpotResolvedObjectType; import jdk.vm.ci.meta.DeoptimizationAction; import jdk.vm.ci.meta.DeoptimizationReason; -import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.ResolvedJavaMethod; import jdk.vm.ci.meta.ResolvedJavaType; public class ArrayCopySnippets implements Snippets { - private static int checkArrayType(KlassPointer nonNullHub) { - int layoutHelper = readLayoutHelper(nonNullHub); - if (probability(SLOW_PATH_PROBABILITY, layoutHelper >= 0)) { - DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); - } - return layoutHelper; - } - - private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length, Counters counters) { - if (probability(SLOW_PATH_PROBABILITY, srcPos < 0)) { - counters.checkAIOOBECounter.inc(); - DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); - } - if (probability(SLOW_PATH_PROBABILITY, destPos < 0)) { - counters.checkAIOOBECounter.inc(); - DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); - } - if (probability(SLOW_PATH_PROBABILITY, length < 0)) { - counters.checkAIOOBECounter.inc(); - DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); - } - if (probability(SLOW_PATH_PROBABILITY, srcPos > ArrayLengthNode.arrayLength(src) - length)) { - counters.checkAIOOBECounter.inc(); - DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); - } - if (probability(SLOW_PATH_PROBABILITY, destPos > ArrayLengthNode.arrayLength(dest) - length)) { - counters.checkAIOOBECounter.inc(); - DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); - } - counters.checkSuccessCounter.inc(); + private enum ArrayCopyTypeCheck { + UNDEFINED_ARRAY_TYPE_CHECK, + // we know that both objects are arrays and have the same type + NO_ARRAY_TYPE_CHECK, + // can be used when we know that one of the objects is a primitive array + HUB_BASED_ARRAY_TYPE_CHECK, + // must be used when we don't have sufficient information to use one of the others + LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK } @Snippet - public static void arraycopyZeroLengthIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) { + public static void arraycopyZeroLengthSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, + @ConstantParameter Counters counters) { Object nonNullSrc = GraalDirectives.guardingNonNull(src); Object nonNullDest = GraalDirectives.guardingNonNull(dest); - KlassPointer srcHub = loadHub(nonNullSrc); - KlassPointer destHub = loadHub(nonNullDest); - checkArrayType(srcHub); - checkArrayType(destHub); + checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck); checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); counters.zeroLengthStaticCounter.inc(); } @Snippet - public static void arraycopyExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind, @ConstantParameter SnippetCounter counter, - @ConstantParameter SnippetCounter copiedCounter, @ConstantParameter Counters counters) { + public static void arraycopyExactSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, + @ConstantParameter JavaKind elementKind, @ConstantParameter SnippetCounter elementKindCounter, @ConstantParameter SnippetCounter elementKindCopiedCounter, + @ConstantParameter Counters counters) { Object nonNullSrc = GraalDirectives.guardingNonNull(src); Object nonNullDest = GraalDirectives.guardingNonNull(dest); + checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck); checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); - counter.inc(); - copiedCounter.add(length); + incrementLengthCounter(length, counters); + + elementKindCounter.inc(); + elementKindCopiedCounter.add(length); ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind); - if (length == 0) { - counters.zeroLengthDynamicCounter.inc(); - } else { - counters.nonZeroLengthDynamicCounter.inc(); - counters.nonZeroLengthDynamicCopiedCounter.add(length); - } } - /** - * This intrinsic is useful for the case where we know something statically about one of the - * inputs but not the other. - */ @Snippet - public static void arraycopyPredictedExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind, - @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter, @ConstantParameter Counters counters) { + public static void arraycopyUnrolledSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, + @ConstantParameter JavaKind elementKind, @ConstantParameter int unrolledLength, @ConstantParameter Counters counters) { Object nonNullSrc = GraalDirectives.guardingNonNull(src); Object nonNullDest = GraalDirectives.guardingNonNull(dest); - KlassPointer srcHub = loadHub(nonNullSrc); - KlassPointer destHub = loadHub(nonNullDest); - if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) { - DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); - } + checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck); checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); - counter.inc(); - copiedCounter.add(length); - ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind); - if (length == 0) { - counters.zeroLengthDynamicCounter.inc(); - } else { - counters.nonZeroLengthDynamicCounter.inc(); - counters.nonZeroLengthDynamicCopiedCounter.add(length); - } - } + incrementLengthCounter(length, counters); - @Snippet - public static void arraycopyPredictedObjectWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, KlassPointer objectArrayKlass, - @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter, @ConstantParameter Counters counters) { - if (length > 0) { - KlassPointer srcHub = loadHub(PiNode.asNonNullObject(nonNullSrc)); - KlassPointer destHub = loadHub(PiNode.asNonNullObject(nonNullDest)); - if (probability(FAST_PATH_PROBABILITY, srcHub == destHub || destHub == objectArrayKlass)) { - counter.inc(); - copiedCounter.add(length); - counters.predictedObjectArrayCopyFastPathCounter.inc(); - counters.predictedObjectArrayCopyFastPathCopiedCounter.add(length); - ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length); - } else { - counters.predictedObjectArrayCopySlowPathCounter.inc(); - counters.predictedObjectArrayCopySlowPathCopiedCounter.add(length); - System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length); - } - } + unrolledArraycopyWork(nonNullSrc, srcPos, nonNullDest, destPos, unrolledLength, elementKind); } - /** - * This is the basic template for the full arraycopy checks, including a check that the - * underlying type is really an array type. - */ @Snippet - public static void arraycopySlowPathIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, KlassPointer predictedKlass, @ConstantParameter JavaKind elementKind, - @ConstantParameter SnippetInfo slowPath, @ConstantParameter Object slowPathArgument, @ConstantParameter Counters counters) { + public static void arraycopyCheckcastSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, + @ConstantParameter Counters counters, @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind) { Object nonNullSrc = GraalDirectives.guardingNonNull(src); Object nonNullDest = GraalDirectives.guardingNonNull(dest); - KlassPointer srcHub = loadHub(nonNullSrc); - KlassPointer destHub = loadHub(nonNullDest); - checkArrayType(srcHub); - checkArrayType(destHub); + checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck); checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); - if (length == 0) { - counters.zeroLengthDynamicCounter.inc(); - } else { - counters.nonZeroLengthDynamicCounter.inc(); - counters.nonZeroLengthDynamicCopiedCounter.add(length); - } - ArrayCopySlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, predictedKlass, elementKind, slowPath, slowPathArgument); + incrementLengthCounter(length, counters); + + ArrayCopyWithSlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind); } - /** - * Snippet for unrolled arraycopy. - */ @Snippet - public static void arraycopyUnrolledIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter int unrolledLength, @ConstantParameter JavaKind elementKind, - @ConstantParameter Counters counters) { + public static void arraycopyGenericSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, @ConstantParameter Counters counters, + @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind) { Object nonNullSrc = GraalDirectives.guardingNonNull(src); Object nonNullDest = GraalDirectives.guardingNonNull(dest); + checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck); checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); - if (length == 0) { - counters.zeroLengthDynamicCounter.inc(); - } else { - counters.nonZeroLengthDynamicCounter.inc(); - counters.nonZeroLengthDynamicCopiedCounter.add(length); - } - ArrayCopyUnrollNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, unrolledLength, elementKind); + incrementLengthCounter(length, counters); + + ArrayCopyWithSlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind); } @Snippet - public static void checkcastArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, @ConstantParameter Counters counters) { - if (length > 0) { - KlassPointer destKlass = loadHub(nonNullDest); + public static void arraycopyNativeSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) { + // all checks are done in the native method, so no need to emit additional checks here + incrementLengthCounter(length, counters); + counters.systemArraycopyCounter.inc(); + counters.systemArraycopyCopiedCounter.add(length); + + System.arraycopy(src, srcPos, dest, destPos, length); + } + + @Fold + static LocationIdentity getArrayLocation(JavaKind kind) { + return NamedLocationIdentity.getArrayLocation(kind); + } + + private static void unrolledArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, JavaKind elementKind) { + int scale = arrayIndexScale(elementKind); + int arrayBaseOffset = arrayBaseOffset(elementKind); + LocationIdentity arrayLocation = getArrayLocation(elementKind); + + long sourceOffset = arrayBaseOffset + (long) srcPos * scale; + long destOffset = arrayBaseOffset + (long) destPos * scale; + long position = 0; + long delta = scale; + if (probability(NOT_FREQUENT_PROBABILITY, nonNullSrc == nonNullDest && srcPos < destPos)) { + // bad aliased case so we need to copy the array from back to front + position = (long) (length - 1) * scale; + delta = -delta; + } + + // the length was already checked before - we can emit unconditional instructions + ExplodeLoopNode.explodeLoop(); + for (int iteration = 0; iteration < length; iteration++) { + Object value = RawLoadNode.load(nonNullSrc, sourceOffset + position, elementKind, arrayLocation); + RawStoreNode.storeObject(nonNullDest, destOffset + position, value, elementKind, arrayLocation, false); + position += delta; + } + } + + @Snippet(allowPartialIntrinsicArgumentMismatch = true) + public static void checkcastArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) { + if (probability(FREQUENT_PROBABILITY, length > 0)) { + Object nonNullSrc = PiNode.asNonNullObject(src); + Object nonNullDest = PiNode.asNonNullObject(dest); KlassPointer srcKlass = loadHub(nonNullSrc); - if (probability(SLOW_PATH_PROBABILITY, srcKlass == destKlass)) { + KlassPointer destKlass = loadHub(nonNullDest); + if (probability(LIKELY_PROBABILITY, srcKlass == destKlass)) { // no storecheck required. counters.objectCheckcastSameTypeCounter.inc(); counters.objectCheckcastSameTypeCopiedCounter.add(length); @@ -250,13 +213,15 @@ } else { KlassPointer destElemKlass = destKlass.readKlassPointer(arrayClassElementOffset(INJECTED_VMCONFIG), OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION); Word superCheckOffset = WordFactory.signed(destElemKlass.readInt(superCheckOffsetOffset(INJECTED_VMCONFIG), KLASS_SUPER_CHECK_OFFSET_LOCATION)); - counters.objectCheckcastCounter.inc(); - counters.objectCheckcastCopiedCounter.add(length); + + counters.objectCheckcastDifferentTypeCounter.inc(); + counters.objectCheckcastDifferentTypeCopiedCounter.add(length); + int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false); - if (copiedElements != 0) { + if (probability(SLOW_PATH_PROBABILITY, copiedElements != 0)) { /* - * the checkcast stub doesn't throw the ArrayStoreException, but returns the - * number of copied elements (xor'd with -1). + * the stub doesn't throw the ArrayStoreException, but returns the number of + * copied elements (xor'd with -1). */ copiedElements ^= -1; System.arraycopy(nonNullSrc, srcPos + copiedElements, nonNullDest, destPos + copiedElements, length - copiedElements); @@ -265,360 +230,288 @@ } } - @Snippet - public static void arraycopyGeneric(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) { - Object nonNullSrc = GraalDirectives.guardingNonNull(src); - Object nonNullDest = GraalDirectives.guardingNonNull(dest); - KlassPointer srcHub = loadHub(nonNullSrc); - KlassPointer destHub = loadHub(nonNullDest); - if (probability(FAST_PATH_PROBABILITY, srcHub.equal(destHub)) && probability(FAST_PATH_PROBABILITY, nonNullSrc != nonNullDest)) { - int layoutHelper = checkArrayType(srcHub); - final boolean isObjectArray = ((layoutHelper & layoutHelperElementTypePrimitiveInPlace(INJECTED_VMCONFIG)) == 0); - checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters); - if (probability(FAST_PATH_PROBABILITY, isObjectArray)) { - counters.genericObjectExactCallCounter.inc(); - counters.genericObjectExactCallCopiedCounter.add(length); - ArrayCopyCallNode.disjointArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, JavaKind.Object); - } else { - counters.genericPrimitiveCallCounter.inc(); - counters.genericPrimitiveCallCopiedCounter.add(length); - UnsafeArrayCopyNode.arraycopyPrimitive(nonNullSrc, srcPos, nonNullDest, destPos, length, layoutHelper); + @Snippet(allowPartialIntrinsicArgumentMismatch = true) + public static void genericArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) { + if (probability(FREQUENT_PROBABILITY, length > 0)) { + counters.genericArraycopyDifferentTypeCounter.inc(); + counters.genericArraycopyDifferentTypeCopiedCounter.add(length); + int copiedElements = GenericArrayCopyCallNode.genericArraycopy(src, srcPos, dest, destPos, length); + if (probability(SLOW_PATH_PROBABILITY, copiedElements != 0)) { + /* + * the stub doesn't throw the ArrayStoreException, but returns the number of copied + * elements (xor'd with -1). + */ + copiedElements ^= -1; + System.arraycopy(src, srcPos + copiedElements, dest, destPos + copiedElements, length - copiedElements); } - } else { - counters.systemArraycopyCounter.inc(); - counters.systemArraycopyCopiedCounter.add(length); - System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length); } } - @Fold - static LocationIdentity getArrayLocation(JavaKind kind) { - return NamedLocationIdentity.getArrayLocation(kind); + private static void incrementLengthCounter(int length, Counters counters) { + counters.lengthHistogram.inc(length); } - @Snippet - public static void arraycopyUnrolledWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, @ConstantParameter int length, @ConstantParameter JavaKind elementKind) { - final int scale = arrayIndexScale(elementKind); - int arrayBaseOffset = arrayBaseOffset(elementKind); - LocationIdentity arrayLocation = getArrayLocation(elementKind); - if (nonNullSrc == nonNullDest && srcPos < destPos) { // bad aliased case - long start = (long) (length - 1) * scale; - long i = start; - ExplodeLoopNode.explodeLoop(); - for (int iteration = 0; iteration < length; iteration++) { - if (i >= 0) { - Object a = RawLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation); - RawStoreNode.storeObject(nonNullDest, arrayBaseOffset + i + (long) destPos * scale, a, elementKind, arrayLocation, false); - i -= scale; - } - } + private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length, Counters counters) { + if (probability(SLOW_PATH_PROBABILITY, srcPos < 0) || + probability(SLOW_PATH_PROBABILITY, destPos < 0) || + probability(SLOW_PATH_PROBABILITY, length < 0) || + probability(SLOW_PATH_PROBABILITY, srcPos > ArrayLengthNode.arrayLength(src) - length) || + probability(SLOW_PATH_PROBABILITY, destPos > ArrayLengthNode.arrayLength(dest) - length)) { + counters.checkAIOOBECounter.inc(); + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); + } + counters.checkSuccessCounter.inc(); + } + + private static void checkArrayTypes(Object nonNullSrc, Object nonNullDest, ArrayCopyTypeCheck arrayTypeCheck) { + if (arrayTypeCheck == ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK) { + // nothing to do + } else if (arrayTypeCheck == ArrayCopyTypeCheck.HUB_BASED_ARRAY_TYPE_CHECK) { + KlassPointer srcHub = loadHub(nonNullSrc); + KlassPointer destHub = loadHub(nonNullDest); + if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) { + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); + } + } else if (arrayTypeCheck == ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK) { + KlassPointer srcHub = loadHub(nonNullSrc); + KlassPointer destHub = loadHub(nonNullDest); + checkArrayType(srcHub); + checkArrayType(destHub); } else { - long end = (long) length * scale; - long i = 0; - ExplodeLoopNode.explodeLoop(); - for (int iteration = 0; iteration < length; iteration++) { - if (i < end) { - Object a = RawLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation); - RawStoreNode.storeObject(nonNullDest, arrayBaseOffset + i + (long) destPos * scale, a, elementKind, arrayLocation, false); - i += scale; - } - } + ReplacementsUtil.staticAssert(false, "unknown array type check"); } } + private static int checkArrayType(KlassPointer nonNullHub) { + int layoutHelper = readLayoutHelper(nonNullHub); + if (probability(SLOW_PATH_PROBABILITY, layoutHelper >= 0)) { + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); + } + return layoutHelper; + } + static class Counters { final SnippetCounter checkSuccessCounter; final SnippetCounter checkAIOOBECounter; - final SnippetCounter objectCheckcastCounter; - final SnippetCounter objectCheckcastSameTypeCounter; - final SnippetCounter predictedObjectArrayCopySlowPathCounter; - final SnippetCounter predictedObjectArrayCopyFastPathCounter; - - final SnippetCounter genericPrimitiveCallCounter; - final SnippetCounter genericObjectExactCallCounter; - final SnippetCounter systemArraycopyCounter; - final SnippetCounter zeroLengthStaticCounter; - final SnippetCounter zeroLengthDynamicCounter; - final SnippetCounter nonZeroLengthDynamicCounter; + final SnippetIntegerHistogram lengthHistogram; - final SnippetCounter nonZeroLengthDynamicCopiedCounter; - final SnippetCounter genericPrimitiveCallCopiedCounter; - final SnippetCounter genericObjectExactCallCopiedCounter; + final SnippetCounter systemArraycopyCounter; final SnippetCounter systemArraycopyCopiedCounter; - final SnippetCounter objectCheckcastCopiedCounter; + final SnippetCounter genericArraycopyDifferentTypeCopiedCounter; + final SnippetCounter genericArraycopyDifferentTypeCounter; + final SnippetCounter objectCheckcastSameTypeCopiedCounter; - final SnippetCounter predictedObjectArrayCopySlowPathCopiedCounter; - final SnippetCounter predictedObjectArrayCopyFastPathCopiedCounter; + final SnippetCounter objectCheckcastSameTypeCounter; + final SnippetCounter objectCheckcastDifferentTypeCopiedCounter; + final SnippetCounter objectCheckcastDifferentTypeCounter; final EnumMap arraycopyCallCounters = new EnumMap<>(JavaKind.class); - final EnumMap arraycopyCounters = new EnumMap<>(JavaKind.class); - final EnumMap arraycopyCallCopiedCounters = new EnumMap<>(JavaKind.class); - final EnumMap arraycopyCopiedCounters = new EnumMap<>(JavaKind.class); Counters(SnippetCounter.Group.Factory factory) { final Group checkCounters = factory.createSnippetCounterGroup("System.arraycopy checkInputs"); - final Group counters = factory.createSnippetCounterGroup("System.arraycopy"); - final Group copiedCounters = factory.createSnippetCounterGroup("System.arraycopy copied elements"); - final Group lengthCounters = factory.createSnippetCounterGroup("System.arraycopy 0-length checks"); + final Group callCounters = factory.createSnippetCounterGroup("System.arraycopy calls"); + final Group copiedElementsCounters = factory.createSnippetCounterGroup("System.arraycopy copied elements"); + final Group lengthCounters = factory.createSnippetCounterGroup("System.arraycopy with 0-length"); checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess"); checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE"); - objectCheckcastCounter = new SnippetCounter(counters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays"); - objectCheckcastSameTypeCounter = new SnippetCounter(counters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays"); - predictedObjectArrayCopySlowPathCounter = new SnippetCounter(counters, "Object[]{slow-path}", "used System.arraycopy slow path for predicted Object[] arrays"); - predictedObjectArrayCopyFastPathCounter = new SnippetCounter(counters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays"); - genericPrimitiveCallCounter = new SnippetCounter(counters, "genericPrimitive", "generic arraycopy snippet for primitive arrays"); - genericObjectExactCallCounter = new SnippetCounter(counters, "genericObjectExact", "generic arraycopy snippet for special object arrays"); - systemArraycopyCounter = new SnippetCounter(counters, "genericObject", "call to System.arraycopy"); - - zeroLengthStaticCounter = new SnippetCounter(lengthCounters, "0-lengthcopy static", "arraycopy where the length is statically 0"); - zeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "0-lengthcopy dynamically", "arraycopy where the length is dynamically 0"); - nonZeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero"); - - nonZeroLengthDynamicCopiedCounter = new SnippetCounter(copiedCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero"); - genericPrimitiveCallCopiedCounter = new SnippetCounter(copiedCounters, "genericPrimitive", "generic arraycopy snippet for primitive arrays"); - genericObjectExactCallCopiedCounter = new SnippetCounter(copiedCounters, "genericObjectExact", "generic arraycopy snippet for special object arrays"); - systemArraycopyCopiedCounter = new SnippetCounter(copiedCounters, "genericObject", "call to System.arraycopy"); - - objectCheckcastCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays"); - objectCheckcastSameTypeCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays"); - predictedObjectArrayCopySlowPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{slow-path}", - "used System.arraycopy slow path for predicted Object[] arrays"); - predictedObjectArrayCopyFastPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays"); - createArraycopyCounter(JavaKind.Byte, counters, copiedCounters); - createArraycopyCounter(JavaKind.Boolean, counters, copiedCounters); - createArraycopyCounter(JavaKind.Char, counters, copiedCounters); - createArraycopyCounter(JavaKind.Short, counters, copiedCounters); - createArraycopyCounter(JavaKind.Int, counters, copiedCounters); - createArraycopyCounter(JavaKind.Long, counters, copiedCounters); - createArraycopyCounter(JavaKind.Float, counters, copiedCounters); - createArraycopyCounter(JavaKind.Double, counters, copiedCounters); - createArraycopyCounter(JavaKind.Object, counters, copiedCounters); + zeroLengthStaticCounter = new SnippetCounter(lengthCounters, "0-length copy static", "calls where the length is statically 0"); + lengthHistogram = new SnippetIntegerHistogram(lengthCounters, 2, "length", "length"); + + systemArraycopyCounter = new SnippetCounter(callCounters, "native System.arraycopy", "JNI-based System.arraycopy call"); + systemArraycopyCopiedCounter = new SnippetCounter(copiedElementsCounters, "native System.arraycopy", "JNI-based System.arraycopy call"); + + genericArraycopyDifferentTypeCounter = new SnippetCounter(callCounters, "generic[] stub", "generic arraycopy stub"); + genericArraycopyDifferentTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "generic[] stub", "generic arraycopy stub"); + + objectCheckcastSameTypeCounter = new SnippetCounter(callCounters, "checkcast object[] (same-type)", "checkcast object[] stub but src.klass == dest.klass Object[] arrays"); + objectCheckcastSameTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "checkcast object[] (same-type)", "checkcast object[] stub but src.klass == dest.klass Object[] arrays"); + objectCheckcastDifferentTypeCounter = new SnippetCounter(callCounters, "checkcast object[] (store-check)", "checkcast object[] stub with store check"); + objectCheckcastDifferentTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "checkcast object[] (store-check)", "checkcast object[] stub with store check"); + + createArraycopyCounter(JavaKind.Byte, callCounters, copiedElementsCounters); + createArraycopyCounter(JavaKind.Boolean, callCounters, copiedElementsCounters); + createArraycopyCounter(JavaKind.Char, callCounters, copiedElementsCounters); + createArraycopyCounter(JavaKind.Short, callCounters, copiedElementsCounters); + createArraycopyCounter(JavaKind.Int, callCounters, copiedElementsCounters); + createArraycopyCounter(JavaKind.Long, callCounters, copiedElementsCounters); + createArraycopyCounter(JavaKind.Float, callCounters, copiedElementsCounters); + createArraycopyCounter(JavaKind.Double, callCounters, copiedElementsCounters); + createArraycopyCounter(JavaKind.Object, callCounters, copiedElementsCounters); } void createArraycopyCounter(JavaKind kind, Group counters, Group copiedCounters) { - arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays")); - arraycopyCounters.put(kind, new SnippetCounter(counters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays")); - - arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays")); - arraycopyCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays")); + arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[] stub", "arraycopy call for " + kind + "[] arrays")); + arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[] stub", "arraycopy call for " + kind + "[] arrays")); } } public static class Templates extends SnippetTemplate.AbstractTemplates { + private final SnippetInfo arraycopyGenericSnippet = snippet("arraycopyGenericSnippet"); + private final SnippetInfo arraycopyUnrolledSnippet = snippet("arraycopyUnrolledSnippet"); + private final SnippetInfo arraycopyExactSnippet = snippet("arraycopyExactSnippet"); + private final SnippetInfo arraycopyZeroLengthSnippet = snippet("arraycopyZeroLengthSnippet"); + private final SnippetInfo arraycopyCheckcastSnippet = snippet("arraycopyCheckcastSnippet"); + private final SnippetInfo arraycopyNativeSnippet = snippet("arraycopyNativeSnippet"); + + private final SnippetInfo checkcastArraycopyWithSlowPathWork = snippet("checkcastArraycopyWithSlowPathWork"); + private final SnippetInfo genericArraycopyWithSlowPathWork = snippet("genericArraycopyWithSlowPathWork"); + + private ResolvedJavaMethod originalArraycopy; + private final Counters counters; public Templates(OptionValues options, Iterable factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target) { super(options, factories, providers, providers.getSnippetReflection(), target); this.counters = new Counters(factory); } - private ResolvedJavaMethod originalArraycopy() throws GraalError { - if (originalArraycopy == null) { - Method method; - try { - method = System.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class); - } catch (NoSuchMethodException | SecurityException e) { - throw new GraalError(e); - } - originalArraycopy = providers.getMetaAccess().lookupJavaMethod(method); - } - return originalArraycopy; - } - - private ResolvedJavaMethod originalArraycopy; - - private final SnippetInfo checkcastArraycopyWorkSnippet = snippet("checkcastArraycopyWork"); - private final SnippetInfo arraycopyGenericSnippet = snippet("arraycopyGeneric"); - - private final SnippetInfo arraycopySlowPathIntrinsicSnippet = snippet("arraycopySlowPathIntrinsic"); - private final SnippetInfo arraycopyUnrolledIntrinsicSnippet = snippet("arraycopyUnrolledIntrinsic"); - private final SnippetInfo arraycopyExactIntrinsicSnippet = snippet("arraycopyExactIntrinsic"); - private final SnippetInfo arraycopyZeroLengthIntrinsicSnippet = snippet("arraycopyZeroLengthIntrinsic"); - private final SnippetInfo arraycopyPredictedExactIntrinsicSnippet = snippet("arraycopyPredictedExactIntrinsic"); - private final SnippetInfo arraycopyPredictedObjectWorkSnippet = snippet("arraycopyPredictedObjectWork"); - - private final SnippetInfo arraycopyUnrolledWorkSnippet = snippet("arraycopyUnrolledWork"); - - private final Counters counters; - protected SnippetInfo snippet(String methodName) { SnippetInfo info = snippet(ArrayCopySnippets.class, methodName, LocationIdentity.any()); info.setOriginalMethod(originalArraycopy()); return info; } - public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy) { - return selectComponentKind(arraycopy, true); - } + public void lower(ArrayCopyNode arraycopy, LoweringTool tool) { + JavaKind elementKind = selectComponentKind(arraycopy); + SnippetInfo snippetInfo; + ArrayCopyTypeCheck arrayTypeCheck; - public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy, boolean exact) { ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp()); ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp()); + if (!canBeArray(srcType) || !canBeArray(destType)) { + // at least one of the objects is definitely not an array - use the native call + // right away as the copying will fail anyways + snippetInfo = arraycopyNativeSnippet; + arrayTypeCheck = ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK; + } else { + ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType(); + ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType(); - if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) { - if (!exact) { - JavaKind component = getComponentKind(srcType); - if (component != null) { - return component; + if (arraycopy.isExact()) { + // there is a sufficient type match - we don't need any additional type checks + snippetInfo = arraycopyExactSnippet; + arrayTypeCheck = ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK; + } else if (srcComponentType == null && destComponentType == null) { + // we don't know anything about the types - use the generic copying + snippetInfo = arraycopyGenericSnippet; + arrayTypeCheck = ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK; + } else if (srcComponentType != null && destComponentType != null) { + if (!srcComponentType.isPrimitive() && !destComponentType.isPrimitive()) { + // it depends on the array content if the copy succeeds - we need + // a type check for every store + snippetInfo = arraycopyCheckcastSnippet; + arrayTypeCheck = ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK; + } else { + // one object is an object array, the other one is a primitive array. + // this copy will always fail - use the native call right away + assert !srcComponentType.equals(destComponentType) : "must be handled by arraycopy.isExact()"; + snippetInfo = arraycopyNativeSnippet; + arrayTypeCheck = ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK; + } + } else { + ResolvedJavaType nonNullComponentType = srcComponentType != null ? srcComponentType : destComponentType; + if (nonNullComponentType.isPrimitive()) { + // one involved object is a primitive array - we can safely assume that we + // are copying primitive arrays + snippetInfo = arraycopyExactSnippet; + arrayTypeCheck = ArrayCopyTypeCheck.HUB_BASED_ARRAY_TYPE_CHECK; + elementKind = nonNullComponentType.getJavaKind(); + } else { + // one involved object is an object array - we can safely assume that we are + // copying object arrays that might require a store check + snippetInfo = arraycopyCheckcastSnippet; + arrayTypeCheck = ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK; } - return getComponentKind(destType); - } - return null; - } - if (exact) { - if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) { - return null; - } - if (!arraycopy.isExact()) { - return null; } } - return srcType.getComponentType().getJavaKind(); - } - - private static JavaKind getComponentKind(ResolvedJavaType type) { - if (type != null && type.isArray()) { - return type.getComponentType().getJavaKind(); - } - return null; - } - - private static boolean shouldUnroll(ValueNode length) { - return length.isConstant() && length.asJavaConstant().asInt() <= 8 && length.asJavaConstant().asInt() != 0; - } - - public void lower(ArrayCopyNode arraycopy, LoweringTool tool) { - JavaKind componentKind = selectComponentKind(arraycopy); - SnippetInfo snippetInfo = null; - SnippetInfo slowPathSnippetInfo = null; - Object slowPathArgument = null; + // a few special cases that are easier to handle when all other variables already have a + // value if (arraycopy.getLength().isConstant() && arraycopy.getLength().asJavaConstant().asLong() == 0) { - snippetInfo = arraycopyZeroLengthIntrinsicSnippet; - } else if (arraycopy.isExact()) { - snippetInfo = arraycopyExactIntrinsicSnippet; - if (shouldUnroll(arraycopy.getLength())) { - snippetInfo = arraycopyUnrolledIntrinsicSnippet; - } - } else { - if (componentKind == JavaKind.Object) { - ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp()); - ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp()); - ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType(); - ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType(); - if (srcComponentType != null && destComponentType != null && !srcComponentType.isPrimitive() && !destComponentType.isPrimitive()) { - snippetInfo = arraycopySlowPathIntrinsicSnippet; - slowPathSnippetInfo = checkcastArraycopyWorkSnippet; - slowPathArgument = LocationIdentity.any(); - /* - * Because this snippet has to use Sysytem.arraycopy as a slow path, we must - * pretend to kill any() so clear the componentKind. - */ - componentKind = null; - } - } - if (componentKind == null && snippetInfo == null) { - JavaKind predictedKind = selectComponentKind(arraycopy, false); - if (predictedKind != null) { - /* - * At least one array is of a known type requiring no store checks, so - * assume the other is of the same type. Generally this is working around - * deficiencies in our propagation of type information. - */ - componentKind = predictedKind; - if (predictedKind == JavaKind.Object) { - snippetInfo = arraycopySlowPathIntrinsicSnippet; - slowPathSnippetInfo = arraycopyPredictedObjectWorkSnippet; - slowPathArgument = predictedKind; - componentKind = null; - } else { - snippetInfo = arraycopyPredictedExactIntrinsicSnippet; - } - } - } - if (snippetInfo == null) { - snippetInfo = arraycopyGenericSnippet; - } + snippetInfo = arraycopyZeroLengthSnippet; + } else if (snippetInfo == arraycopyExactSnippet && shouldUnroll(arraycopy.getLength())) { + snippetInfo = arraycopyUnrolledSnippet; } + + // create the snippet Arguments args = new Arguments(snippetInfo, arraycopy.graph().getGuardsStage(), tool.getLoweringStage()); args.add("src", arraycopy.getSource()); args.add("srcPos", arraycopy.getSourcePosition()); args.add("dest", arraycopy.getDestination()); args.add("destPos", arraycopy.getDestinationPosition()); args.add("length", arraycopy.getLength()); - if (snippetInfo == arraycopyUnrolledIntrinsicSnippet) { + if (snippetInfo != arraycopyNativeSnippet) { + assert arrayTypeCheck != ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK; + args.addConst("arrayTypeCheck", arrayTypeCheck); + } + if (snippetInfo == arraycopyUnrolledSnippet) { + args.addConst("elementKind", elementKind != null ? elementKind : JavaKind.Illegal); args.addConst("unrolledLength", arraycopy.getLength().asJavaConstant().asInt()); - args.addConst("elementKind", componentKind != null ? componentKind : JavaKind.Illegal); - } else if (snippetInfo == arraycopySlowPathIntrinsicSnippet) { - ValueNode predictedKlass = null; - if (slowPathArgument == arraycopyPredictedObjectWorkSnippet) { - HotSpotResolvedObjectType arrayClass = (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(Object[].class); - predictedKlass = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayClass.klass(), tool.getMetaAccess(), arraycopy.graph()); - } else { - predictedKlass = ConstantNode.forConstant(KlassPointerStamp.klassAlwaysNull(), JavaConstant.NULL_POINTER, tool.getMetaAccess(), arraycopy.graph()); - } - args.add("predictedKlass", predictedKlass); - args.addConst("elementKind", componentKind != null ? componentKind : JavaKind.Illegal); - args.addConst("slowPath", slowPathSnippetInfo); - assert slowPathArgument != null; - args.addConst("slowPathArgument", slowPathArgument); - } else if (snippetInfo == arraycopyExactIntrinsicSnippet || snippetInfo == arraycopyPredictedExactIntrinsicSnippet) { - assert componentKind != null; - args.addConst("elementKind", componentKind); - args.addConst("counter", counters.arraycopyCallCounters.get(componentKind)); - args.addConst("copiedCounter", counters.arraycopyCallCopiedCounters.get(componentKind)); + } + if (snippetInfo == arraycopyExactSnippet) { + assert elementKind != null; + args.addConst("elementKind", elementKind); + args.addConst("elementKindCounter", counters.arraycopyCallCounters.get(elementKind)); + args.addConst("elementKindCopiedCounter", counters.arraycopyCallCopiedCounters.get(elementKind)); } args.addConst("counters", counters); + if (snippetInfo == arraycopyCheckcastSnippet) { + args.addConst("workSnippet", checkcastArraycopyWithSlowPathWork); + args.addConst("elementKind", JavaKind.Illegal); + } + if (snippetInfo == arraycopyGenericSnippet) { + args.addConst("workSnippet", genericArraycopyWithSlowPathWork); + args.addConst("elementKind", JavaKind.Illegal); + } + instantiate(args, arraycopy); } - public void lower(ArrayCopySlowPathNode arraycopy, LoweringTool tool) { + public void lower(ArrayCopyWithSlowPathNode arraycopy, LoweringTool tool) { StructuredGraph graph = arraycopy.graph(); if (!graph.getGuardsStage().areFrameStatesAtDeopts()) { - // Can't be lowered yet + // if an arraycopy contains a slow path, we can't lower it right away return; } + SnippetInfo snippetInfo = arraycopy.getSnippet(); Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage()); - args.add("nonNullSrc", arraycopy.getSource()); + args.add("src", arraycopy.getSource()); args.add("srcPos", arraycopy.getSourcePosition()); - args.add("nonNullDest", arraycopy.getDestination()); + args.add("dest", arraycopy.getDestination()); args.add("destPos", arraycopy.getDestinationPosition()); - if (snippetInfo == arraycopyUnrolledWorkSnippet) { - args.addConst("length", ((Integer) arraycopy.getArgument()).intValue()); - args.addConst("elementKind", arraycopy.getElementKind()); - } else { - args.add("length", arraycopy.getLength()); - } - if (snippetInfo == arraycopyPredictedObjectWorkSnippet) { - args.add("objectArrayKlass", arraycopy.getPredictedKlass()); - args.addConst("counter", counters.arraycopyCallCounters.get(JavaKind.Object)); - args.addConst("copiedCounter", counters.arraycopyCallCopiedCounters.get(JavaKind.Object)); - args.addConst("counters", counters); - } + args.add("length", arraycopy.getLength()); + args.addConst("counters", counters); instantiate(args, arraycopy); } - public void lower(ArrayCopyUnrollNode arraycopy, LoweringTool tool) { - StructuredGraph graph = arraycopy.graph(); - if (!graph.getGuardsStage().areFrameStatesAtDeopts()) { - // Can't be lowered yet - return; + private static boolean canBeArray(ResolvedJavaType type) { + return type == null || type.isJavaLangObject() || type.isArray(); + } + + public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy) { + ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp()); + ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp()); + + if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) { + return null; } - SnippetInfo snippetInfo = arraycopyUnrolledWorkSnippet; - Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage()); - args.add("nonNullSrc", arraycopy.getSource()); - args.add("srcPos", arraycopy.getSourcePosition()); - args.add("nonNullDest", arraycopy.getDestination()); - args.add("destPos", arraycopy.getDestinationPosition()); - args.addConst("length", arraycopy.getUnrollLength()); - args.addConst("elementKind", arraycopy.getElementKind()); - template(graph.getDebug(), args).instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args); + if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) { + return null; + } + if (!arraycopy.isExact()) { + return null; + } + return srcType.getComponentType().getJavaKind(); + } + + private static boolean shouldUnroll(ValueNode length) { + return length.isConstant() && length.asJavaConstant().asInt() <= 8 && length.asJavaConstant().asInt() != 0; } /** @@ -650,8 +543,8 @@ newInvoke.setStateAfter(arraycopy.stateAfter()); } graph.replaceFixedWithFixed((InvokeNode) invoke.asNode(), newInvoke); - } else if (originalNode instanceof ArrayCopySlowPathNode) { - ArrayCopySlowPathNode slowPath = (ArrayCopySlowPathNode) replacements.get(originalNode); + } else if (originalNode instanceof ArrayCopyWithSlowPathNode) { + ArrayCopyWithSlowPathNode slowPath = (ArrayCopyWithSlowPathNode) replacements.get(originalNode); assert arraycopy.stateAfter() != null : arraycopy; assert slowPath.stateAfter() == arraycopy.stateAfter(); slowPath.setBci(arraycopy.getBci()); @@ -659,5 +552,18 @@ } GraphUtil.killCFG(arraycopy); } + + private ResolvedJavaMethod originalArraycopy() throws GraalError { + if (originalArraycopy == null) { + Method method; + try { + method = System.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class); + } catch (NoSuchMethodException | SecurityException e) { + throw new GraalError(e); + } + originalArraycopy = providers.getMetaAccess().lookupJavaMethod(method); + } + return originalArraycopy; + } } }