--- /dev/null 2016-05-31 09:42:47.975716356 -0700 +++ new/src/jdk.vm.compiler/share/classes/org.graalvm.compiler.hotspot/src/org/graalvm/compiler/hotspot/replacements/arraycopy/ArrayCopySnippets.java 2016-12-07 13:50:47.608879863 -0800 @@ -0,0 +1,628 @@ +/* + * Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.graalvm.compiler.hotspot.replacements.arraycopy; + +import static org.graalvm.compiler.core.common.GraalOptions.SnippetCounters; +import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG; +import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.KLASS_SUPER_CHECK_OFFSET_LOCATION; +import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION; +import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset; +import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayClassElementOffset; +import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayIndexScale; +import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperElementTypePrimitiveInPlace; +import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadHub; +import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper; +import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.superCheckOffsetOffset; +import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY; +import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY; +import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; + +import java.lang.reflect.Method; +import java.util.EnumMap; +import java.util.Map; + +import org.graalvm.compiler.api.directives.GraalDirectives; +import org.graalvm.compiler.api.replacements.Fold; +import org.graalvm.compiler.api.replacements.Snippet; +import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter; +import org.graalvm.compiler.core.common.LocationIdentity; +import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.graph.Node; +import org.graalvm.compiler.hotspot.meta.HotSpotProviders; +import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp; +import org.graalvm.compiler.hotspot.word.KlassPointer; +import org.graalvm.compiler.nodes.CallTargetNode; +import org.graalvm.compiler.nodes.ConstantNode; +import org.graalvm.compiler.nodes.DeoptimizeNode; +import org.graalvm.compiler.nodes.Invoke; +import org.graalvm.compiler.nodes.InvokeNode; +import org.graalvm.compiler.nodes.NamedLocationIdentity; +import org.graalvm.compiler.nodes.PiNode; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.extended.UnsafeLoadNode; +import org.graalvm.compiler.nodes.java.ArrayLengthNode; +import org.graalvm.compiler.nodes.spi.LoweringTool; +import org.graalvm.compiler.nodes.type.StampTool; +import org.graalvm.compiler.replacements.SnippetCounter; +import org.graalvm.compiler.replacements.SnippetTemplate; +import org.graalvm.compiler.replacements.SnippetTemplate.Arguments; +import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo; +import org.graalvm.compiler.replacements.Snippets; +import org.graalvm.compiler.replacements.nodes.BasicArrayCopyNode; +import org.graalvm.compiler.replacements.nodes.DirectObjectStoreNode; +import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode; +import org.graalvm.compiler.word.Word; + +import jdk.vm.ci.code.TargetDescription; +import jdk.vm.ci.hotspot.HotSpotResolvedObjectType; +import jdk.vm.ci.meta.DeoptimizationAction; +import jdk.vm.ci.meta.DeoptimizationReason; +import jdk.vm.ci.meta.JavaConstant; +import jdk.vm.ci.meta.JavaKind; +import jdk.vm.ci.meta.ResolvedJavaMethod; +import jdk.vm.ci.meta.ResolvedJavaType; + +public class ArrayCopySnippets implements Snippets { + + private static int checkArrayType(KlassPointer hub) { + int layoutHelper = readLayoutHelper(hub); + if (probability(SLOW_PATH_PROBABILITY, layoutHelper >= 0)) { + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); + } + return layoutHelper; + } + + private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length) { + if (probability(SLOW_PATH_PROBABILITY, srcPos < 0)) { + checkAIOOBECounter.inc(); + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); + } + if (probability(SLOW_PATH_PROBABILITY, destPos < 0)) { + checkAIOOBECounter.inc(); + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); + } + if (probability(SLOW_PATH_PROBABILITY, length < 0)) { + checkAIOOBECounter.inc(); + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); + } + if (probability(SLOW_PATH_PROBABILITY, srcPos > ArrayLengthNode.arrayLength(src) - length)) { + checkAIOOBECounter.inc(); + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); + } + if (probability(SLOW_PATH_PROBABILITY, destPos > ArrayLengthNode.arrayLength(dest) - length)) { + checkAIOOBECounter.inc(); + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); + } + checkSuccessCounter.inc(); + } + + @Snippet + public static void arraycopyZeroLengthIntrinsic(Object src, int srcPos, Object dest, int destPos, int length) { + Object nonNullSrc = GraalDirectives.guardingNonNull(src); + Object nonNullDest = GraalDirectives.guardingNonNull(dest); + KlassPointer srcHub = loadHub(nonNullSrc); + KlassPointer destHub = loadHub(nonNullDest); + checkArrayType(srcHub); + checkArrayType(destHub); + checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); + zeroLengthStaticCounter.inc(); + } + + @Snippet + public static void arraycopyExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind, @ConstantParameter SnippetCounter counter, + @ConstantParameter SnippetCounter copiedCounter) { + Object nonNullSrc = GraalDirectives.guardingNonNull(src); + Object nonNullDest = GraalDirectives.guardingNonNull(dest); + checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); + counter.inc(); + copiedCounter.add(length); + ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind); + if (length == 0) { + zeroLengthDynamicCounter.inc(); + } else { + nonZeroLengthDynamicCounter.inc(); + nonZeroLengthDynamicCopiedCounter.add(length); + } + } + + /** + * This intrinsic is useful for the case where we know something statically about one of the + * inputs but not the other. + */ + @Snippet + public static void arraycopyPredictedExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind, + @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter) { + Object nonNullSrc = GraalDirectives.guardingNonNull(src); + Object nonNullDest = GraalDirectives.guardingNonNull(dest); + KlassPointer srcHub = loadHub(nonNullSrc); + KlassPointer destHub = loadHub(nonNullDest); + if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) { + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); + } + checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); + counter.inc(); + copiedCounter.add(length); + ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind); + if (length == 0) { + zeroLengthDynamicCounter.inc(); + } else { + nonZeroLengthDynamicCounter.inc(); + nonZeroLengthDynamicCopiedCounter.add(length); + } + } + + @Snippet + public static void arraycopyPredictedObjectWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, KlassPointer objectArrayKlass, + @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter) { + if (length > 0) { + KlassPointer srcHub = loadHub(PiNode.asNonNullObject(nonNullSrc)); + KlassPointer destHub = loadHub(PiNode.asNonNullObject(nonNullDest)); + if (probability(FAST_PATH_PROBABILITY, srcHub == destHub || destHub == objectArrayKlass)) { + counter.inc(); + copiedCounter.add(length); + predictedObjectArrayCopyFastPathCounter.inc(); + predictedObjectArrayCopyFastPathCopiedCounter.add(length); + ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length); + } else { + predictedObjectArrayCopySlowPathCounter.inc(); + predictedObjectArrayCopySlowPathCopiedCounter.add(length); + System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length); + } + } + } + + /** + * This is the basic template for the full arraycopy checks, including a check that the + * underlying type is really an array type. + */ + @Snippet + public static void arraycopySlowPathIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, KlassPointer predictedKlass, @ConstantParameter JavaKind elementKind, + @ConstantParameter SnippetInfo slowPath, @ConstantParameter Object slowPathArgument) { + Object nonNullSrc = GraalDirectives.guardingNonNull(src); + Object nonNullDest = GraalDirectives.guardingNonNull(dest); + KlassPointer srcHub = loadHub(nonNullSrc); + KlassPointer destHub = loadHub(nonNullDest); + checkArrayType(srcHub); + checkArrayType(destHub); + checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); + if (length == 0) { + zeroLengthDynamicCounter.inc(); + } else { + nonZeroLengthDynamicCounter.inc(); + nonZeroLengthDynamicCopiedCounter.add(length); + } + ArrayCopySlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, predictedKlass, elementKind, slowPath, slowPathArgument); + } + + /** + * Snippet for unrolled arraycopy. + */ + @Snippet + public static void arraycopyUnrolledIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter int unrolledLength, @ConstantParameter JavaKind elementKind) { + Object nonNullSrc = GraalDirectives.guardingNonNull(src); + Object nonNullDest = GraalDirectives.guardingNonNull(dest); + checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); + if (length == 0) { + zeroLengthDynamicCounter.inc(); + } else { + nonZeroLengthDynamicCounter.inc(); + nonZeroLengthDynamicCopiedCounter.add(length); + } + ArrayCopyUnrollNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, unrolledLength, elementKind); + } + + @Snippet + public static void checkcastArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length) { + if (length > 0) { + KlassPointer destKlass = loadHub(nonNullDest); + KlassPointer srcKlass = loadHub(nonNullSrc); + if (probability(SLOW_PATH_PROBABILITY, srcKlass == destKlass)) { + // no storecheck required. + objectCheckcastSameTypeCounter.inc(); + objectCheckcastSameTypeCopiedCounter.add(length); + ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length); + } else { + KlassPointer destElemKlass = destKlass.readKlassPointer(arrayClassElementOffset(INJECTED_VMCONFIG), OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION); + Word superCheckOffset = Word.signed(destElemKlass.readInt(superCheckOffsetOffset(INJECTED_VMCONFIG), KLASS_SUPER_CHECK_OFFSET_LOCATION)); + objectCheckcastCounter.inc(); + objectCheckcastCopiedCounter.add(length); + int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false); + if (copiedElements != 0) { + /* + * the checkcast stub doesn't throw the ArrayStoreException, but returns the + * number of copied elements (xor'd with -1). + */ + copiedElements ^= -1; + System.arraycopy(nonNullSrc, srcPos + copiedElements, nonNullDest, destPos + copiedElements, length - copiedElements); + } + } + } + } + + @Snippet + public static void arraycopyGeneric(Object src, int srcPos, Object dest, int destPos, int length) { + Object nonNullSrc = GraalDirectives.guardingNonNull(src); + Object nonNullDest = GraalDirectives.guardingNonNull(dest); + KlassPointer srcHub = loadHub(nonNullSrc); + KlassPointer destHub = loadHub(nonNullDest); + if (probability(FAST_PATH_PROBABILITY, srcHub.equal(destHub)) && probability(FAST_PATH_PROBABILITY, nonNullSrc != nonNullDest)) { + int layoutHelper = checkArrayType(srcHub); + final boolean isObjectArray = ((layoutHelper & layoutHelperElementTypePrimitiveInPlace(INJECTED_VMCONFIG)) == 0); + checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); + if (probability(FAST_PATH_PROBABILITY, isObjectArray)) { + genericObjectExactCallCounter.inc(); + genericObjectExactCallCopiedCounter.add(length); + ArrayCopyCallNode.disjointArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, JavaKind.Object); + } else { + genericPrimitiveCallCounter.inc(); + genericPrimitiveCallCopiedCounter.add(length); + UnsafeArrayCopyNode.arraycopyPrimitive(nonNullSrc, srcPos, nonNullDest, destPos, length, layoutHelper); + } + } else { + SystemArraycopyCounter.inc(); + SystemArraycopyCopiedCounter.add(length); + System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length); + } + } + + @Fold + static LocationIdentity getArrayLocation(JavaKind kind) { + return NamedLocationIdentity.getArrayLocation(kind); + } + + @Snippet + public static void arraycopyUnrolledWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, @ConstantParameter int length, @ConstantParameter JavaKind elementKind) { + final int scale = arrayIndexScale(elementKind); + int arrayBaseOffset = arrayBaseOffset(elementKind); + LocationIdentity arrayLocation = getArrayLocation(elementKind); + if (nonNullSrc == nonNullDest && srcPos < destPos) { // bad aliased case + long start = (long) (length - 1) * scale; + long i = start; + ExplodeLoopNode.explodeLoop(); + for (int iteration = 0; iteration < length; iteration++) { + if (i >= 0) { + Object a = UnsafeLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation); + DirectObjectStoreNode.storeObject(nonNullDest, arrayBaseOffset, i + (long) destPos * scale, a, arrayLocation, elementKind); + i -= scale; + } + } + } else { + long end = (long) length * scale; + long i = 0; + ExplodeLoopNode.explodeLoop(); + for (int iteration = 0; iteration < length; iteration++) { + if (i < end) { + Object a = UnsafeLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation); + DirectObjectStoreNode.storeObject(nonNullDest, arrayBaseOffset, i + (long) destPos * scale, a, arrayLocation, elementKind); + i += scale; + } + } + } + } + + private static final SnippetCounter.Group checkCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy checkInputs") : null; + private static final SnippetCounter checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess"); + private static final SnippetCounter checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE"); + + private static final SnippetCounter.Group counters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy") : null; + + private static final SnippetCounter objectCheckcastCounter = new SnippetCounter(counters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays"); + private static final SnippetCounter objectCheckcastSameTypeCounter = new SnippetCounter(counters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays"); + private static final SnippetCounter predictedObjectArrayCopySlowPathCounter = new SnippetCounter(counters, "Object[]{slow-path}", "used System.arraycopy slow path for predicted Object[] arrays"); + private static final SnippetCounter predictedObjectArrayCopyFastPathCounter = new SnippetCounter(counters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays"); + + private static final EnumMap arraycopyCallCounters = new EnumMap<>(JavaKind.class); + private static final EnumMap arraycopyCounters = new EnumMap<>(JavaKind.class); + + private static final EnumMap arraycopyCallCopiedCounters = new EnumMap<>(JavaKind.class); + private static final EnumMap arraycopyCopiedCounters = new EnumMap<>(JavaKind.class); + + static void createArraycopyCounter(JavaKind kind) { + arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays")); + arraycopyCounters.put(kind, new SnippetCounter(counters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays")); + + arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays")); + arraycopyCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays")); + } + + static { + createArraycopyCounter(JavaKind.Byte); + createArraycopyCounter(JavaKind.Boolean); + createArraycopyCounter(JavaKind.Char); + createArraycopyCounter(JavaKind.Short); + createArraycopyCounter(JavaKind.Int); + createArraycopyCounter(JavaKind.Long); + createArraycopyCounter(JavaKind.Float); + createArraycopyCounter(JavaKind.Double); + createArraycopyCounter(JavaKind.Object); + } + + private static final SnippetCounter genericPrimitiveCallCounter = new SnippetCounter(counters, "genericPrimitive", "generic arraycopy snippet for primitive arrays"); + private static final SnippetCounter genericObjectExactCallCounter = new SnippetCounter(counters, "genericObjectExact", "generic arraycopy snippet for special object arrays"); + private static final SnippetCounter SystemArraycopyCounter = new SnippetCounter(counters, "genericObject", "call to System.arraycopy"); + + private static final SnippetCounter.Group lengthCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy 0-length checks") : null; + + private static final SnippetCounter zeroLengthStaticCounter = new SnippetCounter(lengthCounters, "0-lengthcopy static", "arraycopy where the length is statically 0"); + private static final SnippetCounter zeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "0-lengthcopy dynamically", "arraycopy where the length is dynamically 0"); + private static final SnippetCounter nonZeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero"); + + private static final SnippetCounter.Group copiedCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy copied elements") : null; + + private static final SnippetCounter nonZeroLengthDynamicCopiedCounter = new SnippetCounter(copiedCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero"); + private static final SnippetCounter genericPrimitiveCallCopiedCounter = new SnippetCounter(copiedCounters, "genericPrimitive", "generic arraycopy snippet for primitive arrays"); + private static final SnippetCounter genericObjectExactCallCopiedCounter = new SnippetCounter(copiedCounters, "genericObjectExact", "generic arraycopy snippet for special object arrays"); + private static final SnippetCounter SystemArraycopyCopiedCounter = new SnippetCounter(copiedCounters, "genericObject", "call to System.arraycopy"); + + private static final SnippetCounter objectCheckcastCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays"); + private static final SnippetCounter objectCheckcastSameTypeCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays"); + private static final SnippetCounter predictedObjectArrayCopySlowPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{slow-path}", + "used System.arraycopy slow path for predicted Object[] arrays"); + private static final SnippetCounter predictedObjectArrayCopyFastPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays"); + + public static class Templates extends SnippetTemplate.AbstractTemplates { + + public Templates(HotSpotProviders providers, TargetDescription target) { + super(providers, providers.getSnippetReflection(), target); + } + + private ResolvedJavaMethod originalArraycopy() throws GraalError { + if (originalArraycopy == null) { + Method method; + try { + method = System.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class); + } catch (NoSuchMethodException | SecurityException e) { + throw new GraalError(e); + } + originalArraycopy = providers.getMetaAccess().lookupJavaMethod(method); + } + return originalArraycopy; + } + + private ResolvedJavaMethod originalArraycopy; + + private final SnippetInfo checkcastArraycopyWorkSnippet = snippet("checkcastArraycopyWork"); + private final SnippetInfo arraycopyGenericSnippet = snippet("arraycopyGeneric"); + + private final SnippetInfo arraycopySlowPathIntrinsicSnippet = snippet("arraycopySlowPathIntrinsic"); + private final SnippetInfo arraycopyUnrolledIntrinsicSnippet = snippet("arraycopyUnrolledIntrinsic"); + private final SnippetInfo arraycopyExactIntrinsicSnippet = snippet("arraycopyExactIntrinsic"); + private final SnippetInfo arraycopyZeroLengthIntrinsicSnippet = snippet("arraycopyZeroLengthIntrinsic"); + private final SnippetInfo arraycopyPredictedExactIntrinsicSnippet = snippet("arraycopyPredictedExactIntrinsic"); + private final SnippetInfo arraycopyPredictedObjectWorkSnippet = snippet("arraycopyPredictedObjectWork"); + + private final SnippetInfo arraycopyUnrolledWorkSnippet = snippet("arraycopyUnrolledWork"); + + protected SnippetInfo snippet(String methodName) { + SnippetInfo info = snippet(ArrayCopySnippets.class, methodName, LocationIdentity.any()); + info.setOriginalMethod(originalArraycopy()); + return info; + } + + public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy) { + return selectComponentKind(arraycopy, true); + } + + public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy, boolean exact) { + ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp()); + ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp()); + + if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) { + if (!exact) { + JavaKind component = getComponentKind(srcType); + if (component != null) { + return component; + } + return getComponentKind(destType); + } + return null; + } + if (exact) { + if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) { + return null; + } + if (!arraycopy.isExact()) { + return null; + } + } + return srcType.getComponentType().getJavaKind(); + } + + private static JavaKind getComponentKind(ResolvedJavaType type) { + if (type != null && type.isArray()) { + return type.getComponentType().getJavaKind(); + } + return null; + } + + private static boolean shouldUnroll(ValueNode length) { + return length.isConstant() && length.asJavaConstant().asInt() <= 8 && length.asJavaConstant().asInt() != 0; + } + + public void lower(ArrayCopyNode arraycopy, LoweringTool tool) { + JavaKind componentKind = selectComponentKind(arraycopy); + SnippetInfo snippetInfo = null; + SnippetInfo slowPathSnippetInfo = null; + Object slowPathArgument = null; + + if (arraycopy.getLength().isConstant() && arraycopy.getLength().asJavaConstant().asLong() == 0) { + snippetInfo = arraycopyZeroLengthIntrinsicSnippet; + } else if (arraycopy.isExact()) { + snippetInfo = arraycopyExactIntrinsicSnippet; + if (shouldUnroll(arraycopy.getLength())) { + snippetInfo = arraycopyUnrolledIntrinsicSnippet; + } + } else { + if (componentKind == JavaKind.Object) { + ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp()); + ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp()); + ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType(); + ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType(); + if (srcComponentType != null && destComponentType != null && !srcComponentType.isPrimitive() && !destComponentType.isPrimitive()) { + snippetInfo = arraycopySlowPathIntrinsicSnippet; + slowPathSnippetInfo = checkcastArraycopyWorkSnippet; + slowPathArgument = LocationIdentity.any(); + /* + * Because this snippet has to use Sysytem.arraycopy as a slow path, we must + * pretend to kill any() so clear the componentKind. + */ + componentKind = null; + } + } + if (componentKind == null && snippetInfo == null) { + JavaKind predictedKind = selectComponentKind(arraycopy, false); + if (predictedKind != null) { + /* + * At least one array is of a known type requiring no store checks, so + * assume the other is of the same type. Generally this is working around + * deficiencies in our propagation of type information. + */ + componentKind = predictedKind; + if (predictedKind == JavaKind.Object) { + snippetInfo = arraycopySlowPathIntrinsicSnippet; + slowPathSnippetInfo = arraycopyPredictedObjectWorkSnippet; + slowPathArgument = predictedKind; + componentKind = null; + } else { + snippetInfo = arraycopyPredictedExactIntrinsicSnippet; + } + } + } + if (snippetInfo == null) { + snippetInfo = arraycopyGenericSnippet; + } + } + Arguments args = new Arguments(snippetInfo, arraycopy.graph().getGuardsStage(), tool.getLoweringStage()); + args.add("src", arraycopy.getSource()); + args.add("srcPos", arraycopy.getSourcePosition()); + args.add("dest", arraycopy.getDestination()); + args.add("destPos", arraycopy.getDestinationPosition()); + args.add("length", arraycopy.getLength()); + if (snippetInfo == arraycopyUnrolledIntrinsicSnippet) { + args.addConst("unrolledLength", arraycopy.getLength().asJavaConstant().asInt()); + args.addConst("elementKind", componentKind != null ? componentKind : JavaKind.Illegal); + } else if (snippetInfo == arraycopySlowPathIntrinsicSnippet) { + ValueNode predictedKlass = null; + if (slowPathArgument == arraycopyPredictedObjectWorkSnippet) { + HotSpotResolvedObjectType arrayClass = (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(Object[].class); + predictedKlass = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayClass.klass(), tool.getMetaAccess(), arraycopy.graph()); + } else { + predictedKlass = ConstantNode.forConstant(KlassPointerStamp.klassAlwaysNull(), JavaConstant.NULL_POINTER, tool.getMetaAccess(), arraycopy.graph()); + } + args.add("predictedKlass", predictedKlass); + args.addConst("elementKind", componentKind != null ? componentKind : JavaKind.Illegal); + args.addConst("slowPath", slowPathSnippetInfo); + assert slowPathArgument != null; + args.addConst("slowPathArgument", slowPathArgument); + } else if (snippetInfo == arraycopyExactIntrinsicSnippet || snippetInfo == arraycopyPredictedExactIntrinsicSnippet) { + assert componentKind != null; + args.addConst("elementKind", componentKind); + args.addConst("counter", arraycopyCallCounters.get(componentKind)); + args.addConst("copiedCounter", arraycopyCallCopiedCounters.get(componentKind)); + } + instantiate(args, arraycopy); + } + + public void lower(ArrayCopySlowPathNode arraycopy, LoweringTool tool) { + StructuredGraph graph = arraycopy.graph(); + if (!graph.getGuardsStage().areFrameStatesAtDeopts()) { + // Can't be lowered yet + return; + } + SnippetInfo snippetInfo = arraycopy.getSnippet(); + Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage()); + args.add("nonNullSrc", arraycopy.getSource()); + args.add("srcPos", arraycopy.getSourcePosition()); + args.add("nonNullDest", arraycopy.getDestination()); + args.add("destPos", arraycopy.getDestinationPosition()); + if (snippetInfo == arraycopyUnrolledWorkSnippet) { + args.addConst("length", ((Integer) arraycopy.getArgument()).intValue()); + args.addConst("elementKind", arraycopy.getElementKind()); + } else { + args.add("length", arraycopy.getLength()); + } + if (snippetInfo == arraycopyPredictedObjectWorkSnippet) { + args.add("objectArrayKlass", arraycopy.getPredictedKlass()); + args.addConst("counter", arraycopyCallCounters.get(JavaKind.Object)); + args.addConst("copiedCounter", arraycopyCallCopiedCounters.get(JavaKind.Object)); + } + instantiate(args, arraycopy); + } + + public void lower(ArrayCopyUnrollNode arraycopy, LoweringTool tool) { + StructuredGraph graph = arraycopy.graph(); + if (!graph.getGuardsStage().areFrameStatesAtDeopts()) { + // Can't be lowered yet + return; + } + SnippetInfo snippetInfo = arraycopyUnrolledWorkSnippet; + Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage()); + args.add("nonNullSrc", arraycopy.getSource()); + args.add("srcPos", arraycopy.getSourcePosition()); + args.add("nonNullDest", arraycopy.getDestination()); + args.add("destPos", arraycopy.getDestinationPosition()); + args.addConst("length", arraycopy.getUnrollLength()); + args.addConst("elementKind", arraycopy.getElementKind()); + template(args).instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args); + } + + /** + * Instantiate the snippet template and fix up the FrameState of any Invokes of + * System.arraycopy and propagate the captured bci in the ArrayCopySlowPathNode. + * + * @param args + * @param arraycopy + */ + private void instantiate(Arguments args, BasicArrayCopyNode arraycopy) { + StructuredGraph graph = arraycopy.graph(); + SnippetTemplate template = template(args); + Map replacements = template.instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args); + for (Node originalNode : replacements.keySet()) { + if (originalNode instanceof Invoke) { + Invoke invoke = (Invoke) replacements.get(originalNode); + assert invoke.asNode().graph() == graph; + CallTargetNode call = invoke.callTarget(); + + if (!call.targetMethod().equals(originalArraycopy)) { + throw new GraalError("unexpected invoke %s in snippet", call.targetMethod()); + } + // Here we need to fix the bci of the invoke + InvokeNode newInvoke = graph.add(new InvokeNode(invoke.callTarget(), arraycopy.getBci())); + if (arraycopy.stateDuring() != null) { + newInvoke.setStateDuring(arraycopy.stateDuring()); + } else { + assert arraycopy.stateAfter() != null; + newInvoke.setStateAfter(arraycopy.stateAfter()); + } + graph.replaceFixedWithFixed((InvokeNode) invoke.asNode(), newInvoke); + } else if (originalNode instanceof ArrayCopySlowPathNode) { + ArrayCopySlowPathNode slowPath = (ArrayCopySlowPathNode) replacements.get(originalNode); + assert arraycopy.stateAfter() != null; + slowPath.setStateAfter(arraycopy.stateAfter()); + slowPath.setBci(arraycopy.getBci()); + } + } + } + } +}