1 /*
   2  * Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 package org.graalvm.compiler.hotspot.replacements.arraycopy;
  24 
  25 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
  26 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.KLASS_SUPER_CHECK_OFFSET_LOCATION;
  27 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION;
  28 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset;
  29 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayClassElementOffset;
  30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayIndexScale;
  31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperElementTypePrimitiveInPlace;
  32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadHub;
  33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
  34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.superCheckOffsetOffset;
  35 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
  36 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
  37 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
  38 
  39 import java.lang.reflect.Method;
  40 import java.util.EnumMap;
  41 
  42 import org.graalvm.compiler.api.directives.GraalDirectives;
  43 import org.graalvm.compiler.api.replacements.Fold;
  44 import org.graalvm.compiler.api.replacements.Snippet;
  45 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
  46 import org.graalvm.compiler.debug.DebugHandlersFactory;
  47 import org.graalvm.compiler.debug.GraalError;
  48 import org.graalvm.compiler.graph.Node;
  49 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
  50 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
  51 import org.graalvm.compiler.hotspot.word.KlassPointer;
  52 import org.graalvm.compiler.nodes.CallTargetNode;
  53 import org.graalvm.compiler.nodes.ConstantNode;
  54 import org.graalvm.compiler.nodes.DeoptimizeNode;
  55 import org.graalvm.compiler.nodes.Invoke;
  56 import org.graalvm.compiler.nodes.InvokeNode;
  57 import org.graalvm.compiler.nodes.NamedLocationIdentity;
  58 import org.graalvm.compiler.nodes.PiNode;
  59 import org.graalvm.compiler.nodes.StructuredGraph;
  60 import org.graalvm.compiler.nodes.ValueNode;
  61 import org.graalvm.compiler.nodes.extended.RawLoadNode;
  62 import org.graalvm.compiler.nodes.extended.RawStoreNode;
  63 import org.graalvm.compiler.nodes.java.ArrayLengthNode;
  64 import org.graalvm.compiler.nodes.spi.LoweringTool;
  65 import org.graalvm.compiler.nodes.type.StampTool;
  66 import org.graalvm.compiler.nodes.util.GraphUtil;
  67 import org.graalvm.compiler.options.OptionValues;
  68 import org.graalvm.compiler.replacements.SnippetCounter;
  69 import org.graalvm.compiler.replacements.SnippetCounter.Group;
  70 import org.graalvm.compiler.replacements.SnippetTemplate;
  71 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
  72 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
  73 import org.graalvm.compiler.replacements.Snippets;
  74 import org.graalvm.compiler.replacements.nodes.BasicArrayCopyNode;
  75 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
  76 import org.graalvm.compiler.word.Word;
  77 import org.graalvm.util.UnmodifiableEconomicMap;
  78 import org.graalvm.word.LocationIdentity;
  79 import org.graalvm.word.WordFactory;
  80 
  81 import jdk.vm.ci.code.TargetDescription;
  82 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
  83 import jdk.vm.ci.meta.DeoptimizationAction;
  84 import jdk.vm.ci.meta.DeoptimizationReason;
  85 import jdk.vm.ci.meta.JavaConstant;
  86 import jdk.vm.ci.meta.JavaKind;
  87 import jdk.vm.ci.meta.ResolvedJavaMethod;
  88 import jdk.vm.ci.meta.ResolvedJavaType;
  89 
  90 public class ArrayCopySnippets implements Snippets {
  91 
  92     private static int checkArrayType(KlassPointer nonNullHub) {
  93         int layoutHelper = readLayoutHelper(nonNullHub);
  94         if (probability(SLOW_PATH_PROBABILITY, layoutHelper >= 0)) {
  95             DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
  96         }
  97         return layoutHelper;
  98     }
  99 
 100     private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length, Counters counters) {
 101         if (probability(SLOW_PATH_PROBABILITY, srcPos < 0)) {
 102             counters.checkAIOOBECounter.inc();
 103             DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
 104         }
 105         if (probability(SLOW_PATH_PROBABILITY, destPos < 0)) {
 106             counters.checkAIOOBECounter.inc();
 107             DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
 108         }
 109         if (probability(SLOW_PATH_PROBABILITY, length < 0)) {
 110             counters.checkAIOOBECounter.inc();
 111             DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
 112         }
 113         if (probability(SLOW_PATH_PROBABILITY, srcPos > ArrayLengthNode.arrayLength(src) - length)) {
 114             counters.checkAIOOBECounter.inc();
 115             DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
 116         }
 117         if (probability(SLOW_PATH_PROBABILITY, destPos > ArrayLengthNode.arrayLength(dest) - length)) {
 118             counters.checkAIOOBECounter.inc();
 119             DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
 120         }
 121         counters.checkSuccessCounter.inc();
 122     }
 123 
 124     @Snippet
 125     public static void arraycopyZeroLengthIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
 126         Object nonNullSrc = GraalDirectives.guardingNonNull(src);
 127         Object nonNullDest = GraalDirectives.guardingNonNull(dest);
 128         KlassPointer srcHub = loadHub(nonNullSrc);
 129         KlassPointer destHub = loadHub(nonNullDest);
 130         checkArrayType(srcHub);
 131         checkArrayType(destHub);
 132         checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
 133         counters.zeroLengthStaticCounter.inc();
 134     }
 135 
 136     @Snippet
 137     public static void arraycopyExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind, @ConstantParameter SnippetCounter counter,
 138                     @ConstantParameter SnippetCounter copiedCounter, @ConstantParameter Counters counters) {
 139         Object nonNullSrc = GraalDirectives.guardingNonNull(src);
 140         Object nonNullDest = GraalDirectives.guardingNonNull(dest);
 141         checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
 142         counter.inc();
 143         copiedCounter.add(length);
 144         ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind);
 145         if (length == 0) {
 146             counters.zeroLengthDynamicCounter.inc();
 147         } else {
 148             counters.nonZeroLengthDynamicCounter.inc();
 149             counters.nonZeroLengthDynamicCopiedCounter.add(length);
 150         }
 151     }
 152 
 153     /**
 154      * This intrinsic is useful for the case where we know something statically about one of the
 155      * inputs but not the other.
 156      */
 157     @Snippet
 158     public static void arraycopyPredictedExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind,
 159                     @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter, @ConstantParameter Counters counters) {
 160         Object nonNullSrc = GraalDirectives.guardingNonNull(src);
 161         Object nonNullDest = GraalDirectives.guardingNonNull(dest);
 162         KlassPointer srcHub = loadHub(nonNullSrc);
 163         KlassPointer destHub = loadHub(nonNullDest);
 164         if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) {
 165             DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
 166         }
 167         checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
 168         counter.inc();
 169         copiedCounter.add(length);
 170         ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind);
 171         if (length == 0) {
 172             counters.zeroLengthDynamicCounter.inc();
 173         } else {
 174             counters.nonZeroLengthDynamicCounter.inc();
 175             counters.nonZeroLengthDynamicCopiedCounter.add(length);
 176         }
 177     }
 178 
 179     @Snippet
 180     public static void arraycopyPredictedObjectWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, KlassPointer objectArrayKlass,
 181                     @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter, @ConstantParameter Counters counters) {
 182         if (length > 0) {
 183             KlassPointer srcHub = loadHub(PiNode.asNonNullObject(nonNullSrc));
 184             KlassPointer destHub = loadHub(PiNode.asNonNullObject(nonNullDest));
 185             if (probability(FAST_PATH_PROBABILITY, srcHub == destHub || destHub == objectArrayKlass)) {
 186                 counter.inc();
 187                 copiedCounter.add(length);
 188                 counters.predictedObjectArrayCopyFastPathCounter.inc();
 189                 counters.predictedObjectArrayCopyFastPathCopiedCounter.add(length);
 190                 ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length);
 191             } else {
 192                 counters.predictedObjectArrayCopySlowPathCounter.inc();
 193                 counters.predictedObjectArrayCopySlowPathCopiedCounter.add(length);
 194                 System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length);
 195             }
 196         }
 197     }
 198 
 199     /**
 200      * This is the basic template for the full arraycopy checks, including a check that the
 201      * underlying type is really an array type.
 202      */
 203     @Snippet
 204     public static void arraycopySlowPathIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, KlassPointer predictedKlass, @ConstantParameter JavaKind elementKind,
 205                     @ConstantParameter SnippetInfo slowPath, @ConstantParameter Object slowPathArgument, @ConstantParameter Counters counters) {
 206         Object nonNullSrc = GraalDirectives.guardingNonNull(src);
 207         Object nonNullDest = GraalDirectives.guardingNonNull(dest);
 208         KlassPointer srcHub = loadHub(nonNullSrc);
 209         KlassPointer destHub = loadHub(nonNullDest);
 210         checkArrayType(srcHub);
 211         checkArrayType(destHub);
 212         checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
 213         if (length == 0) {
 214             counters.zeroLengthDynamicCounter.inc();
 215         } else {
 216             counters.nonZeroLengthDynamicCounter.inc();
 217             counters.nonZeroLengthDynamicCopiedCounter.add(length);
 218         }
 219         ArrayCopySlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, predictedKlass, elementKind, slowPath, slowPathArgument);
 220     }
 221 
 222     /**
 223      * Snippet for unrolled arraycopy.
 224      */
 225     @Snippet
 226     public static void arraycopyUnrolledIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter int unrolledLength, @ConstantParameter JavaKind elementKind,
 227                     @ConstantParameter Counters counters) {
 228         Object nonNullSrc = GraalDirectives.guardingNonNull(src);
 229         Object nonNullDest = GraalDirectives.guardingNonNull(dest);
 230         checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
 231         if (length == 0) {
 232             counters.zeroLengthDynamicCounter.inc();
 233         } else {
 234             counters.nonZeroLengthDynamicCounter.inc();
 235             counters.nonZeroLengthDynamicCopiedCounter.add(length);
 236         }
 237         ArrayCopyUnrollNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, unrolledLength, elementKind);
 238     }
 239 
 240     @Snippet
 241     public static void checkcastArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, @ConstantParameter Counters counters) {
 242         if (length > 0) {
 243             KlassPointer destKlass = loadHub(nonNullDest);
 244             KlassPointer srcKlass = loadHub(nonNullSrc);
 245             if (probability(SLOW_PATH_PROBABILITY, srcKlass == destKlass)) {
 246                 // no storecheck required.
 247                 counters.objectCheckcastSameTypeCounter.inc();
 248                 counters.objectCheckcastSameTypeCopiedCounter.add(length);
 249                 ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length);
 250             } else {
 251                 KlassPointer destElemKlass = destKlass.readKlassPointer(arrayClassElementOffset(INJECTED_VMCONFIG), OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION);
 252                 Word superCheckOffset = WordFactory.signed(destElemKlass.readInt(superCheckOffsetOffset(INJECTED_VMCONFIG), KLASS_SUPER_CHECK_OFFSET_LOCATION));
 253                 counters.objectCheckcastCounter.inc();
 254                 counters.objectCheckcastCopiedCounter.add(length);
 255                 int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false);
 256                 if (copiedElements != 0) {
 257                     /*
 258                      * the checkcast stub doesn't throw the ArrayStoreException, but returns the
 259                      * number of copied elements (xor'd with -1).
 260                      */
 261                     copiedElements ^= -1;
 262                     System.arraycopy(nonNullSrc, srcPos + copiedElements, nonNullDest, destPos + copiedElements, length - copiedElements);
 263                 }
 264             }
 265         }
 266     }
 267 
 268     @Snippet
 269     public static void arraycopyGeneric(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
 270         Object nonNullSrc = GraalDirectives.guardingNonNull(src);
 271         Object nonNullDest = GraalDirectives.guardingNonNull(dest);
 272         KlassPointer srcHub = loadHub(nonNullSrc);
 273         KlassPointer destHub = loadHub(nonNullDest);
 274         if (probability(FAST_PATH_PROBABILITY, srcHub.equal(destHub)) && probability(FAST_PATH_PROBABILITY, nonNullSrc != nonNullDest)) {
 275             int layoutHelper = checkArrayType(srcHub);
 276             final boolean isObjectArray = ((layoutHelper & layoutHelperElementTypePrimitiveInPlace(INJECTED_VMCONFIG)) == 0);
 277             checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
 278             if (probability(FAST_PATH_PROBABILITY, isObjectArray)) {
 279                 counters.genericObjectExactCallCounter.inc();
 280                 counters.genericObjectExactCallCopiedCounter.add(length);
 281                 ArrayCopyCallNode.disjointArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, JavaKind.Object);
 282             } else {
 283                 counters.genericPrimitiveCallCounter.inc();
 284                 counters.genericPrimitiveCallCopiedCounter.add(length);
 285                 UnsafeArrayCopyNode.arraycopyPrimitive(nonNullSrc, srcPos, nonNullDest, destPos, length, layoutHelper);
 286             }
 287         } else {
 288             counters.systemArraycopyCounter.inc();
 289             counters.systemArraycopyCopiedCounter.add(length);
 290             System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length);
 291         }
 292     }
 293 
 294     @Fold
 295     static LocationIdentity getArrayLocation(JavaKind kind) {
 296         return NamedLocationIdentity.getArrayLocation(kind);
 297     }
 298 
 299     @Snippet
 300     public static void arraycopyUnrolledWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, @ConstantParameter int length, @ConstantParameter JavaKind elementKind) {
 301         final int scale = arrayIndexScale(elementKind);
 302         int arrayBaseOffset = arrayBaseOffset(elementKind);
 303         LocationIdentity arrayLocation = getArrayLocation(elementKind);
 304         if (nonNullSrc == nonNullDest && srcPos < destPos) { // bad aliased case
 305             long start = (long) (length - 1) * scale;
 306             long i = start;
 307             ExplodeLoopNode.explodeLoop();
 308             for (int iteration = 0; iteration < length; iteration++) {
 309                 if (i >= 0) {
 310                     Object a = RawLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation);
 311                     RawStoreNode.storeObject(nonNullDest, arrayBaseOffset + i + (long) destPos * scale, a, elementKind, arrayLocation, false);
 312                     i -= scale;
 313                 }
 314             }
 315         } else {
 316             long end = (long) length * scale;
 317             long i = 0;
 318             ExplodeLoopNode.explodeLoop();
 319             for (int iteration = 0; iteration < length; iteration++) {
 320                 if (i < end) {
 321                     Object a = RawLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation);
 322                     RawStoreNode.storeObject(nonNullDest, arrayBaseOffset + i + (long) destPos * scale, a, elementKind, arrayLocation, false);
 323                     i += scale;
 324                 }
 325             }
 326         }
 327     }
 328 
 329     static class Counters {
 330         final SnippetCounter checkSuccessCounter;
 331         final SnippetCounter checkAIOOBECounter;
 332 
 333         final SnippetCounter objectCheckcastCounter;
 334         final SnippetCounter objectCheckcastSameTypeCounter;
 335         final SnippetCounter predictedObjectArrayCopySlowPathCounter;
 336         final SnippetCounter predictedObjectArrayCopyFastPathCounter;
 337 
 338         final SnippetCounter genericPrimitiveCallCounter;
 339         final SnippetCounter genericObjectExactCallCounter;
 340         final SnippetCounter systemArraycopyCounter;
 341 
 342         final SnippetCounter zeroLengthStaticCounter;
 343         final SnippetCounter zeroLengthDynamicCounter;
 344         final SnippetCounter nonZeroLengthDynamicCounter;
 345 
 346         final SnippetCounter nonZeroLengthDynamicCopiedCounter;
 347         final SnippetCounter genericPrimitiveCallCopiedCounter;
 348         final SnippetCounter genericObjectExactCallCopiedCounter;
 349         final SnippetCounter systemArraycopyCopiedCounter;
 350 
 351         final SnippetCounter objectCheckcastCopiedCounter;
 352         final SnippetCounter objectCheckcastSameTypeCopiedCounter;
 353         final SnippetCounter predictedObjectArrayCopySlowPathCopiedCounter;
 354         final SnippetCounter predictedObjectArrayCopyFastPathCopiedCounter;
 355 
 356         final EnumMap<JavaKind, SnippetCounter> arraycopyCallCounters = new EnumMap<>(JavaKind.class);
 357         final EnumMap<JavaKind, SnippetCounter> arraycopyCounters = new EnumMap<>(JavaKind.class);
 358 
 359         final EnumMap<JavaKind, SnippetCounter> arraycopyCallCopiedCounters = new EnumMap<>(JavaKind.class);
 360         final EnumMap<JavaKind, SnippetCounter> arraycopyCopiedCounters = new EnumMap<>(JavaKind.class);
 361 
 362         Counters(SnippetCounter.Group.Factory factory) {
 363             final Group checkCounters = factory.createSnippetCounterGroup("System.arraycopy checkInputs");
 364             final Group counters = factory.createSnippetCounterGroup("System.arraycopy");
 365             final Group copiedCounters = factory.createSnippetCounterGroup("System.arraycopy copied elements");
 366             final Group lengthCounters = factory.createSnippetCounterGroup("System.arraycopy 0-length checks");
 367 
 368             checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess");
 369             checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE");
 370 
 371             objectCheckcastCounter = new SnippetCounter(counters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays");
 372             objectCheckcastSameTypeCounter = new SnippetCounter(counters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays");
 373             predictedObjectArrayCopySlowPathCounter = new SnippetCounter(counters, "Object[]{slow-path}", "used System.arraycopy slow path for predicted Object[] arrays");
 374             predictedObjectArrayCopyFastPathCounter = new SnippetCounter(counters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays");
 375             genericPrimitiveCallCounter = new SnippetCounter(counters, "genericPrimitive", "generic arraycopy snippet for primitive arrays");
 376             genericObjectExactCallCounter = new SnippetCounter(counters, "genericObjectExact", "generic arraycopy snippet for special object arrays");
 377             systemArraycopyCounter = new SnippetCounter(counters, "genericObject", "call to System.arraycopy");
 378 
 379             zeroLengthStaticCounter = new SnippetCounter(lengthCounters, "0-lengthcopy static", "arraycopy where the length is statically 0");
 380             zeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "0-lengthcopy dynamically", "arraycopy where the length is dynamically 0");
 381             nonZeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero");
 382 
 383             nonZeroLengthDynamicCopiedCounter = new SnippetCounter(copiedCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero");
 384             genericPrimitiveCallCopiedCounter = new SnippetCounter(copiedCounters, "genericPrimitive", "generic arraycopy snippet for primitive arrays");
 385             genericObjectExactCallCopiedCounter = new SnippetCounter(copiedCounters, "genericObjectExact", "generic arraycopy snippet for special object arrays");
 386             systemArraycopyCopiedCounter = new SnippetCounter(copiedCounters, "genericObject", "call to System.arraycopy");
 387 
 388             objectCheckcastCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays");
 389             objectCheckcastSameTypeCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays");
 390             predictedObjectArrayCopySlowPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{slow-path}",
 391                             "used System.arraycopy slow path for predicted Object[] arrays");
 392             predictedObjectArrayCopyFastPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays");
 393             createArraycopyCounter(JavaKind.Byte, counters, copiedCounters);
 394             createArraycopyCounter(JavaKind.Boolean, counters, copiedCounters);
 395             createArraycopyCounter(JavaKind.Char, counters, copiedCounters);
 396             createArraycopyCounter(JavaKind.Short, counters, copiedCounters);
 397             createArraycopyCounter(JavaKind.Int, counters, copiedCounters);
 398             createArraycopyCounter(JavaKind.Long, counters, copiedCounters);
 399             createArraycopyCounter(JavaKind.Float, counters, copiedCounters);
 400             createArraycopyCounter(JavaKind.Double, counters, copiedCounters);
 401             createArraycopyCounter(JavaKind.Object, counters, copiedCounters);
 402         }
 403 
 404         void createArraycopyCounter(JavaKind kind, Group counters, Group copiedCounters) {
 405             arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays"));
 406             arraycopyCounters.put(kind, new SnippetCounter(counters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays"));
 407 
 408             arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays"));
 409             arraycopyCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays"));
 410         }
 411     }
 412 
 413     public static class Templates extends SnippetTemplate.AbstractTemplates {
 414 
 415         public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target) {
 416             super(options, factories, providers, providers.getSnippetReflection(), target);
 417             this.counters = new Counters(factory);
 418         }
 419 
 420         private ResolvedJavaMethod originalArraycopy() throws GraalError {
 421             if (originalArraycopy == null) {
 422                 Method method;
 423                 try {
 424                     method = System.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class);
 425                 } catch (NoSuchMethodException | SecurityException e) {
 426                     throw new GraalError(e);
 427                 }
 428                 originalArraycopy = providers.getMetaAccess().lookupJavaMethod(method);
 429             }
 430             return originalArraycopy;
 431         }
 432 
 433         private ResolvedJavaMethod originalArraycopy;
 434 
 435         private final SnippetInfo checkcastArraycopyWorkSnippet = snippet("checkcastArraycopyWork");
 436         private final SnippetInfo arraycopyGenericSnippet = snippet("arraycopyGeneric");
 437 
 438         private final SnippetInfo arraycopySlowPathIntrinsicSnippet = snippet("arraycopySlowPathIntrinsic");
 439         private final SnippetInfo arraycopyUnrolledIntrinsicSnippet = snippet("arraycopyUnrolledIntrinsic");
 440         private final SnippetInfo arraycopyExactIntrinsicSnippet = snippet("arraycopyExactIntrinsic");
 441         private final SnippetInfo arraycopyZeroLengthIntrinsicSnippet = snippet("arraycopyZeroLengthIntrinsic");
 442         private final SnippetInfo arraycopyPredictedExactIntrinsicSnippet = snippet("arraycopyPredictedExactIntrinsic");
 443         private final SnippetInfo arraycopyPredictedObjectWorkSnippet = snippet("arraycopyPredictedObjectWork");
 444 
 445         private final SnippetInfo arraycopyUnrolledWorkSnippet = snippet("arraycopyUnrolledWork");
 446 
 447         private final Counters counters;
 448 
 449         protected SnippetInfo snippet(String methodName) {
 450             SnippetInfo info = snippet(ArrayCopySnippets.class, methodName, LocationIdentity.any());
 451             info.setOriginalMethod(originalArraycopy());
 452             return info;
 453         }
 454 
 455         public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy) {
 456             return selectComponentKind(arraycopy, true);
 457         }
 458 
 459         public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy, boolean exact) {
 460             ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp());
 461             ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp());
 462 
 463             if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) {
 464                 if (!exact) {
 465                     JavaKind component = getComponentKind(srcType);
 466                     if (component != null) {
 467                         return component;
 468                     }
 469                     return getComponentKind(destType);
 470                 }
 471                 return null;
 472             }
 473             if (exact) {
 474                 if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) {
 475                     return null;
 476                 }
 477                 if (!arraycopy.isExact()) {
 478                     return null;
 479                 }
 480             }
 481             return srcType.getComponentType().getJavaKind();
 482         }
 483 
 484         private static JavaKind getComponentKind(ResolvedJavaType type) {
 485             if (type != null && type.isArray()) {
 486                 return type.getComponentType().getJavaKind();
 487             }
 488             return null;
 489         }
 490 
 491         private static boolean shouldUnroll(ValueNode length) {
 492             return length.isConstant() && length.asJavaConstant().asInt() <= 8 && length.asJavaConstant().asInt() != 0;
 493         }
 494 
 495         public void lower(ArrayCopyNode arraycopy, LoweringTool tool) {
 496             JavaKind componentKind = selectComponentKind(arraycopy);
 497             SnippetInfo snippetInfo = null;
 498             SnippetInfo slowPathSnippetInfo = null;
 499             Object slowPathArgument = null;
 500 
 501             if (arraycopy.getLength().isConstant() && arraycopy.getLength().asJavaConstant().asLong() == 0) {
 502                 snippetInfo = arraycopyZeroLengthIntrinsicSnippet;
 503             } else if (arraycopy.isExact()) {
 504                 snippetInfo = arraycopyExactIntrinsicSnippet;
 505                 if (shouldUnroll(arraycopy.getLength())) {
 506                     snippetInfo = arraycopyUnrolledIntrinsicSnippet;
 507                 }
 508             } else {
 509                 if (componentKind == JavaKind.Object) {
 510                     ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp());
 511                     ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp());
 512                     ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType();
 513                     ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType();
 514                     if (srcComponentType != null && destComponentType != null && !srcComponentType.isPrimitive() && !destComponentType.isPrimitive()) {
 515                         snippetInfo = arraycopySlowPathIntrinsicSnippet;
 516                         slowPathSnippetInfo = checkcastArraycopyWorkSnippet;
 517                         slowPathArgument = LocationIdentity.any();
 518                         /*
 519                          * Because this snippet has to use Sysytem.arraycopy as a slow path, we must
 520                          * pretend to kill any() so clear the componentKind.
 521                          */
 522                         componentKind = null;
 523                     }
 524                 }
 525                 if (componentKind == null && snippetInfo == null) {
 526                     JavaKind predictedKind = selectComponentKind(arraycopy, false);
 527                     if (predictedKind != null) {
 528                         /*
 529                          * At least one array is of a known type requiring no store checks, so
 530                          * assume the other is of the same type. Generally this is working around
 531                          * deficiencies in our propagation of type information.
 532                          */
 533                         componentKind = predictedKind;
 534                         if (predictedKind == JavaKind.Object) {
 535                             snippetInfo = arraycopySlowPathIntrinsicSnippet;
 536                             slowPathSnippetInfo = arraycopyPredictedObjectWorkSnippet;
 537                             slowPathArgument = predictedKind;
 538                             componentKind = null;
 539                         } else {
 540                             snippetInfo = arraycopyPredictedExactIntrinsicSnippet;
 541                         }
 542                     }
 543                 }
 544                 if (snippetInfo == null) {
 545                     snippetInfo = arraycopyGenericSnippet;
 546                 }
 547             }
 548             Arguments args = new Arguments(snippetInfo, arraycopy.graph().getGuardsStage(), tool.getLoweringStage());
 549             args.add("src", arraycopy.getSource());
 550             args.add("srcPos", arraycopy.getSourcePosition());
 551             args.add("dest", arraycopy.getDestination());
 552             args.add("destPos", arraycopy.getDestinationPosition());
 553             args.add("length", arraycopy.getLength());
 554             if (snippetInfo == arraycopyUnrolledIntrinsicSnippet) {
 555                 args.addConst("unrolledLength", arraycopy.getLength().asJavaConstant().asInt());
 556                 args.addConst("elementKind", componentKind != null ? componentKind : JavaKind.Illegal);
 557             } else if (snippetInfo == arraycopySlowPathIntrinsicSnippet) {
 558                 ValueNode predictedKlass = null;
 559                 if (slowPathArgument == arraycopyPredictedObjectWorkSnippet) {
 560                     HotSpotResolvedObjectType arrayClass = (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(Object[].class);
 561                     predictedKlass = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayClass.klass(), tool.getMetaAccess(), arraycopy.graph());
 562                 } else {
 563                     predictedKlass = ConstantNode.forConstant(KlassPointerStamp.klassAlwaysNull(), JavaConstant.NULL_POINTER, tool.getMetaAccess(), arraycopy.graph());
 564                 }
 565                 args.add("predictedKlass", predictedKlass);
 566                 args.addConst("elementKind", componentKind != null ? componentKind : JavaKind.Illegal);
 567                 args.addConst("slowPath", slowPathSnippetInfo);
 568                 assert slowPathArgument != null;
 569                 args.addConst("slowPathArgument", slowPathArgument);
 570             } else if (snippetInfo == arraycopyExactIntrinsicSnippet || snippetInfo == arraycopyPredictedExactIntrinsicSnippet) {
 571                 assert componentKind != null;
 572                 args.addConst("elementKind", componentKind);
 573                 args.addConst("counter", counters.arraycopyCallCounters.get(componentKind));
 574                 args.addConst("copiedCounter", counters.arraycopyCallCopiedCounters.get(componentKind));
 575             }
 576             args.addConst("counters", counters);
 577             instantiate(args, arraycopy);
 578         }
 579 
 580         public void lower(ArrayCopySlowPathNode arraycopy, LoweringTool tool) {
 581             StructuredGraph graph = arraycopy.graph();
 582             if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
 583                 // Can't be lowered yet
 584                 return;
 585             }
 586             SnippetInfo snippetInfo = arraycopy.getSnippet();
 587             Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage());
 588             args.add("nonNullSrc", arraycopy.getSource());
 589             args.add("srcPos", arraycopy.getSourcePosition());
 590             args.add("nonNullDest", arraycopy.getDestination());
 591             args.add("destPos", arraycopy.getDestinationPosition());
 592             if (snippetInfo == arraycopyUnrolledWorkSnippet) {
 593                 args.addConst("length", ((Integer) arraycopy.getArgument()).intValue());
 594                 args.addConst("elementKind", arraycopy.getElementKind());
 595             } else {
 596                 args.add("length", arraycopy.getLength());
 597             }
 598             if (snippetInfo == arraycopyPredictedObjectWorkSnippet) {
 599                 args.add("objectArrayKlass", arraycopy.getPredictedKlass());
 600                 args.addConst("counter", counters.arraycopyCallCounters.get(JavaKind.Object));
 601                 args.addConst("copiedCounter", counters.arraycopyCallCopiedCounters.get(JavaKind.Object));
 602                 args.addConst("counters", counters);
 603             }
 604             instantiate(args, arraycopy);
 605         }
 606 
 607         public void lower(ArrayCopyUnrollNode arraycopy, LoweringTool tool) {
 608             StructuredGraph graph = arraycopy.graph();
 609             if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
 610                 // Can't be lowered yet
 611                 return;
 612             }
 613             SnippetInfo snippetInfo = arraycopyUnrolledWorkSnippet;
 614             Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage());
 615             args.add("nonNullSrc", arraycopy.getSource());
 616             args.add("srcPos", arraycopy.getSourcePosition());
 617             args.add("nonNullDest", arraycopy.getDestination());
 618             args.add("destPos", arraycopy.getDestinationPosition());
 619             args.addConst("length", arraycopy.getUnrollLength());
 620             args.addConst("elementKind", arraycopy.getElementKind());
 621             template(graph.getDebug(), args).instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args);
 622         }
 623 
 624         /**
 625          * Instantiate the snippet template and fix up the FrameState of any Invokes of
 626          * System.arraycopy and propagate the captured bci in the ArrayCopySlowPathNode.
 627          *
 628          * @param args
 629          * @param arraycopy
 630          */
 631         private void instantiate(Arguments args, BasicArrayCopyNode arraycopy) {
 632             StructuredGraph graph = arraycopy.graph();
 633             SnippetTemplate template = template(graph.getDebug(), args);
 634             UnmodifiableEconomicMap<Node, Node> replacements = template.instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args, false);
 635             for (Node originalNode : replacements.getKeys()) {
 636                 if (originalNode instanceof Invoke) {
 637                     Invoke invoke = (Invoke) replacements.get(originalNode);
 638                     assert invoke.asNode().graph() == graph;
 639                     CallTargetNode call = invoke.callTarget();
 640 
 641                     if (!call.targetMethod().equals(originalArraycopy)) {
 642                         throw new GraalError("unexpected invoke %s in snippet", call.targetMethod());
 643                     }
 644                     // Here we need to fix the bci of the invoke
 645                     InvokeNode newInvoke = graph.add(new InvokeNode(invoke.callTarget(), arraycopy.getBci()));
 646                     if (arraycopy.stateDuring() != null) {
 647                         newInvoke.setStateDuring(arraycopy.stateDuring());
 648                     } else {
 649                         assert arraycopy.stateAfter() != null : arraycopy;
 650                         newInvoke.setStateAfter(arraycopy.stateAfter());
 651                     }
 652                     graph.replaceFixedWithFixed((InvokeNode) invoke.asNode(), newInvoke);
 653                 } else if (originalNode instanceof ArrayCopySlowPathNode) {
 654                     ArrayCopySlowPathNode slowPath = (ArrayCopySlowPathNode) replacements.get(originalNode);
 655                     assert arraycopy.stateAfter() != null : arraycopy;
 656                     assert slowPath.stateAfter() == arraycopy.stateAfter();
 657                     slowPath.setBci(arraycopy.getBci());
 658                 }
 659             }
 660             GraphUtil.killCFG(arraycopy);
 661         }
 662     }
 663 }