1 /*
   2  * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 
  25 package org.graalvm.compiler.replacements.arraycopy;
  26 
  27 import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE;
  28 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
  29 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.LIKELY_PROBABILITY;
  30 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY;
  31 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
  32 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
  33 
  34 import java.util.EnumMap;
  35 
  36 import jdk.internal.vm.compiler.collections.UnmodifiableEconomicMap;
  37 import org.graalvm.compiler.api.directives.GraalDirectives;
  38 import org.graalvm.compiler.api.replacements.Fold.InjectedParameter;
  39 import org.graalvm.compiler.api.replacements.Snippet;
  40 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
  41 import org.graalvm.compiler.api.replacements.SnippetReflectionProvider;
  42 import org.graalvm.compiler.debug.DebugHandlersFactory;
  43 import org.graalvm.compiler.debug.GraalError;
  44 import org.graalvm.compiler.graph.Node;
  45 import org.graalvm.compiler.nodes.CallTargetNode;
  46 import org.graalvm.compiler.nodes.DeoptimizeNode;
  47 import org.graalvm.compiler.nodes.InvokeNode;
  48 import org.graalvm.compiler.nodes.InvokeWithExceptionNode;
  49 import org.graalvm.compiler.nodes.NamedLocationIdentity;
  50 import org.graalvm.compiler.nodes.NodeView;
  51 import org.graalvm.compiler.nodes.PiNode;
  52 import org.graalvm.compiler.nodes.SnippetAnchorNode;
  53 import org.graalvm.compiler.nodes.StructuredGraph;
  54 import org.graalvm.compiler.nodes.extended.GuardedUnsafeLoadNode;
  55 import org.graalvm.compiler.nodes.extended.GuardingNode;
  56 import org.graalvm.compiler.nodes.extended.RawStoreNode;
  57 import org.graalvm.compiler.nodes.java.ArrayLengthNode;
  58 import org.graalvm.compiler.nodes.spi.LoweringTool;
  59 import org.graalvm.compiler.nodes.type.StampTool;
  60 import org.graalvm.compiler.nodes.util.GraphUtil;
  61 import org.graalvm.compiler.options.OptionValues;
  62 import org.graalvm.compiler.phases.util.Providers;
  63 import org.graalvm.compiler.replacements.ReplacementsUtil;
  64 import org.graalvm.compiler.replacements.SnippetCounter;
  65 import org.graalvm.compiler.replacements.SnippetCounter.Group;
  66 import org.graalvm.compiler.replacements.SnippetCounter.Group.Factory;
  67 import org.graalvm.compiler.replacements.SnippetIntegerHistogram;
  68 import org.graalvm.compiler.replacements.SnippetTemplate;
  69 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
  70 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
  71 import org.graalvm.compiler.replacements.Snippets;
  72 import org.graalvm.compiler.replacements.nodes.BasicArrayCopyNode;
  73 import org.graalvm.compiler.word.Word;
  74 import jdk.internal.vm.compiler.word.LocationIdentity;
  75 import jdk.internal.vm.compiler.word.Pointer;
  76 
  77 import jdk.vm.ci.code.TargetDescription;
  78 import jdk.vm.ci.meta.DeoptimizationAction;
  79 import jdk.vm.ci.meta.DeoptimizationReason;
  80 import jdk.vm.ci.meta.JavaKind;
  81 import jdk.vm.ci.meta.MetaAccessProvider;
  82 import jdk.vm.ci.meta.ResolvedJavaMethod;
  83 import jdk.vm.ci.meta.ResolvedJavaType;
  84 
  85 public abstract class ArrayCopySnippets implements Snippets {
  86 
  87     private enum ArrayCopyTypeCheck {
  88         UNDEFINED_ARRAY_TYPE_CHECK,
  89         // either we know that both objects are arrays and have the same type,
  90         // or we apply generic array copy snippet, which enforces type check
  91         NO_ARRAY_TYPE_CHECK,
  92         // can be used when we know that one of the objects is a primitive array
  93         HUB_BASED_ARRAY_TYPE_CHECK,
  94         // can be used when we know that one of the objects is an object array
  95         LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK
  96     }
  97 
  98     /** Marker value for the {@link InjectedParameter} injected parameter. */
  99     static final MetaAccessProvider INJECTED_META_ACCESS = null;
 100 
 101     public abstract Pointer loadHub(Object nonNullSrc);
 102 
 103     public abstract Pointer getDestElemClass(Pointer destKlass);
 104 
 105     public abstract Word getSuperCheckOffset(Pointer destElemKlass);
 106 
 107     public abstract int getReadLayoutHelper(Pointer srcHub);
 108 
 109     protected abstract int heapWordSize();
 110 
 111     @SuppressWarnings("unused")
 112     @Snippet
 113     public void arraycopyExactSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
 114                     @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind, @ConstantParameter LocationIdentity locationIdentity,
 115                     @ConstantParameter SnippetCounter elementKindCounter, @ConstantParameter SnippetCounter elementKindCopiedCounter, @ConstantParameter Counters counters) {
 116         Object nonNullSrc = GraalDirectives.guardingNonNull(src);
 117         Object nonNullDest = GraalDirectives.guardingNonNull(dest);
 118         checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
 119         checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
 120         incrementLengthCounter(length, counters);
 121 
 122         elementKindCounter.inc();
 123         elementKindCopiedCounter.add(length);
 124 
 125         ArrayCopyWithDelayedLoweringNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind);
 126     }
 127 
 128     @SuppressWarnings("unused")
 129     @Snippet
 130     public void arraycopyExactStubCallSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
 131                     @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind, @ConstantParameter LocationIdentity locationIdentity,
 132                     @ConstantParameter SnippetCounter elementKindCounter, @ConstantParameter SnippetCounter elementKindCopiedCounter, @ConstantParameter Counters counters) {
 133         Object nonNullSrc = GraalDirectives.guardingNonNull(src);
 134         Object nonNullDest = GraalDirectives.guardingNonNull(dest);
 135         checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
 136         checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
 137         incrementLengthCounter(length, counters);
 138 
 139         elementKindCounter.inc();
 140         elementKindCopiedCounter.add(length);
 141 
 142         ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind, locationIdentity, heapWordSize());
 143     }
 144 
 145     @Snippet
 146     public void arraycopyCheckcastSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, @ConstantParameter Counters counters,
 147                     @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind) {
 148         Object nonNullSrc = GraalDirectives.guardingNonNull(src);
 149         Object nonNullDest = GraalDirectives.guardingNonNull(dest);
 150         checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
 151         checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
 152         incrementLengthCounter(length, counters);
 153 
 154         ArrayCopyWithDelayedLoweringNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind);
 155     }
 156 
 157     @Snippet
 158     public void arraycopyGenericSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, @ConstantParameter Counters counters,
 159                     @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind) {
 160         Object nonNullSrc = GraalDirectives.guardingNonNull(src);
 161         Object nonNullDest = GraalDirectives.guardingNonNull(dest);
 162         checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
 163         checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
 164         incrementLengthCounter(length, counters);
 165 
 166         ArrayCopyWithDelayedLoweringNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind);
 167     }
 168 
 169     @Snippet
 170     public static void arraycopyNativeSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
 171         // all checks are done in the native method, so no need to emit additional checks here
 172         incrementLengthCounter(length, counters);
 173         counters.systemArraycopyCounter.inc();
 174         counters.systemArraycopyCopiedCounter.add(length);
 175 
 176         System.arraycopy(src, srcPos, dest, destPos, length);
 177     }
 178 
 179     @SuppressWarnings("unused")
 180     @Snippet(allowPartialIntrinsicArgumentMismatch = true)
 181     public void exactArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind, @ConstantParameter LocationIdentity arrayLocation,
 182                     @ConstantParameter Counters counters) {
 183         int scale = ReplacementsUtil.arrayIndexScale(INJECTED_META_ACCESS, elementKind);
 184         int arrayBaseOffset = ReplacementsUtil.getArrayBaseOffset(INJECTED_META_ACCESS, elementKind);
 185         long sourceOffset = arrayBaseOffset + (long) srcPos * scale;
 186         long destOffset = arrayBaseOffset + (long) destPos * scale;
 187 
 188         GuardingNode anchor = SnippetAnchorNode.anchor();
 189         if (probability(NOT_FREQUENT_PROBABILITY, src == dest && srcPos < destPos)) {
 190             // bad aliased case so we need to copy the array from back to front
 191             for (int position = length - 1; position >= 0; position--) {
 192                 Object value = GuardedUnsafeLoadNode.guardedLoad(src, sourceOffset + ((long) position) * scale, elementKind, arrayLocation, anchor);
 193                 RawStoreNode.storeObject(dest, destOffset + ((long) position) * scale, value, elementKind, arrayLocation, true);
 194             }
 195         } else {
 196             for (int position = 0; position < length; position++) {
 197                 Object value = GuardedUnsafeLoadNode.guardedLoad(src, sourceOffset + ((long) position) * scale, elementKind, arrayLocation, anchor);
 198                 RawStoreNode.storeObject(dest, destOffset + ((long) position) * scale, value, elementKind, arrayLocation, true);
 199             }
 200         }
 201     }
 202 
 203     @SuppressWarnings("unused")
 204     @Snippet(allowPartialIntrinsicArgumentMismatch = true)
 205     public void checkcastArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind,
 206                     @ConstantParameter LocationIdentity arrayLocation,
 207                     @ConstantParameter Counters counters) {
 208         if (probability(FREQUENT_PROBABILITY, length > 0)) {
 209             Object nonNullSrc = PiNode.asNonNullObject(src);
 210             Object nonNullDest = PiNode.asNonNullObject(dest);
 211             Pointer srcKlass = loadHub(nonNullSrc);
 212             Pointer destKlass = loadHub(nonNullDest);
 213             if (probability(LIKELY_PROBABILITY, srcKlass == destKlass)) {
 214                 // no storecheck required.
 215                 counters.objectCheckcastSameTypeCounter.inc();
 216                 counters.objectCheckcastSameTypeCopiedCounter.add(length);
 217                 ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length, heapWordSize());
 218             } else {
 219                 Pointer destElemKlass = getDestElemClass(destKlass);
 220                 Word superCheckOffset = getSuperCheckOffset(destElemKlass);
 221 
 222                 counters.objectCheckcastDifferentTypeCounter.inc();
 223                 counters.objectCheckcastDifferentTypeCopiedCounter.add(length);
 224 
 225                 int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false);
 226                 if (probability(SLOW_PATH_PROBABILITY, copiedElements != 0)) {
 227                     /*
 228                      * the stub doesn't throw the ArrayStoreException, but returns the number of
 229                      * copied elements (xor'd with -1).
 230                      */
 231                     copiedElements ^= -1;
 232                     System.arraycopy(nonNullSrc, srcPos + copiedElements, nonNullDest, destPos + copiedElements, length - copiedElements);
 233                 }
 234             }
 235         }
 236     }
 237 
 238     @SuppressWarnings("unused")
 239     @Snippet(allowPartialIntrinsicArgumentMismatch = true)
 240     public void genericArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind,
 241                     @ConstantParameter LocationIdentity arrayLocation,
 242                     @ConstantParameter Counters counters) {
 243         // The length > 0 check should not be placed here because generic array copy stub should
 244         // enforce type check. This is fine performance-wise because this snippet is rarely used.
 245         counters.genericArraycopyDifferentTypeCounter.inc();
 246         counters.genericArraycopyDifferentTypeCopiedCounter.add(length);
 247         int copiedElements = GenericArrayCopyCallNode.genericArraycopy(src, srcPos, dest, destPos, length);
 248         if (probability(SLOW_PATH_PROBABILITY, copiedElements != 0)) {
 249             /*
 250              * the stub doesn't throw the ArrayStoreException, but returns the number of copied
 251              * elements (xor'd with -1).
 252              */
 253             copiedElements ^= -1;
 254             System.arraycopy(src, srcPos + copiedElements, dest, destPos + copiedElements, length - copiedElements);
 255         }
 256     }
 257 
 258     private static void incrementLengthCounter(int length, Counters counters) {
 259         if (!IS_BUILDING_NATIVE_IMAGE) {
 260             counters.lengthHistogram.inc(length);
 261         }
 262     }
 263 
 264     private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length, Counters counters) {
 265         if (probability(SLOW_PATH_PROBABILITY, srcPos < 0) ||
 266                         probability(SLOW_PATH_PROBABILITY, destPos < 0) ||
 267                         probability(SLOW_PATH_PROBABILITY, length < 0) ||
 268                         probability(SLOW_PATH_PROBABILITY, srcPos > ArrayLengthNode.arrayLength(src) - length) ||
 269                         probability(SLOW_PATH_PROBABILITY, destPos > ArrayLengthNode.arrayLength(dest) - length)) {
 270             counters.checkAIOOBECounter.inc();
 271             DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
 272         }
 273         counters.checkSuccessCounter.inc();
 274     }
 275 
 276     private void checkArrayTypes(Object nonNullSrc, Object nonNullDest, ArrayCopyTypeCheck arrayTypeCheck) {
 277         if (arrayTypeCheck == ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK) {
 278             // nothing to do
 279         } else if (arrayTypeCheck == ArrayCopyTypeCheck.HUB_BASED_ARRAY_TYPE_CHECK) {
 280             Pointer srcHub = loadHub(nonNullSrc);
 281             Pointer destHub = loadHub(nonNullDest);
 282             if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) {
 283                 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
 284             }
 285         } else if (arrayTypeCheck == ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK) {
 286             Pointer srcHub = loadHub(nonNullSrc);
 287             Pointer destHub = loadHub(nonNullDest);
 288             if (probability(SLOW_PATH_PROBABILITY, getReadLayoutHelper(srcHub) != getReadLayoutHelper(destHub))) {
 289                 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
 290             }
 291         } else {
 292             ReplacementsUtil.staticAssert(false, "unknown array type check ", arrayTypeCheck);
 293         }
 294     }
 295 
 296     static class Counters {
 297         final SnippetCounter checkSuccessCounter;
 298         final SnippetCounter checkAIOOBECounter;
 299 
 300         final SnippetCounter zeroLengthStaticCounter;
 301         final SnippetIntegerHistogram lengthHistogram;
 302 
 303         final SnippetCounter systemArraycopyCounter;
 304         final SnippetCounter systemArraycopyCopiedCounter;
 305 
 306         final SnippetCounter genericArraycopyDifferentTypeCopiedCounter;
 307         final SnippetCounter genericArraycopyDifferentTypeCounter;
 308 
 309         final SnippetCounter objectCheckcastSameTypeCopiedCounter;
 310         final SnippetCounter objectCheckcastSameTypeCounter;
 311         final SnippetCounter objectCheckcastDifferentTypeCopiedCounter;
 312         final SnippetCounter objectCheckcastDifferentTypeCounter;
 313 
 314         final EnumMap<JavaKind, SnippetCounter> arraycopyCallCounters = new EnumMap<>(JavaKind.class);
 315         final EnumMap<JavaKind, SnippetCounter> arraycopyCallCopiedCounters = new EnumMap<>(JavaKind.class);
 316 
 317         Counters(SnippetCounter.Group.Factory factory) {
 318             final Group checkCounters = factory.createSnippetCounterGroup("System.arraycopy checkInputs");
 319             final Group callCounters = factory.createSnippetCounterGroup("System.arraycopy calls");
 320             final Group copiedElementsCounters = factory.createSnippetCounterGroup("System.arraycopy copied elements");
 321             final Group lengthCounters = factory.createSnippetCounterGroup("System.arraycopy with 0-length");
 322 
 323             checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess");
 324             checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE");
 325 
 326             zeroLengthStaticCounter = new SnippetCounter(lengthCounters, "0-length copy static", "calls where the length is statically 0");
 327             lengthHistogram = new SnippetIntegerHistogram(lengthCounters, 2, "length", "length");
 328 
 329             systemArraycopyCounter = new SnippetCounter(callCounters, "native System.arraycopy", "JNI-based System.arraycopy call");
 330             systemArraycopyCopiedCounter = new SnippetCounter(copiedElementsCounters, "native System.arraycopy", "JNI-based System.arraycopy call");
 331 
 332             genericArraycopyDifferentTypeCounter = new SnippetCounter(callCounters, "generic[] stub", "generic arraycopy stub");
 333             genericArraycopyDifferentTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "generic[] stub", "generic arraycopy stub");
 334 
 335             objectCheckcastSameTypeCounter = new SnippetCounter(callCounters, "checkcast object[] (same-type)", "checkcast object[] stub but src.klass == dest.klass Object[] arrays");
 336             objectCheckcastSameTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "checkcast object[] (same-type)", "checkcast object[] stub but src.klass == dest.klass Object[] arrays");
 337             objectCheckcastDifferentTypeCounter = new SnippetCounter(callCounters, "checkcast object[] (store-check)", "checkcast object[] stub with store check");
 338             objectCheckcastDifferentTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "checkcast object[] (store-check)", "checkcast object[] stub with store check");
 339 
 340             createArraycopyCounter(JavaKind.Byte, callCounters, copiedElementsCounters);
 341             createArraycopyCounter(JavaKind.Boolean, callCounters, copiedElementsCounters);
 342             createArraycopyCounter(JavaKind.Char, callCounters, copiedElementsCounters);
 343             createArraycopyCounter(JavaKind.Short, callCounters, copiedElementsCounters);
 344             createArraycopyCounter(JavaKind.Int, callCounters, copiedElementsCounters);
 345             createArraycopyCounter(JavaKind.Long, callCounters, copiedElementsCounters);
 346             createArraycopyCounter(JavaKind.Float, callCounters, copiedElementsCounters);
 347             createArraycopyCounter(JavaKind.Double, callCounters, copiedElementsCounters);
 348             createArraycopyCounter(JavaKind.Object, callCounters, copiedElementsCounters);
 349         }
 350 
 351         void createArraycopyCounter(JavaKind kind, Group counters, Group copiedCounters) {
 352             arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[] stub", "arraycopy call for " + kind + "[] arrays"));
 353             arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[] stub", "arraycopy call for " + kind + "[] arrays"));
 354         }
 355     }
 356 
 357     public static class Templates extends SnippetTemplate.AbstractTemplates {
 358         private final SnippetInfo arraycopyGenericSnippet;
 359         private final SnippetInfo arraycopyExactSnippet;
 360         private final SnippetInfo arraycopyExactStubCallSnippet;
 361         private final SnippetInfo arraycopyCheckcastSnippet;
 362         private final SnippetInfo arraycopyNativeSnippet;
 363         private final SnippetInfo checkcastArraycopyWithSlowPathWork;
 364         private final SnippetInfo genericArraycopyWithSlowPathWork;
 365         private final SnippetInfo exactArraycopyWithSlowPathWork;
 366 
 367         private ResolvedJavaMethod originalArraycopy;
 368         private final Counters counters;
 369         private boolean expandArraycopyLoop;
 370 
 371         public Templates(ArrayCopySnippets receiver, OptionValues options, Iterable<DebugHandlersFactory> factories, Factory factory, Providers providers,
 372                         SnippetReflectionProvider snippetReflection, TargetDescription target) {
 373             super(options, factories, providers, snippetReflection, target);
 374             this.counters = new Counters(factory);
 375 
 376             arraycopyGenericSnippet = snippet(receiver, "arraycopyGenericSnippet");
 377             arraycopyExactSnippet = snippet(receiver, "arraycopyExactSnippet");
 378             arraycopyExactStubCallSnippet = snippet(receiver, "arraycopyExactStubCallSnippet");
 379             arraycopyCheckcastSnippet = snippet(receiver, "arraycopyCheckcastSnippet");
 380             arraycopyNativeSnippet = snippet(null, "arraycopyNativeSnippet");
 381             checkcastArraycopyWithSlowPathWork = snippet(receiver, "checkcastArraycopyWithSlowPathWork");
 382             genericArraycopyWithSlowPathWork = snippet(receiver, "genericArraycopyWithSlowPathWork");
 383             exactArraycopyWithSlowPathWork = snippet(receiver, "exactArraycopyWithSlowPathWork");
 384         }
 385 
 386         protected SnippetInfo snippet(ArrayCopySnippets receiver, String methodName) {
 387             SnippetInfo info = snippet(ArrayCopySnippets.class, methodName, originalArraycopy(), receiver, LocationIdentity.any());
 388             return info;
 389         }
 390 
 391         public void lower(ArrayCopyNode arraycopy, LoweringTool tool) {
 392             JavaKind elementKind = selectComponentKind(arraycopy);
 393             SnippetInfo snippetInfo;
 394             ArrayCopyTypeCheck arrayTypeCheck;
 395 
 396             ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp(NodeView.DEFAULT));
 397             ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp(NodeView.DEFAULT));
 398             if (!canBeArray(srcType) || !canBeArray(destType)) {
 399                 // at least one of the objects is definitely not an array - use the native call
 400                 // right away as the copying will fail anyways
 401                 snippetInfo = arraycopyNativeSnippet;
 402                 arrayTypeCheck = ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK;
 403             } else {
 404                 ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType();
 405                 ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType();
 406 
 407                 if (arraycopy.isExact()) {
 408                     // there is a sufficient type match - we don't need any additional type checks
 409                     snippetInfo = arraycopyExactStubCallSnippet;
 410                     arrayTypeCheck = ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK;
 411                 } else if (srcComponentType == null && destComponentType == null) {
 412                     // we don't know anything about the types - use the generic copying
 413                     snippetInfo = arraycopyGenericSnippet;
 414                     // no need for additional type check to avoid duplicated work
 415                     arrayTypeCheck = ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK;
 416                 } else if (srcComponentType != null && destComponentType != null) {
 417                     if (!srcComponentType.isPrimitive() && !destComponentType.isPrimitive()) {
 418                         // it depends on the array content if the copy succeeds - we need
 419                         // a type check for every store
 420                         snippetInfo = arraycopyCheckcastSnippet;
 421                         arrayTypeCheck = ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK;
 422                     } else {
 423                         // one object is an object array, the other one is a primitive array.
 424                         // this copy will always fail - use the native call right away
 425                         assert !srcComponentType.equals(destComponentType) : "must be handled by arraycopy.isExact()";
 426                         snippetInfo = arraycopyNativeSnippet;
 427                         arrayTypeCheck = ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK;
 428                     }
 429                 } else {
 430                     ResolvedJavaType nonNullComponentType = srcComponentType != null ? srcComponentType : destComponentType;
 431                     if (nonNullComponentType.isPrimitive()) {
 432                         // one involved object is a primitive array - it is sufficient to directly
 433                         // compare the hub.
 434                         snippetInfo = arraycopyExactStubCallSnippet;
 435                         arrayTypeCheck = ArrayCopyTypeCheck.HUB_BASED_ARRAY_TYPE_CHECK;
 436                         elementKind = nonNullComponentType.getJavaKind();
 437                     } else {
 438                         // one involved object is an object array - the other array's element type
 439                         // may be primitive or object, hence we compare the layout helper.
 440                         snippetInfo = arraycopyCheckcastSnippet;
 441                         arrayTypeCheck = ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK;
 442                     }
 443                 }
 444             }
 445 
 446             if (this.expandArraycopyLoop && snippetInfo == arraycopyExactStubCallSnippet) {
 447                 snippetInfo = arraycopyExactSnippet;
 448             }
 449 
 450             // create the snippet
 451             Arguments args = new Arguments(snippetInfo, arraycopy.graph().getGuardsStage(), tool.getLoweringStage());
 452             args.add("src", arraycopy.getSource());
 453             args.add("srcPos", arraycopy.getSourcePosition());
 454             args.add("dest", arraycopy.getDestination());
 455             args.add("destPos", arraycopy.getDestinationPosition());
 456             args.add("length", arraycopy.getLength());
 457             if (snippetInfo != arraycopyNativeSnippet) {
 458                 assert arrayTypeCheck != ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK;
 459                 args.addConst("arrayTypeCheck", arrayTypeCheck);
 460             }
 461             Object locationIdentity = arraycopy.killsAnyLocation() ? LocationIdentity.any() : NamedLocationIdentity.getArrayLocation(elementKind);
 462             if (snippetInfo == arraycopyExactStubCallSnippet || snippetInfo == arraycopyExactSnippet) {
 463                 assert elementKind != null;
 464                 args.addConst("workSnippet", exactArraycopyWithSlowPathWork);
 465                 args.addConst("elementKind", elementKind);
 466                 args.addConst("locationIdentity", locationIdentity);
 467                 args.addConst("elementKindCounter", counters.arraycopyCallCounters.get(elementKind));
 468                 args.addConst("elementKindCopiedCounter", counters.arraycopyCallCopiedCounters.get(elementKind));
 469             }
 470             args.addConst("counters", counters);
 471             if (snippetInfo == arraycopyCheckcastSnippet) {
 472                 args.addConst("workSnippet", checkcastArraycopyWithSlowPathWork);
 473                 args.addConst("elementKind", JavaKind.Illegal);
 474             }
 475             if (snippetInfo == arraycopyGenericSnippet) {
 476                 args.addConst("workSnippet", genericArraycopyWithSlowPathWork);
 477                 args.addConst("elementKind", JavaKind.Illegal);
 478             }
 479 
 480             instantiate(args, arraycopy);
 481         }
 482 
 483         public void lower(ArrayCopyWithDelayedLoweringNode arraycopy, LoweringTool tool) {
 484             StructuredGraph graph = arraycopy.graph();
 485 
 486             if (arraycopy.getSnippet() == exactArraycopyWithSlowPathWork && this.expandArraycopyLoop) {
 487                 if (!graph.getGuardsStage().areDeoptsFixed()) {
 488                     // Don't lower until floating guards are fixed.
 489                     return;
 490                 }
 491             } else {
 492                 if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
 493                     // Don't lower until frame states are assigned to deoptimization points.
 494                     return;
 495                 }
 496             }
 497 
 498             SnippetInfo snippetInfo = arraycopy.getSnippet();
 499             Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage());
 500             args.add("src", arraycopy.getSource());
 501             args.add("srcPos", arraycopy.getSourcePosition());
 502             args.add("dest", arraycopy.getDestination());
 503             args.add("destPos", arraycopy.getDestinationPosition());
 504             args.add("length", arraycopy.getLength());
 505 
 506             JavaKind elementKind = arraycopy.getElementKind();
 507             args.addConst("elementKind", (elementKind == null) ? JavaKind.Illegal : elementKind);
 508 
 509             Object locationIdentity = (elementKind == null) ? LocationIdentity.any() : NamedLocationIdentity.getArrayLocation(arraycopy.getElementKind());
 510             args.addConst("arrayLocation", locationIdentity);
 511             args.addConst("counters", counters);
 512             instantiate(args, arraycopy);
 513         }
 514 
 515         private static boolean canBeArray(ResolvedJavaType type) {
 516             return type == null || type.isJavaLangObject() || type.isArray();
 517         }
 518 
 519         public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy) {
 520             ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp(NodeView.DEFAULT));
 521             ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp(NodeView.DEFAULT));
 522 
 523             if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) {
 524                 return null;
 525             }
 526             if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) {
 527                 return null;
 528             }
 529             if (!arraycopy.isExact()) {
 530                 return null;
 531             }
 532             return srcType.getComponentType().getJavaKind();
 533         }
 534 
 535         /**
 536          * Instantiate the snippet template and fix up the FrameState of any Invokes of
 537          * System.arraycopy and propagate the captured bci in the ArrayCopySlowPathNode.
 538          *
 539          * @param args
 540          * @param arraycopy
 541          */
 542         private void instantiate(Arguments args, BasicArrayCopyNode arraycopy) {
 543             StructuredGraph graph = arraycopy.graph();
 544             SnippetTemplate template = template(arraycopy, args);
 545             UnmodifiableEconomicMap<Node, Node> replacements = template.instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args, false);
 546             for (Node originalNode : replacements.getKeys()) {
 547                 if (originalNode instanceof InvokeNode) {
 548                     InvokeNode invoke = (InvokeNode) replacements.get(originalNode);
 549                     assert invoke.asNode().graph() == graph;
 550                     CallTargetNode call = invoke.callTarget();
 551 
 552                     if (!call.targetMethod().equals(originalArraycopy)) {
 553                         throw new GraalError("unexpected invoke %s in snippet", call.targetMethod());
 554                     }
 555                     // Here we need to fix the bci of the invoke
 556                     invoke.replaceBci(arraycopy.getBci());
 557                     invoke.setStateDuring(null);
 558                     invoke.setStateAfter(null);
 559                     if (arraycopy.stateDuring() != null) {
 560                         invoke.setStateDuring(arraycopy.stateDuring());
 561                     } else {
 562                         assert arraycopy.stateAfter() != null : arraycopy;
 563                         invoke.setStateAfter(arraycopy.stateAfter());
 564                     }
 565                 } else if (originalNode instanceof InvokeWithExceptionNode) {
 566                     throw new GraalError("unexpected invoke with exception %s in snippet", originalNode);
 567                 } else if (originalNode instanceof ArrayCopyWithDelayedLoweringNode) {
 568                     ArrayCopyWithDelayedLoweringNode slowPath = (ArrayCopyWithDelayedLoweringNode) replacements.get(originalNode);
 569                     assert arraycopy.stateAfter() != null : arraycopy;
 570                     assert slowPath.stateAfter() == arraycopy.stateAfter();
 571                     slowPath.setBci(arraycopy.getBci());
 572                 }
 573             }
 574             GraphUtil.killCFG(arraycopy);
 575         }
 576 
 577         private ResolvedJavaMethod originalArraycopy() throws GraalError {
 578             if (originalArraycopy == null) {
 579                 try {
 580                     originalArraycopy = findMethod(providers.getMetaAccess(), System.class, "arraycopy");
 581                 } catch (SecurityException e) {
 582                     throw new GraalError(e);
 583                 }
 584             }
 585             return originalArraycopy;
 586         }
 587 
 588         public void setExpandArraycopyLoop(boolean b) {
 589             this.expandArraycopyLoop = b;
 590         }
 591     }
 592 }