< prev index next >

src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/arraycopy/ArrayCopySnippets.java

Print this page




   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 
  25 package org.graalvm.compiler.replacements.arraycopy;
  26 

  27 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
  28 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.LIKELY_PROBABILITY;
  29 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY;
  30 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
  31 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
  32 
  33 import java.util.EnumMap;
  34 
  35 import jdk.internal.vm.compiler.collections.UnmodifiableEconomicMap;
  36 import org.graalvm.compiler.api.directives.GraalDirectives;
  37 import org.graalvm.compiler.api.replacements.Fold;
  38 import org.graalvm.compiler.api.replacements.Fold.InjectedParameter;
  39 import org.graalvm.compiler.api.replacements.Snippet;
  40 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
  41 import org.graalvm.compiler.api.replacements.SnippetReflectionProvider;
  42 import org.graalvm.compiler.debug.DebugHandlersFactory;
  43 import org.graalvm.compiler.debug.GraalError;
  44 import org.graalvm.compiler.graph.Node;
  45 import org.graalvm.compiler.nodes.CallTargetNode;
  46 import org.graalvm.compiler.nodes.DeoptimizeNode;
  47 import org.graalvm.compiler.nodes.InvokeNode;
  48 import org.graalvm.compiler.nodes.InvokeWithExceptionNode;
  49 import org.graalvm.compiler.nodes.NamedLocationIdentity;
  50 import org.graalvm.compiler.nodes.NodeView;
  51 import org.graalvm.compiler.nodes.PiNode;
  52 import org.graalvm.compiler.nodes.StructuredGraph;
  53 import org.graalvm.compiler.nodes.ValueNode;
  54 import org.graalvm.compiler.nodes.extended.RawLoadNode;
  55 import org.graalvm.compiler.nodes.extended.RawStoreNode;
  56 import org.graalvm.compiler.nodes.java.ArrayLengthNode;
  57 import org.graalvm.compiler.nodes.spi.LoweringTool;


 103     public abstract Pointer getDestElemClass(Pointer destKlass);
 104 
 105     public abstract Word getSuperCheckOffset(Pointer destElemKlass);
 106 
 107     public abstract int getReadLayoutHelper(Pointer srcHub);
 108 
 109     protected abstract int heapWordSize();
 110 
 111     @Snippet
 112     public void arraycopyZeroLengthSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
 113                     @ConstantParameter Counters counters) {
 114         Object nonNullSrc = GraalDirectives.guardingNonNull(src);
 115         Object nonNullDest = GraalDirectives.guardingNonNull(dest);
 116         this.checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
 117         checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
 118         counters.zeroLengthStaticCounter.inc();
 119     }
 120 
 121     @Snippet
 122     public void arraycopyExactSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
 123                     @ConstantParameter JavaKind elementKind, @ConstantParameter SnippetCounter elementKindCounter, @ConstantParameter SnippetCounter elementKindCopiedCounter,
 124                     @ConstantParameter Counters counters) {
 125         Object nonNullSrc = GraalDirectives.guardingNonNull(src);
 126         Object nonNullDest = GraalDirectives.guardingNonNull(dest);
 127         checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
 128         checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
 129         incrementLengthCounter(length, counters);
 130 
 131         elementKindCounter.inc();
 132         elementKindCopiedCounter.add(length);
 133         ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind, heapWordSize());
 134     }
 135 
 136     @Snippet
 137     public void arraycopyUnrolledSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
 138                     @ConstantParameter JavaKind elementKind, @ConstantParameter int unrolledLength, @ConstantParameter Counters counters) {
 139         Object nonNullSrc = GraalDirectives.guardingNonNull(src);
 140         Object nonNullDest = GraalDirectives.guardingNonNull(dest);
 141         checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
 142         checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
 143         incrementLengthCounter(length, counters);
 144 
 145         unrolledArraycopyWork(nonNullSrc, srcPos, nonNullDest, destPos, unrolledLength, elementKind);
 146     }
 147 
 148     @Snippet
 149     public void arraycopyCheckcastSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
 150                     @ConstantParameter Counters counters, @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind) {
 151         Object nonNullSrc = GraalDirectives.guardingNonNull(src);
 152         Object nonNullDest = GraalDirectives.guardingNonNull(dest);
 153         checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
 154         checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
 155         incrementLengthCounter(length, counters);
 156 
 157         ArrayCopyWithSlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind);
 158     }
 159 
 160     @Snippet
 161     public void arraycopyGenericSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, @ConstantParameter Counters counters,
 162                     @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind) {
 163         Object nonNullSrc = GraalDirectives.guardingNonNull(src);
 164         Object nonNullDest = GraalDirectives.guardingNonNull(dest);
 165         checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
 166         checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
 167         incrementLengthCounter(length, counters);
 168 
 169         ArrayCopyWithSlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind);
 170     }
 171 
 172     @Snippet
 173     public static void arraycopyNativeSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
 174         // all checks are done in the native method, so no need to emit additional checks here
 175         incrementLengthCounter(length, counters);
 176         counters.systemArraycopyCounter.inc();
 177         counters.systemArraycopyCopiedCounter.add(length);
 178 
 179         System.arraycopy(src, srcPos, dest, destPos, length);
 180     }
 181 
 182     @Fold
 183     static LocationIdentity getArrayLocation(JavaKind kind) {
 184         return NamedLocationIdentity.getArrayLocation(kind);
 185     }
 186 
 187     private static void unrolledArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, JavaKind elementKind) {
 188         int scale = ReplacementsUtil.arrayIndexScale(INJECTED_META_ACCESS, elementKind);
 189         int arrayBaseOffset = ReplacementsUtil.getArrayBaseOffset(INJECTED_META_ACCESS, elementKind);
 190         LocationIdentity arrayLocation = getArrayLocation(elementKind);
 191 
 192         long sourceOffset = arrayBaseOffset + (long) srcPos * scale;
 193         long destOffset = arrayBaseOffset + (long) destPos * scale;
 194         long position = 0;
 195         long delta = scale;
 196         if (probability(NOT_FREQUENT_PROBABILITY, nonNullSrc == nonNullDest && srcPos < destPos)) {
 197             // bad aliased case so we need to copy the array from back to front
 198             position = (long) (length - 1) * scale;
 199             delta = -delta;
 200         }
 201 
 202         // the length was already checked before - we can emit unconditional instructions
 203         ExplodeLoopNode.explodeLoop();
 204         for (int iteration = 0; iteration < length; iteration++) {
 205             Object value = RawLoadNode.load(nonNullSrc, sourceOffset + position, elementKind, arrayLocation);
 206             RawStoreNode.storeObject(nonNullDest, destOffset + position, value, elementKind, arrayLocation, false);
 207             position += delta;
 208         }
 209     }
 210 


 241     }
 242 
 243     @Snippet(allowPartialIntrinsicArgumentMismatch = true)
 244     public void genericArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
 245         // The length > 0 check should not be placed here because generic array copy stub should
 246         // enforce type check. This is fine performance-wise because this snippet is rarely used.
 247         counters.genericArraycopyDifferentTypeCounter.inc();
 248         counters.genericArraycopyDifferentTypeCopiedCounter.add(length);
 249         int copiedElements = GenericArrayCopyCallNode.genericArraycopy(src, srcPos, dest, destPos, length);
 250         if (probability(SLOW_PATH_PROBABILITY, copiedElements != 0)) {
 251             /*
 252              * the stub doesn't throw the ArrayStoreException, but returns the number of copied
 253              * elements (xor'd with -1).
 254              */
 255             copiedElements ^= -1;
 256             System.arraycopy(src, srcPos + copiedElements, dest, destPos + copiedElements, length - copiedElements);
 257         }
 258     }
 259 
 260     private static void incrementLengthCounter(int length, Counters counters) {
 261         counters.lengthHistogram.inc(length);


 262     }
 263 
 264     private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length, Counters counters) {
 265         if (probability(SLOW_PATH_PROBABILITY, srcPos < 0) ||
 266                         probability(SLOW_PATH_PROBABILITY, destPos < 0) ||
 267                         probability(SLOW_PATH_PROBABILITY, length < 0) ||
 268                         probability(SLOW_PATH_PROBABILITY, srcPos > ArrayLengthNode.arrayLength(src) - length) ||
 269                         probability(SLOW_PATH_PROBABILITY, destPos > ArrayLengthNode.arrayLength(dest) - length)) {
 270             counters.checkAIOOBECounter.inc();
 271             DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
 272         }
 273         counters.checkSuccessCounter.inc();
 274     }
 275 
 276     private void checkArrayTypes(Object nonNullSrc, Object nonNullDest, ArrayCopyTypeCheck arrayTypeCheck) {
 277         if (arrayTypeCheck == ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK) {
 278             // nothing to do
 279         } else if (arrayTypeCheck == ArrayCopyTypeCheck.HUB_BASED_ARRAY_TYPE_CHECK) {
 280             Pointer srcHub = loadHub(nonNullSrc);
 281             Pointer destHub = loadHub(nonNullDest);


 449                 // exception - once we pass the preliminary element type checks that we are not
 450                 // mixing arrays of different basic types, ArrayStoreException is only thrown when
 451                 // an *astore would have thrown it. Therefore, copying null between object arrays
 452                 // with conflicting types will also succeed (we do not optimize for such case here).
 453                 snippetInfo = arraycopyZeroLengthSnippet;
 454             } else if (snippetInfo == arraycopyExactSnippet && shouldUnroll(arraycopy.getLength())) {
 455                 snippetInfo = arraycopyUnrolledSnippet;
 456             }
 457 
 458             // create the snippet
 459             Arguments args = new Arguments(snippetInfo, arraycopy.graph().getGuardsStage(), tool.getLoweringStage());
 460             args.add("src", arraycopy.getSource());
 461             args.add("srcPos", arraycopy.getSourcePosition());
 462             args.add("dest", arraycopy.getDestination());
 463             args.add("destPos", arraycopy.getDestinationPosition());
 464             args.add("length", arraycopy.getLength());
 465             if (snippetInfo != arraycopyNativeSnippet) {
 466                 assert arrayTypeCheck != ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK;
 467                 args.addConst("arrayTypeCheck", arrayTypeCheck);
 468             }

 469             if (snippetInfo == arraycopyUnrolledSnippet) {
 470                 args.addConst("elementKind", elementKind != null ? elementKind : JavaKind.Illegal);

 471                 args.addConst("unrolledLength", arraycopy.getLength().asJavaConstant().asInt());
 472             }
 473             if (snippetInfo == arraycopyExactSnippet) {
 474                 assert elementKind != null;
 475                 args.addConst("elementKind", elementKind);

 476                 args.addConst("elementKindCounter", counters.arraycopyCallCounters.get(elementKind));
 477                 args.addConst("elementKindCopiedCounter", counters.arraycopyCallCopiedCounters.get(elementKind));
 478             }
 479             args.addConst("counters", counters);
 480             if (snippetInfo == arraycopyCheckcastSnippet) {
 481                 args.addConst("workSnippet", checkcastArraycopyWithSlowPathWork);
 482                 args.addConst("elementKind", JavaKind.Illegal);
 483             }
 484             if (snippetInfo == arraycopyGenericSnippet) {
 485                 args.addConst("workSnippet", genericArraycopyWithSlowPathWork);
 486                 args.addConst("elementKind", JavaKind.Illegal);
 487             }
 488 
 489             instantiate(args, arraycopy);
 490         }
 491 
 492         public void lower(ArrayCopyWithSlowPathNode arraycopy, LoweringTool tool) {
 493             StructuredGraph graph = arraycopy.graph();
 494             if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
 495                 // if an arraycopy contains a slow path, we can't lower it right away




   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 
  25 package org.graalvm.compiler.replacements.arraycopy;
  26 
  27 import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE;
  28 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
  29 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.LIKELY_PROBABILITY;
  30 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY;
  31 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
  32 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
  33 
  34 import java.util.EnumMap;
  35 
  36 import jdk.internal.vm.compiler.collections.UnmodifiableEconomicMap;
  37 import org.graalvm.compiler.api.directives.GraalDirectives;

  38 import org.graalvm.compiler.api.replacements.Fold.InjectedParameter;
  39 import org.graalvm.compiler.api.replacements.Snippet;
  40 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
  41 import org.graalvm.compiler.api.replacements.SnippetReflectionProvider;
  42 import org.graalvm.compiler.debug.DebugHandlersFactory;
  43 import org.graalvm.compiler.debug.GraalError;
  44 import org.graalvm.compiler.graph.Node;
  45 import org.graalvm.compiler.nodes.CallTargetNode;
  46 import org.graalvm.compiler.nodes.DeoptimizeNode;
  47 import org.graalvm.compiler.nodes.InvokeNode;
  48 import org.graalvm.compiler.nodes.InvokeWithExceptionNode;
  49 import org.graalvm.compiler.nodes.NamedLocationIdentity;
  50 import org.graalvm.compiler.nodes.NodeView;
  51 import org.graalvm.compiler.nodes.PiNode;
  52 import org.graalvm.compiler.nodes.StructuredGraph;
  53 import org.graalvm.compiler.nodes.ValueNode;
  54 import org.graalvm.compiler.nodes.extended.RawLoadNode;
  55 import org.graalvm.compiler.nodes.extended.RawStoreNode;
  56 import org.graalvm.compiler.nodes.java.ArrayLengthNode;
  57 import org.graalvm.compiler.nodes.spi.LoweringTool;


 103     public abstract Pointer getDestElemClass(Pointer destKlass);
 104 
 105     public abstract Word getSuperCheckOffset(Pointer destElemKlass);
 106 
 107     public abstract int getReadLayoutHelper(Pointer srcHub);
 108 
 109     protected abstract int heapWordSize();
 110 
 111     @Snippet
 112     public void arraycopyZeroLengthSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
 113                     @ConstantParameter Counters counters) {
 114         Object nonNullSrc = GraalDirectives.guardingNonNull(src);
 115         Object nonNullDest = GraalDirectives.guardingNonNull(dest);
 116         this.checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
 117         checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
 118         counters.zeroLengthStaticCounter.inc();
 119     }
 120 
 121     @Snippet
 122     public void arraycopyExactSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
 123                     @ConstantParameter JavaKind elementKind, @ConstantParameter LocationIdentity locationIdentity,
 124                     @ConstantParameter SnippetCounter elementKindCounter, @ConstantParameter SnippetCounter elementKindCopiedCounter, @ConstantParameter Counters counters) {
 125         Object nonNullSrc = GraalDirectives.guardingNonNull(src);
 126         Object nonNullDest = GraalDirectives.guardingNonNull(dest);
 127         checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
 128         checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
 129         incrementLengthCounter(length, counters);
 130 
 131         elementKindCounter.inc();
 132         elementKindCopiedCounter.add(length);
 133         ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind, locationIdentity, heapWordSize());
 134     }
 135 
 136     @Snippet
 137     public void arraycopyUnrolledSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
 138                     @ConstantParameter JavaKind elementKind, @ConstantParameter LocationIdentity locationIdentity, @ConstantParameter int unrolledLength, @ConstantParameter Counters counters) {
 139         Object nonNullSrc = GraalDirectives.guardingNonNull(src);
 140         Object nonNullDest = GraalDirectives.guardingNonNull(dest);
 141         checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
 142         checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
 143         incrementLengthCounter(length, counters);
 144 
 145         unrolledArraycopyWork(nonNullSrc, srcPos, nonNullDest, destPos, unrolledLength, elementKind, locationIdentity);
 146     }
 147 
 148     @Snippet
 149     public void arraycopyCheckcastSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
 150                     @ConstantParameter Counters counters, @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind) {
 151         Object nonNullSrc = GraalDirectives.guardingNonNull(src);
 152         Object nonNullDest = GraalDirectives.guardingNonNull(dest);
 153         checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
 154         checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
 155         incrementLengthCounter(length, counters);
 156 
 157         ArrayCopyWithSlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind);
 158     }
 159 
 160     @Snippet
 161     public void arraycopyGenericSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, @ConstantParameter Counters counters,
 162                     @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind) {
 163         Object nonNullSrc = GraalDirectives.guardingNonNull(src);
 164         Object nonNullDest = GraalDirectives.guardingNonNull(dest);
 165         checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
 166         checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
 167         incrementLengthCounter(length, counters);
 168 
 169         ArrayCopyWithSlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind);
 170     }
 171 
 172     @Snippet
 173     public static void arraycopyNativeSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
 174         // all checks are done in the native method, so no need to emit additional checks here
 175         incrementLengthCounter(length, counters);
 176         counters.systemArraycopyCounter.inc();
 177         counters.systemArraycopyCopiedCounter.add(length);
 178 
 179         System.arraycopy(src, srcPos, dest, destPos, length);
 180     }
 181 
 182     private static void unrolledArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, JavaKind elementKind, LocationIdentity arrayLocation) {





 183         int scale = ReplacementsUtil.arrayIndexScale(INJECTED_META_ACCESS, elementKind);
 184         int arrayBaseOffset = ReplacementsUtil.getArrayBaseOffset(INJECTED_META_ACCESS, elementKind);

 185 
 186         long sourceOffset = arrayBaseOffset + (long) srcPos * scale;
 187         long destOffset = arrayBaseOffset + (long) destPos * scale;
 188         long position = 0;
 189         long delta = scale;
 190         if (probability(NOT_FREQUENT_PROBABILITY, nonNullSrc == nonNullDest && srcPos < destPos)) {
 191             // bad aliased case so we need to copy the array from back to front
 192             position = (long) (length - 1) * scale;
 193             delta = -delta;
 194         }
 195 
 196         // the length was already checked before - we can emit unconditional instructions
 197         ExplodeLoopNode.explodeLoop();
 198         for (int iteration = 0; iteration < length; iteration++) {
 199             Object value = RawLoadNode.load(nonNullSrc, sourceOffset + position, elementKind, arrayLocation);
 200             RawStoreNode.storeObject(nonNullDest, destOffset + position, value, elementKind, arrayLocation, false);
 201             position += delta;
 202         }
 203     }
 204 


 235     }
 236 
 237     @Snippet(allowPartialIntrinsicArgumentMismatch = true)
 238     public void genericArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
 239         // The length > 0 check should not be placed here because generic array copy stub should
 240         // enforce type check. This is fine performance-wise because this snippet is rarely used.
 241         counters.genericArraycopyDifferentTypeCounter.inc();
 242         counters.genericArraycopyDifferentTypeCopiedCounter.add(length);
 243         int copiedElements = GenericArrayCopyCallNode.genericArraycopy(src, srcPos, dest, destPos, length);
 244         if (probability(SLOW_PATH_PROBABILITY, copiedElements != 0)) {
 245             /*
 246              * the stub doesn't throw the ArrayStoreException, but returns the number of copied
 247              * elements (xor'd with -1).
 248              */
 249             copiedElements ^= -1;
 250             System.arraycopy(src, srcPos + copiedElements, dest, destPos + copiedElements, length - copiedElements);
 251         }
 252     }
 253 
 254     private static void incrementLengthCounter(int length, Counters counters) {
 255         if (!IS_BUILDING_NATIVE_IMAGE) {
 256             counters.lengthHistogram.inc(length);
 257         }
 258     }
 259 
 260     private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length, Counters counters) {
 261         if (probability(SLOW_PATH_PROBABILITY, srcPos < 0) ||
 262                         probability(SLOW_PATH_PROBABILITY, destPos < 0) ||
 263                         probability(SLOW_PATH_PROBABILITY, length < 0) ||
 264                         probability(SLOW_PATH_PROBABILITY, srcPos > ArrayLengthNode.arrayLength(src) - length) ||
 265                         probability(SLOW_PATH_PROBABILITY, destPos > ArrayLengthNode.arrayLength(dest) - length)) {
 266             counters.checkAIOOBECounter.inc();
 267             DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
 268         }
 269         counters.checkSuccessCounter.inc();
 270     }
 271 
 272     private void checkArrayTypes(Object nonNullSrc, Object nonNullDest, ArrayCopyTypeCheck arrayTypeCheck) {
 273         if (arrayTypeCheck == ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK) {
 274             // nothing to do
 275         } else if (arrayTypeCheck == ArrayCopyTypeCheck.HUB_BASED_ARRAY_TYPE_CHECK) {
 276             Pointer srcHub = loadHub(nonNullSrc);
 277             Pointer destHub = loadHub(nonNullDest);


 445                 // exception - once we pass the preliminary element type checks that we are not
 446                 // mixing arrays of different basic types, ArrayStoreException is only thrown when
 447                 // an *astore would have thrown it. Therefore, copying null between object arrays
 448                 // with conflicting types will also succeed (we do not optimize for such case here).
 449                 snippetInfo = arraycopyZeroLengthSnippet;
 450             } else if (snippetInfo == arraycopyExactSnippet && shouldUnroll(arraycopy.getLength())) {
 451                 snippetInfo = arraycopyUnrolledSnippet;
 452             }
 453 
 454             // create the snippet
 455             Arguments args = new Arguments(snippetInfo, arraycopy.graph().getGuardsStage(), tool.getLoweringStage());
 456             args.add("src", arraycopy.getSource());
 457             args.add("srcPos", arraycopy.getSourcePosition());
 458             args.add("dest", arraycopy.getDestination());
 459             args.add("destPos", arraycopy.getDestinationPosition());
 460             args.add("length", arraycopy.getLength());
 461             if (snippetInfo != arraycopyNativeSnippet) {
 462                 assert arrayTypeCheck != ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK;
 463                 args.addConst("arrayTypeCheck", arrayTypeCheck);
 464             }
 465             Object locationIdentity = arraycopy.killsAnyLocation() ? LocationIdentity.any() : NamedLocationIdentity.getArrayLocation(elementKind);
 466             if (snippetInfo == arraycopyUnrolledSnippet) {
 467                 args.addConst("elementKind", elementKind != null ? elementKind : JavaKind.Illegal);
 468                 args.addConst("locationIdentity", locationIdentity);
 469                 args.addConst("unrolledLength", arraycopy.getLength().asJavaConstant().asInt());
 470             }
 471             if (snippetInfo == arraycopyExactSnippet) {
 472                 assert elementKind != null;
 473                 args.addConst("elementKind", elementKind);
 474                 args.addConst("locationIdentity", locationIdentity);
 475                 args.addConst("elementKindCounter", counters.arraycopyCallCounters.get(elementKind));
 476                 args.addConst("elementKindCopiedCounter", counters.arraycopyCallCopiedCounters.get(elementKind));
 477             }
 478             args.addConst("counters", counters);
 479             if (snippetInfo == arraycopyCheckcastSnippet) {
 480                 args.addConst("workSnippet", checkcastArraycopyWithSlowPathWork);
 481                 args.addConst("elementKind", JavaKind.Illegal);
 482             }
 483             if (snippetInfo == arraycopyGenericSnippet) {
 484                 args.addConst("workSnippet", genericArraycopyWithSlowPathWork);
 485                 args.addConst("elementKind", JavaKind.Illegal);
 486             }
 487 
 488             instantiate(args, arraycopy);
 489         }
 490 
 491         public void lower(ArrayCopyWithSlowPathNode arraycopy, LoweringTool tool) {
 492             StructuredGraph graph = arraycopy.graph();
 493             if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
 494                 // if an arraycopy contains a slow path, we can't lower it right away


< prev index next >