1 /*
2 * Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24
25 package org.graalvm.compiler.replacements.arraycopy;
26
27 import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE;
28 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
29 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.LIKELY_PROBABILITY;
30 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY;
31 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
32 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
33
34 import java.util.EnumMap;
35
36 import jdk.internal.vm.compiler.collections.UnmodifiableEconomicMap;
37 import org.graalvm.compiler.api.directives.GraalDirectives;
38 import org.graalvm.compiler.api.replacements.Fold.InjectedParameter;
39 import org.graalvm.compiler.api.replacements.Snippet;
40 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
41 import org.graalvm.compiler.api.replacements.SnippetReflectionProvider;
42 import org.graalvm.compiler.debug.DebugHandlersFactory;
43 import org.graalvm.compiler.debug.GraalError;
44 import org.graalvm.compiler.graph.Node;
45 import org.graalvm.compiler.nodes.CallTargetNode;
46 import org.graalvm.compiler.nodes.DeoptimizeNode;
47 import org.graalvm.compiler.nodes.InvokeNode;
48 import org.graalvm.compiler.nodes.InvokeWithExceptionNode;
49 import org.graalvm.compiler.nodes.NamedLocationIdentity;
50 import org.graalvm.compiler.nodes.NodeView;
51 import org.graalvm.compiler.nodes.PiNode;
52 import org.graalvm.compiler.nodes.StructuredGraph;
53 import org.graalvm.compiler.nodes.ValueNode;
54 import org.graalvm.compiler.nodes.extended.RawLoadNode;
55 import org.graalvm.compiler.nodes.extended.RawStoreNode;
56 import org.graalvm.compiler.nodes.java.ArrayLengthNode;
57 import org.graalvm.compiler.nodes.spi.LoweringTool;
58 import org.graalvm.compiler.nodes.type.StampTool;
59 import org.graalvm.compiler.nodes.util.GraphUtil;
60 import org.graalvm.compiler.options.OptionValues;
61 import org.graalvm.compiler.phases.util.Providers;
62 import org.graalvm.compiler.replacements.ReplacementsUtil;
63 import org.graalvm.compiler.replacements.SnippetCounter;
64 import org.graalvm.compiler.replacements.SnippetCounter.Group;
65 import org.graalvm.compiler.replacements.SnippetCounter.Group.Factory;
66 import org.graalvm.compiler.replacements.SnippetIntegerHistogram;
67 import org.graalvm.compiler.replacements.SnippetTemplate;
68 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
69 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
70 import org.graalvm.compiler.replacements.Snippets;
71 import org.graalvm.compiler.replacements.nodes.BasicArrayCopyNode;
72 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
73 import org.graalvm.compiler.word.Word;
74 import jdk.internal.vm.compiler.word.LocationIdentity;
75 import jdk.internal.vm.compiler.word.Pointer;
76
77 import jdk.vm.ci.code.TargetDescription;
78 import jdk.vm.ci.meta.DeoptimizationAction;
79 import jdk.vm.ci.meta.DeoptimizationReason;
80 import jdk.vm.ci.meta.JavaKind;
81 import jdk.vm.ci.meta.MetaAccessProvider;
82 import jdk.vm.ci.meta.ResolvedJavaMethod;
83 import jdk.vm.ci.meta.ResolvedJavaType;
84
85 public abstract class ArrayCopySnippets implements Snippets {
86
87 private enum ArrayCopyTypeCheck {
88 UNDEFINED_ARRAY_TYPE_CHECK,
89 // either we know that both objects are arrays and have the same type,
90 // or we apply generic array copy snippet, which enforces type check
91 NO_ARRAY_TYPE_CHECK,
92 // can be used when we know that one of the objects is a primitive array
93 HUB_BASED_ARRAY_TYPE_CHECK,
94 // can be used when we know that one of the objects is an object array
95 LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK
96 }
97
98 /** Marker value for the {@link InjectedParameter} injected parameter. */
99 static final MetaAccessProvider INJECTED_META_ACCESS = null;
100
101 public abstract Pointer loadHub(Object nonNullSrc);
102
103 public abstract Pointer getDestElemClass(Pointer destKlass);
104
105 public abstract Word getSuperCheckOffset(Pointer destElemKlass);
106
107 public abstract int getReadLayoutHelper(Pointer srcHub);
108
109 protected abstract int heapWordSize();
110
111 @Snippet
112 public void arraycopyZeroLengthSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
113 @ConstantParameter Counters counters) {
114 Object nonNullSrc = GraalDirectives.guardingNonNull(src);
115 Object nonNullDest = GraalDirectives.guardingNonNull(dest);
116 this.checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
117 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
118 counters.zeroLengthStaticCounter.inc();
119 }
120
121 @Snippet
122 public void arraycopyExactSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
123 @ConstantParameter JavaKind elementKind, @ConstantParameter LocationIdentity locationIdentity,
124 @ConstantParameter SnippetCounter elementKindCounter, @ConstantParameter SnippetCounter elementKindCopiedCounter, @ConstantParameter Counters counters) {
125 Object nonNullSrc = GraalDirectives.guardingNonNull(src);
126 Object nonNullDest = GraalDirectives.guardingNonNull(dest);
127 checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
128 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
129 incrementLengthCounter(length, counters);
130
131 elementKindCounter.inc();
132 elementKindCopiedCounter.add(length);
133 ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind, locationIdentity, heapWordSize());
134 }
135
136 @Snippet
137 public void arraycopyUnrolledSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
138 @ConstantParameter JavaKind elementKind, @ConstantParameter LocationIdentity locationIdentity, @ConstantParameter int unrolledLength, @ConstantParameter Counters counters) {
139 Object nonNullSrc = GraalDirectives.guardingNonNull(src);
140 Object nonNullDest = GraalDirectives.guardingNonNull(dest);
141 checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
142 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
143 incrementLengthCounter(length, counters);
144
145 unrolledArraycopyWork(nonNullSrc, srcPos, nonNullDest, destPos, unrolledLength, elementKind, locationIdentity);
146 }
147
148 @Snippet
149 public void arraycopyCheckcastSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
150 @ConstantParameter Counters counters, @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind) {
151 Object nonNullSrc = GraalDirectives.guardingNonNull(src);
152 Object nonNullDest = GraalDirectives.guardingNonNull(dest);
153 checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
154 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
155 incrementLengthCounter(length, counters);
156
157 ArrayCopyWithSlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind);
158 }
159
160 @Snippet
161 public void arraycopyGenericSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, @ConstantParameter Counters counters,
162 @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind) {
163 Object nonNullSrc = GraalDirectives.guardingNonNull(src);
164 Object nonNullDest = GraalDirectives.guardingNonNull(dest);
165 checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
166 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
167 incrementLengthCounter(length, counters);
168
169 ArrayCopyWithSlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind);
170 }
171
172 @Snippet
173 public static void arraycopyNativeSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
174 // all checks are done in the native method, so no need to emit additional checks here
175 incrementLengthCounter(length, counters);
176 counters.systemArraycopyCounter.inc();
177 counters.systemArraycopyCopiedCounter.add(length);
178
179 System.arraycopy(src, srcPos, dest, destPos, length);
180 }
181
182 private static void unrolledArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, JavaKind elementKind, LocationIdentity arrayLocation) {
183 int scale = ReplacementsUtil.arrayIndexScale(INJECTED_META_ACCESS, elementKind);
184 int arrayBaseOffset = ReplacementsUtil.getArrayBaseOffset(INJECTED_META_ACCESS, elementKind);
185
186 long sourceOffset = arrayBaseOffset + (long) srcPos * scale;
187 long destOffset = arrayBaseOffset + (long) destPos * scale;
188 long position = 0;
189 long delta = scale;
190 if (probability(NOT_FREQUENT_PROBABILITY, nonNullSrc == nonNullDest && srcPos < destPos)) {
191 // bad aliased case so we need to copy the array from back to front
192 position = (long) (length - 1) * scale;
193 delta = -delta;
194 }
195
196 // the length was already checked before - we can emit unconditional instructions
197 ExplodeLoopNode.explodeLoop();
198 for (int iteration = 0; iteration < length; iteration++) {
199 Object value = RawLoadNode.load(nonNullSrc, sourceOffset + position, elementKind, arrayLocation);
200 RawStoreNode.storeObject(nonNullDest, destOffset + position, value, elementKind, arrayLocation, false);
201 position += delta;
202 }
203 }
204
205 @Snippet(allowPartialIntrinsicArgumentMismatch = true)
206 public void checkcastArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
207 if (probability(FREQUENT_PROBABILITY, length > 0)) {
208 Object nonNullSrc = PiNode.asNonNullObject(src);
209 Object nonNullDest = PiNode.asNonNullObject(dest);
210 Pointer srcKlass = loadHub(nonNullSrc);
211 Pointer destKlass = loadHub(nonNullDest);
212 if (probability(LIKELY_PROBABILITY, srcKlass == destKlass)) {
213 // no storecheck required.
214 counters.objectCheckcastSameTypeCounter.inc();
215 counters.objectCheckcastSameTypeCopiedCounter.add(length);
216 ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length, heapWordSize());
217 } else {
218 Pointer destElemKlass = getDestElemClass(destKlass);
219 Word superCheckOffset = getSuperCheckOffset(destElemKlass);
220
221 counters.objectCheckcastDifferentTypeCounter.inc();
222 counters.objectCheckcastDifferentTypeCopiedCounter.add(length);
223
224 int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false);
225 if (probability(SLOW_PATH_PROBABILITY, copiedElements != 0)) {
226 /*
227 * the stub doesn't throw the ArrayStoreException, but returns the number of
228 * copied elements (xor'd with -1).
229 */
230 copiedElements ^= -1;
231 System.arraycopy(nonNullSrc, srcPos + copiedElements, nonNullDest, destPos + copiedElements, length - copiedElements);
232 }
233 }
234 }
235 }
236
237 @Snippet(allowPartialIntrinsicArgumentMismatch = true)
238 public void genericArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
239 // The length > 0 check should not be placed here because generic array copy stub should
240 // enforce type check. This is fine performance-wise because this snippet is rarely used.
241 counters.genericArraycopyDifferentTypeCounter.inc();
242 counters.genericArraycopyDifferentTypeCopiedCounter.add(length);
243 int copiedElements = GenericArrayCopyCallNode.genericArraycopy(src, srcPos, dest, destPos, length);
244 if (probability(SLOW_PATH_PROBABILITY, copiedElements != 0)) {
245 /*
246 * the stub doesn't throw the ArrayStoreException, but returns the number of copied
247 * elements (xor'd with -1).
248 */
249 copiedElements ^= -1;
250 System.arraycopy(src, srcPos + copiedElements, dest, destPos + copiedElements, length - copiedElements);
251 }
252 }
253
254 private static void incrementLengthCounter(int length, Counters counters) {
255 if (!IS_BUILDING_NATIVE_IMAGE) {
256 counters.lengthHistogram.inc(length);
257 }
258 }
259
260 private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length, Counters counters) {
261 if (probability(SLOW_PATH_PROBABILITY, srcPos < 0) ||
262 probability(SLOW_PATH_PROBABILITY, destPos < 0) ||
263 probability(SLOW_PATH_PROBABILITY, length < 0) ||
264 probability(SLOW_PATH_PROBABILITY, srcPos > ArrayLengthNode.arrayLength(src) - length) ||
265 probability(SLOW_PATH_PROBABILITY, destPos > ArrayLengthNode.arrayLength(dest) - length)) {
266 counters.checkAIOOBECounter.inc();
267 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
268 }
269 counters.checkSuccessCounter.inc();
270 }
271
272 private void checkArrayTypes(Object nonNullSrc, Object nonNullDest, ArrayCopyTypeCheck arrayTypeCheck) {
273 if (arrayTypeCheck == ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK) {
274 // nothing to do
275 } else if (arrayTypeCheck == ArrayCopyTypeCheck.HUB_BASED_ARRAY_TYPE_CHECK) {
276 Pointer srcHub = loadHub(nonNullSrc);
277 Pointer destHub = loadHub(nonNullDest);
278 if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) {
279 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
280 }
281 } else if (arrayTypeCheck == ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK) {
282 Pointer srcHub = loadHub(nonNullSrc);
283 Pointer destHub = loadHub(nonNullDest);
284 if (probability(SLOW_PATH_PROBABILITY, getReadLayoutHelper(srcHub) != getReadLayoutHelper(destHub))) {
285 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
286 }
287 } else {
288 ReplacementsUtil.staticAssert(false, "unknown array type check ", arrayTypeCheck);
289 }
290 }
291
292 static class Counters {
293 final SnippetCounter checkSuccessCounter;
294 final SnippetCounter checkAIOOBECounter;
295
296 final SnippetCounter zeroLengthStaticCounter;
297 final SnippetIntegerHistogram lengthHistogram;
298
299 final SnippetCounter systemArraycopyCounter;
300 final SnippetCounter systemArraycopyCopiedCounter;
301
302 final SnippetCounter genericArraycopyDifferentTypeCopiedCounter;
303 final SnippetCounter genericArraycopyDifferentTypeCounter;
304
305 final SnippetCounter objectCheckcastSameTypeCopiedCounter;
306 final SnippetCounter objectCheckcastSameTypeCounter;
307 final SnippetCounter objectCheckcastDifferentTypeCopiedCounter;
308 final SnippetCounter objectCheckcastDifferentTypeCounter;
309
310 final EnumMap<JavaKind, SnippetCounter> arraycopyCallCounters = new EnumMap<>(JavaKind.class);
311 final EnumMap<JavaKind, SnippetCounter> arraycopyCallCopiedCounters = new EnumMap<>(JavaKind.class);
312
313 Counters(SnippetCounter.Group.Factory factory) {
314 final Group checkCounters = factory.createSnippetCounterGroup("System.arraycopy checkInputs");
315 final Group callCounters = factory.createSnippetCounterGroup("System.arraycopy calls");
316 final Group copiedElementsCounters = factory.createSnippetCounterGroup("System.arraycopy copied elements");
317 final Group lengthCounters = factory.createSnippetCounterGroup("System.arraycopy with 0-length");
318
319 checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess");
320 checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE");
321
322 zeroLengthStaticCounter = new SnippetCounter(lengthCounters, "0-length copy static", "calls where the length is statically 0");
323 lengthHistogram = new SnippetIntegerHistogram(lengthCounters, 2, "length", "length");
324
325 systemArraycopyCounter = new SnippetCounter(callCounters, "native System.arraycopy", "JNI-based System.arraycopy call");
326 systemArraycopyCopiedCounter = new SnippetCounter(copiedElementsCounters, "native System.arraycopy", "JNI-based System.arraycopy call");
327
328 genericArraycopyDifferentTypeCounter = new SnippetCounter(callCounters, "generic[] stub", "generic arraycopy stub");
329 genericArraycopyDifferentTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "generic[] stub", "generic arraycopy stub");
330
331 objectCheckcastSameTypeCounter = new SnippetCounter(callCounters, "checkcast object[] (same-type)", "checkcast object[] stub but src.klass == dest.klass Object[] arrays");
332 objectCheckcastSameTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "checkcast object[] (same-type)", "checkcast object[] stub but src.klass == dest.klass Object[] arrays");
333 objectCheckcastDifferentTypeCounter = new SnippetCounter(callCounters, "checkcast object[] (store-check)", "checkcast object[] stub with store check");
334 objectCheckcastDifferentTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "checkcast object[] (store-check)", "checkcast object[] stub with store check");
335
336 createArraycopyCounter(JavaKind.Byte, callCounters, copiedElementsCounters);
337 createArraycopyCounter(JavaKind.Boolean, callCounters, copiedElementsCounters);
338 createArraycopyCounter(JavaKind.Char, callCounters, copiedElementsCounters);
339 createArraycopyCounter(JavaKind.Short, callCounters, copiedElementsCounters);
340 createArraycopyCounter(JavaKind.Int, callCounters, copiedElementsCounters);
341 createArraycopyCounter(JavaKind.Long, callCounters, copiedElementsCounters);
342 createArraycopyCounter(JavaKind.Float, callCounters, copiedElementsCounters);
343 createArraycopyCounter(JavaKind.Double, callCounters, copiedElementsCounters);
344 createArraycopyCounter(JavaKind.Object, callCounters, copiedElementsCounters);
345 }
346
347 void createArraycopyCounter(JavaKind kind, Group counters, Group copiedCounters) {
348 arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[] stub", "arraycopy call for " + kind + "[] arrays"));
349 arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[] stub", "arraycopy call for " + kind + "[] arrays"));
350 }
351 }
352
353 public static class Templates extends SnippetTemplate.AbstractTemplates {
354 private final SnippetInfo arraycopyGenericSnippet;
355 private final SnippetInfo arraycopyUnrolledSnippet;
356 private final SnippetInfo arraycopyExactSnippet;
357 private final SnippetInfo arraycopyZeroLengthSnippet;
358 private final SnippetInfo arraycopyCheckcastSnippet;
359 private final SnippetInfo arraycopyNativeSnippet;
360 private final SnippetInfo checkcastArraycopyWithSlowPathWork;
361 private final SnippetInfo genericArraycopyWithSlowPathWork;
362
363 private ResolvedJavaMethod originalArraycopy;
364 private final Counters counters;
365
366 public Templates(ArrayCopySnippets receiver, OptionValues options, Iterable<DebugHandlersFactory> factories, Factory factory, Providers providers,
367 SnippetReflectionProvider snippetReflection, TargetDescription target) {
368 super(options, factories, providers, snippetReflection, target);
369 this.counters = new Counters(factory);
370
371 arraycopyGenericSnippet = snippet(receiver, "arraycopyGenericSnippet");
372 arraycopyUnrolledSnippet = snippet(receiver, "arraycopyUnrolledSnippet");
373 arraycopyExactSnippet = snippet(receiver, "arraycopyExactSnippet");
374 arraycopyZeroLengthSnippet = snippet(receiver, "arraycopyZeroLengthSnippet");
375 arraycopyCheckcastSnippet = snippet(receiver, "arraycopyCheckcastSnippet");
376 arraycopyNativeSnippet = snippet(null, "arraycopyNativeSnippet");
377 checkcastArraycopyWithSlowPathWork = snippet(receiver, "checkcastArraycopyWithSlowPathWork");
378 genericArraycopyWithSlowPathWork = snippet(receiver, "genericArraycopyWithSlowPathWork");
379 }
380
381 protected SnippetInfo snippet(ArrayCopySnippets receiver, String methodName) {
382 SnippetInfo info = snippet(ArrayCopySnippets.class, methodName, originalArraycopy(), receiver, LocationIdentity.any());
383 return info;
384 }
385
386 public void lower(ArrayCopyNode arraycopy, LoweringTool tool) {
387 JavaKind elementKind = selectComponentKind(arraycopy);
388 SnippetInfo snippetInfo;
389 ArrayCopyTypeCheck arrayTypeCheck;
390
391 ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp(NodeView.DEFAULT));
392 ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp(NodeView.DEFAULT));
393 if (!canBeArray(srcType) || !canBeArray(destType)) {
394 // at least one of the objects is definitely not an array - use the native call
395 // right away as the copying will fail anyways
396 snippetInfo = arraycopyNativeSnippet;
397 arrayTypeCheck = ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK;
398 } else {
399 ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType();
400 ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType();
401
402 if (arraycopy.isExact()) {
403 // there is a sufficient type match - we don't need any additional type checks
404 snippetInfo = arraycopyExactSnippet;
405 arrayTypeCheck = ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK;
406 } else if (srcComponentType == null && destComponentType == null) {
407 // we don't know anything about the types - use the generic copying
408 snippetInfo = arraycopyGenericSnippet;
409 // no need for additional type check to avoid duplicated work
410 arrayTypeCheck = ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK;
411 } else if (srcComponentType != null && destComponentType != null) {
412 if (!srcComponentType.isPrimitive() && !destComponentType.isPrimitive()) {
413 // it depends on the array content if the copy succeeds - we need
414 // a type check for every store
415 snippetInfo = arraycopyCheckcastSnippet;
416 arrayTypeCheck = ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK;
417 } else {
418 // one object is an object array, the other one is a primitive array.
419 // this copy will always fail - use the native call right away
420 assert !srcComponentType.equals(destComponentType) : "must be handled by arraycopy.isExact()";
421 snippetInfo = arraycopyNativeSnippet;
422 arrayTypeCheck = ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK;
423 }
424 } else {
425 ResolvedJavaType nonNullComponentType = srcComponentType != null ? srcComponentType : destComponentType;
426 if (nonNullComponentType.isPrimitive()) {
427 // one involved object is a primitive array - it is sufficient to directly
428 // compare the hub.
429 snippetInfo = arraycopyExactSnippet;
430 arrayTypeCheck = ArrayCopyTypeCheck.HUB_BASED_ARRAY_TYPE_CHECK;
431 elementKind = nonNullComponentType.getJavaKind();
432 } else {
433 // one involved object is an object array - the other array's element type
434 // may be primitive or object, hence we compare the layout helper.
435 snippetInfo = arraycopyCheckcastSnippet;
436 arrayTypeCheck = ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK;
437 }
438 }
439 }
440
441 // a few special cases that are easier to handle when all other variables already have a
442 // value
443 if (snippetInfo != arraycopyNativeSnippet && snippetInfo != arraycopyGenericSnippet && arraycopy.getLength().isConstant() && arraycopy.getLength().asJavaConstant().asLong() == 0) {
444 // Copying 0 element between object arrays with conflicting types will not throw an
445 // exception - once we pass the preliminary element type checks that we are not
446 // mixing arrays of different basic types, ArrayStoreException is only thrown when
447 // an *astore would have thrown it. Therefore, copying null between object arrays
448 // with conflicting types will also succeed (we do not optimize for such case here).
449 snippetInfo = arraycopyZeroLengthSnippet;
450 } else if (snippetInfo == arraycopyExactSnippet && shouldUnroll(arraycopy.getLength())) {
451 snippetInfo = arraycopyUnrolledSnippet;
452 }
453
454 // create the snippet
455 Arguments args = new Arguments(snippetInfo, arraycopy.graph().getGuardsStage(), tool.getLoweringStage());
456 args.add("src", arraycopy.getSource());
457 args.add("srcPos", arraycopy.getSourcePosition());
458 args.add("dest", arraycopy.getDestination());
459 args.add("destPos", arraycopy.getDestinationPosition());
460 args.add("length", arraycopy.getLength());
461 if (snippetInfo != arraycopyNativeSnippet) {
462 assert arrayTypeCheck != ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK;
463 args.addConst("arrayTypeCheck", arrayTypeCheck);
464 }
465 Object locationIdentity = arraycopy.killsAnyLocation() ? LocationIdentity.any() : NamedLocationIdentity.getArrayLocation(elementKind);
466 if (snippetInfo == arraycopyUnrolledSnippet) {
467 args.addConst("elementKind", elementKind != null ? elementKind : JavaKind.Illegal);
468 args.addConst("locationIdentity", locationIdentity);
469 args.addConst("unrolledLength", arraycopy.getLength().asJavaConstant().asInt());
470 }
471 if (snippetInfo == arraycopyExactSnippet) {
472 assert elementKind != null;
473 args.addConst("elementKind", elementKind);
474 args.addConst("locationIdentity", locationIdentity);
475 args.addConst("elementKindCounter", counters.arraycopyCallCounters.get(elementKind));
476 args.addConst("elementKindCopiedCounter", counters.arraycopyCallCopiedCounters.get(elementKind));
477 }
478 args.addConst("counters", counters);
479 if (snippetInfo == arraycopyCheckcastSnippet) {
480 args.addConst("workSnippet", checkcastArraycopyWithSlowPathWork);
481 args.addConst("elementKind", JavaKind.Illegal);
482 }
483 if (snippetInfo == arraycopyGenericSnippet) {
484 args.addConst("workSnippet", genericArraycopyWithSlowPathWork);
485 args.addConst("elementKind", JavaKind.Illegal);
486 }
487
488 instantiate(args, arraycopy);
489 }
490
491 public void lower(ArrayCopyWithSlowPathNode arraycopy, LoweringTool tool) {
492 StructuredGraph graph = arraycopy.graph();
493 if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
494 // if an arraycopy contains a slow path, we can't lower it right away
495 return;
496 }
497
498 SnippetInfo snippetInfo = arraycopy.getSnippet();
499 Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage());
500 args.add("src", arraycopy.getSource());
501 args.add("srcPos", arraycopy.getSourcePosition());
502 args.add("dest", arraycopy.getDestination());
503 args.add("destPos", arraycopy.getDestinationPosition());
504 args.add("length", arraycopy.getLength());
505 args.addConst("counters", counters);
506 instantiate(args, arraycopy);
507 }
508
509 private static boolean canBeArray(ResolvedJavaType type) {
510 return type == null || type.isJavaLangObject() || type.isArray();
511 }
512
513 public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy) {
514 ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp(NodeView.DEFAULT));
515 ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp(NodeView.DEFAULT));
516
517 if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) {
518 return null;
519 }
520 if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) {
521 return null;
522 }
523 if (!arraycopy.isExact()) {
524 return null;
525 }
526 return srcType.getComponentType().getJavaKind();
527 }
528
529 private static boolean shouldUnroll(ValueNode length) {
530 return length.isConstant() && length.asJavaConstant().asInt() <= 8 && length.asJavaConstant().asInt() != 0;
531 }
532
533 /**
534 * Instantiate the snippet template and fix up the FrameState of any Invokes of
535 * System.arraycopy and propagate the captured bci in the ArrayCopySlowPathNode.
536 *
537 * @param args
538 * @param arraycopy
539 */
540 private void instantiate(Arguments args, BasicArrayCopyNode arraycopy) {
541 StructuredGraph graph = arraycopy.graph();
542 SnippetTemplate template = template(arraycopy, args);
543 UnmodifiableEconomicMap<Node, Node> replacements = template.instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args, false);
544 for (Node originalNode : replacements.getKeys()) {
545 if (originalNode instanceof InvokeNode) {
546 InvokeNode invoke = (InvokeNode) replacements.get(originalNode);
547 assert invoke.asNode().graph() == graph;
548 CallTargetNode call = invoke.callTarget();
549
550 if (!call.targetMethod().equals(originalArraycopy)) {
551 throw new GraalError("unexpected invoke %s in snippet", call.targetMethod());
552 }
553 // Here we need to fix the bci of the invoke
554 InvokeNode newInvoke = invoke.replaceWithNewBci(arraycopy.getBci());
555 newInvoke.setStateDuring(null);
556 newInvoke.setStateAfter(null);
557 if (arraycopy.stateDuring() != null) {
558 newInvoke.setStateDuring(arraycopy.stateDuring());
559 } else {
560 assert arraycopy.stateAfter() != null : arraycopy;
561 newInvoke.setStateAfter(arraycopy.stateAfter());
562 }
563 } else if (originalNode instanceof InvokeWithExceptionNode) {
564 throw new GraalError("unexpected invoke with exception %s in snippet", originalNode);
565 } else if (originalNode instanceof ArrayCopyWithSlowPathNode) {
566 ArrayCopyWithSlowPathNode slowPath = (ArrayCopyWithSlowPathNode) replacements.get(originalNode);
567 assert arraycopy.stateAfter() != null : arraycopy;
568 assert slowPath.stateAfter() == arraycopy.stateAfter();
569 slowPath.setBci(arraycopy.getBci());
570 }
571 }
572 GraphUtil.killCFG(arraycopy);
573 }
574
575 private ResolvedJavaMethod originalArraycopy() throws GraalError {
576 if (originalArraycopy == null) {
577 try {
578 originalArraycopy = findMethod(providers.getMetaAccess(), System.class, "arraycopy");
579 } catch (SecurityException e) {
580 throw new GraalError(e);
581 }
582 }
583 return originalArraycopy;
584 }
585 }
586 }