1 /*
2 * Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23 package org.graalvm.compiler.hotspot.replacements.arraycopy;
24
25 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
26 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.KLASS_SUPER_CHECK_OFFSET_LOCATION;
27 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION;
28 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset;
29 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayClassElementOffset;
30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayIndexScale;
31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadHub;
32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.superCheckOffsetOffset;
34 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
35 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.LIKELY_PROBABILITY;
36 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY;
37 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
38 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
39
40 import java.lang.reflect.Method;
41 import java.util.EnumMap;
42
43 import org.graalvm.compiler.api.directives.GraalDirectives;
44 import org.graalvm.compiler.api.replacements.Fold;
45 import org.graalvm.compiler.api.replacements.Snippet;
46 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
47 import org.graalvm.compiler.debug.DebugHandlersFactory;
48 import org.graalvm.compiler.debug.GraalError;
49 import org.graalvm.compiler.graph.Node;
50 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
51 import org.graalvm.compiler.hotspot.word.KlassPointer;
52 import org.graalvm.compiler.nodes.CallTargetNode;
53 import org.graalvm.compiler.nodes.DeoptimizeNode;
54 import org.graalvm.compiler.nodes.Invoke;
55 import org.graalvm.compiler.nodes.InvokeNode;
56 import org.graalvm.compiler.nodes.NamedLocationIdentity;
57 import org.graalvm.compiler.nodes.PiNode;
58 import org.graalvm.compiler.nodes.StructuredGraph;
59 import org.graalvm.compiler.nodes.ValueNode;
60 import org.graalvm.compiler.nodes.extended.RawLoadNode;
61 import org.graalvm.compiler.nodes.extended.RawStoreNode;
62 import org.graalvm.compiler.nodes.java.ArrayLengthNode;
63 import org.graalvm.compiler.nodes.spi.LoweringTool;
64 import org.graalvm.compiler.nodes.type.StampTool;
65 import org.graalvm.compiler.nodes.util.GraphUtil;
66 import org.graalvm.compiler.options.OptionValues;
67 import org.graalvm.compiler.replacements.ReplacementsUtil;
68 import org.graalvm.compiler.replacements.SnippetCounter;
69 import org.graalvm.compiler.replacements.SnippetCounter.Group;
70 import org.graalvm.compiler.replacements.SnippetIntegerHistogram;
71 import org.graalvm.compiler.replacements.SnippetTemplate;
72 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
73 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
74 import org.graalvm.compiler.replacements.Snippets;
75 import org.graalvm.compiler.replacements.nodes.BasicArrayCopyNode;
76 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
77 import org.graalvm.compiler.word.Word;
78 import org.graalvm.util.UnmodifiableEconomicMap;
79 import org.graalvm.word.LocationIdentity;
80 import org.graalvm.word.WordFactory;
81
82 import jdk.vm.ci.code.TargetDescription;
83 import jdk.vm.ci.meta.DeoptimizationAction;
84 import jdk.vm.ci.meta.DeoptimizationReason;
85 import jdk.vm.ci.meta.JavaKind;
86 import jdk.vm.ci.meta.ResolvedJavaMethod;
87 import jdk.vm.ci.meta.ResolvedJavaType;
88
89 public class ArrayCopySnippets implements Snippets {
90
91 private enum ArrayCopyTypeCheck {
92 UNDEFINED_ARRAY_TYPE_CHECK,
93 // we know that both objects are arrays and have the same type
94 NO_ARRAY_TYPE_CHECK,
95 // can be used when we know that one of the objects is a primitive array
96 HUB_BASED_ARRAY_TYPE_CHECK,
97 // must be used when we don't have sufficient information to use one of the others
98 LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK
99 }
100
101 @Snippet
102 public static void arraycopyZeroLengthSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
103 @ConstantParameter Counters counters) {
104 Object nonNullSrc = GraalDirectives.guardingNonNull(src);
105 Object nonNullDest = GraalDirectives.guardingNonNull(dest);
106 checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
107 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
108 counters.zeroLengthStaticCounter.inc();
109 }
110
111 @Snippet
112 public static void arraycopyExactSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
113 @ConstantParameter JavaKind elementKind, @ConstantParameter SnippetCounter elementKindCounter, @ConstantParameter SnippetCounter elementKindCopiedCounter,
114 @ConstantParameter Counters counters) {
115 Object nonNullSrc = GraalDirectives.guardingNonNull(src);
116 Object nonNullDest = GraalDirectives.guardingNonNull(dest);
117 checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
118 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
119 incrementLengthCounter(length, counters);
120
121 elementKindCounter.inc();
122 elementKindCopiedCounter.add(length);
123 ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind);
124 }
125
126 @Snippet
127 public static void arraycopyUnrolledSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
128 @ConstantParameter JavaKind elementKind, @ConstantParameter int unrolledLength, @ConstantParameter Counters counters) {
129 Object nonNullSrc = GraalDirectives.guardingNonNull(src);
130 Object nonNullDest = GraalDirectives.guardingNonNull(dest);
131 checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
132 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
133 incrementLengthCounter(length, counters);
134
135 unrolledArraycopyWork(nonNullSrc, srcPos, nonNullDest, destPos, unrolledLength, elementKind);
136 }
137
138 @Snippet
139 public static void arraycopyCheckcastSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck,
140 @ConstantParameter Counters counters, @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind) {
141 Object nonNullSrc = GraalDirectives.guardingNonNull(src);
142 Object nonNullDest = GraalDirectives.guardingNonNull(dest);
143 checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
144 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
145 incrementLengthCounter(length, counters);
146
147 ArrayCopyWithSlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind);
148 }
149
150 @Snippet
151 public static void arraycopyGenericSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter ArrayCopyTypeCheck arrayTypeCheck, @ConstantParameter Counters counters,
152 @ConstantParameter SnippetInfo workSnippet, @ConstantParameter JavaKind elementKind) {
153 Object nonNullSrc = GraalDirectives.guardingNonNull(src);
154 Object nonNullDest = GraalDirectives.guardingNonNull(dest);
155 checkArrayTypes(nonNullSrc, nonNullDest, arrayTypeCheck);
156 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
157 incrementLengthCounter(length, counters);
158
159 ArrayCopyWithSlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, workSnippet, elementKind);
160 }
161
162 @Snippet
163 public static void arraycopyNativeSnippet(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
164 // all checks are done in the native method, so no need to emit additional checks here
165 incrementLengthCounter(length, counters);
166 counters.systemArraycopyCounter.inc();
167 counters.systemArraycopyCopiedCounter.add(length);
168
169 System.arraycopy(src, srcPos, dest, destPos, length);
170 }
171
172 @Fold
173 static LocationIdentity getArrayLocation(JavaKind kind) {
174 return NamedLocationIdentity.getArrayLocation(kind);
175 }
176
177 private static void unrolledArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, JavaKind elementKind) {
178 int scale = arrayIndexScale(elementKind);
179 int arrayBaseOffset = arrayBaseOffset(elementKind);
180 LocationIdentity arrayLocation = getArrayLocation(elementKind);
181
182 long sourceOffset = arrayBaseOffset + (long) srcPos * scale;
183 long destOffset = arrayBaseOffset + (long) destPos * scale;
184 long position = 0;
185 long delta = scale;
186 if (probability(NOT_FREQUENT_PROBABILITY, nonNullSrc == nonNullDest && srcPos < destPos)) {
187 // bad aliased case so we need to copy the array from back to front
188 position = (long) (length - 1) * scale;
189 delta = -delta;
190 }
191
192 // the length was already checked before - we can emit unconditional instructions
193 ExplodeLoopNode.explodeLoop();
194 for (int iteration = 0; iteration < length; iteration++) {
195 Object value = RawLoadNode.load(nonNullSrc, sourceOffset + position, elementKind, arrayLocation);
196 RawStoreNode.storeObject(nonNullDest, destOffset + position, value, elementKind, arrayLocation, false);
197 position += delta;
198 }
199 }
200
201 @Snippet(allowPartialIntrinsicArgumentMismatch = true)
202 public static void checkcastArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
203 if (probability(FREQUENT_PROBABILITY, length > 0)) {
204 Object nonNullSrc = PiNode.asNonNullObject(src);
205 Object nonNullDest = PiNode.asNonNullObject(dest);
206 KlassPointer srcKlass = loadHub(nonNullSrc);
207 KlassPointer destKlass = loadHub(nonNullDest);
208 if (probability(LIKELY_PROBABILITY, srcKlass == destKlass)) {
209 // no storecheck required.
210 counters.objectCheckcastSameTypeCounter.inc();
211 counters.objectCheckcastSameTypeCopiedCounter.add(length);
212 ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length);
213 } else {
214 KlassPointer destElemKlass = destKlass.readKlassPointer(arrayClassElementOffset(INJECTED_VMCONFIG), OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION);
215 Word superCheckOffset = WordFactory.signed(destElemKlass.readInt(superCheckOffsetOffset(INJECTED_VMCONFIG), KLASS_SUPER_CHECK_OFFSET_LOCATION));
216
217 counters.objectCheckcastDifferentTypeCounter.inc();
218 counters.objectCheckcastDifferentTypeCopiedCounter.add(length);
219
220 int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false);
221 if (probability(SLOW_PATH_PROBABILITY, copiedElements != 0)) {
222 /*
223 * the stub doesn't throw the ArrayStoreException, but returns the number of
224 * copied elements (xor'd with -1).
225 */
226 copiedElements ^= -1;
227 System.arraycopy(nonNullSrc, srcPos + copiedElements, nonNullDest, destPos + copiedElements, length - copiedElements);
228 }
229 }
230 }
231 }
232
233 @Snippet(allowPartialIntrinsicArgumentMismatch = true)
234 public static void genericArraycopyWithSlowPathWork(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
235 if (probability(FREQUENT_PROBABILITY, length > 0)) {
236 counters.genericArraycopyDifferentTypeCounter.inc();
237 counters.genericArraycopyDifferentTypeCopiedCounter.add(length);
238 int copiedElements = GenericArrayCopyCallNode.genericArraycopy(src, srcPos, dest, destPos, length);
239 if (probability(SLOW_PATH_PROBABILITY, copiedElements != 0)) {
240 /*
241 * the stub doesn't throw the ArrayStoreException, but returns the number of copied
242 * elements (xor'd with -1).
243 */
244 copiedElements ^= -1;
245 System.arraycopy(src, srcPos + copiedElements, dest, destPos + copiedElements, length - copiedElements);
246 }
247 }
248 }
249
250 private static void incrementLengthCounter(int length, Counters counters) {
251 counters.lengthHistogram.inc(length);
252 }
253
254 private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length, Counters counters) {
255 if (probability(SLOW_PATH_PROBABILITY, srcPos < 0) ||
256 probability(SLOW_PATH_PROBABILITY, destPos < 0) ||
257 probability(SLOW_PATH_PROBABILITY, length < 0) ||
258 probability(SLOW_PATH_PROBABILITY, srcPos > ArrayLengthNode.arrayLength(src) - length) ||
259 probability(SLOW_PATH_PROBABILITY, destPos > ArrayLengthNode.arrayLength(dest) - length)) {
260 counters.checkAIOOBECounter.inc();
261 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
262 }
263 counters.checkSuccessCounter.inc();
264 }
265
266 private static void checkArrayTypes(Object nonNullSrc, Object nonNullDest, ArrayCopyTypeCheck arrayTypeCheck) {
267 if (arrayTypeCheck == ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK) {
268 // nothing to do
269 } else if (arrayTypeCheck == ArrayCopyTypeCheck.HUB_BASED_ARRAY_TYPE_CHECK) {
270 KlassPointer srcHub = loadHub(nonNullSrc);
271 KlassPointer destHub = loadHub(nonNullDest);
272 if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) {
273 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
274 }
275 } else if (arrayTypeCheck == ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK) {
276 KlassPointer srcHub = loadHub(nonNullSrc);
277 KlassPointer destHub = loadHub(nonNullDest);
278 checkArrayType(srcHub);
279 checkArrayType(destHub);
280 } else {
281 ReplacementsUtil.staticAssert(false, "unknown array type check");
282 }
283 }
284
285 private static int checkArrayType(KlassPointer nonNullHub) {
286 int layoutHelper = readLayoutHelper(nonNullHub);
287 if (probability(SLOW_PATH_PROBABILITY, layoutHelper >= 0)) {
288 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
289 }
290 return layoutHelper;
291 }
292
293 static class Counters {
294 final SnippetCounter checkSuccessCounter;
295 final SnippetCounter checkAIOOBECounter;
296
297 final SnippetCounter zeroLengthStaticCounter;
298 final SnippetIntegerHistogram lengthHistogram;
299
300 final SnippetCounter systemArraycopyCounter;
301 final SnippetCounter systemArraycopyCopiedCounter;
302
303 final SnippetCounter genericArraycopyDifferentTypeCopiedCounter;
304 final SnippetCounter genericArraycopyDifferentTypeCounter;
305
306 final SnippetCounter objectCheckcastSameTypeCopiedCounter;
307 final SnippetCounter objectCheckcastSameTypeCounter;
308 final SnippetCounter objectCheckcastDifferentTypeCopiedCounter;
309 final SnippetCounter objectCheckcastDifferentTypeCounter;
310
311 final EnumMap<JavaKind, SnippetCounter> arraycopyCallCounters = new EnumMap<>(JavaKind.class);
312 final EnumMap<JavaKind, SnippetCounter> arraycopyCallCopiedCounters = new EnumMap<>(JavaKind.class);
313
314 Counters(SnippetCounter.Group.Factory factory) {
315 final Group checkCounters = factory.createSnippetCounterGroup("System.arraycopy checkInputs");
316 final Group callCounters = factory.createSnippetCounterGroup("System.arraycopy calls");
317 final Group copiedElementsCounters = factory.createSnippetCounterGroup("System.arraycopy copied elements");
318 final Group lengthCounters = factory.createSnippetCounterGroup("System.arraycopy with 0-length");
319
320 checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess");
321 checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE");
322
323 zeroLengthStaticCounter = new SnippetCounter(lengthCounters, "0-length copy static", "calls where the length is statically 0");
324 lengthHistogram = new SnippetIntegerHistogram(lengthCounters, 2, "length", "length");
325
326 systemArraycopyCounter = new SnippetCounter(callCounters, "native System.arraycopy", "JNI-based System.arraycopy call");
327 systemArraycopyCopiedCounter = new SnippetCounter(copiedElementsCounters, "native System.arraycopy", "JNI-based System.arraycopy call");
328
329 genericArraycopyDifferentTypeCounter = new SnippetCounter(callCounters, "generic[] stub", "generic arraycopy stub");
330 genericArraycopyDifferentTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "generic[] stub", "generic arraycopy stub");
331
332 objectCheckcastSameTypeCounter = new SnippetCounter(callCounters, "checkcast object[] (same-type)", "checkcast object[] stub but src.klass == dest.klass Object[] arrays");
333 objectCheckcastSameTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "checkcast object[] (same-type)", "checkcast object[] stub but src.klass == dest.klass Object[] arrays");
334 objectCheckcastDifferentTypeCounter = new SnippetCounter(callCounters, "checkcast object[] (store-check)", "checkcast object[] stub with store check");
335 objectCheckcastDifferentTypeCopiedCounter = new SnippetCounter(copiedElementsCounters, "checkcast object[] (store-check)", "checkcast object[] stub with store check");
336
337 createArraycopyCounter(JavaKind.Byte, callCounters, copiedElementsCounters);
338 createArraycopyCounter(JavaKind.Boolean, callCounters, copiedElementsCounters);
339 createArraycopyCounter(JavaKind.Char, callCounters, copiedElementsCounters);
340 createArraycopyCounter(JavaKind.Short, callCounters, copiedElementsCounters);
341 createArraycopyCounter(JavaKind.Int, callCounters, copiedElementsCounters);
342 createArraycopyCounter(JavaKind.Long, callCounters, copiedElementsCounters);
343 createArraycopyCounter(JavaKind.Float, callCounters, copiedElementsCounters);
344 createArraycopyCounter(JavaKind.Double, callCounters, copiedElementsCounters);
345 createArraycopyCounter(JavaKind.Object, callCounters, copiedElementsCounters);
346 }
347
348 void createArraycopyCounter(JavaKind kind, Group counters, Group copiedCounters) {
349 arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[] stub", "arraycopy call for " + kind + "[] arrays"));
350 arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[] stub", "arraycopy call for " + kind + "[] arrays"));
351 }
352 }
353
354 public static class Templates extends SnippetTemplate.AbstractTemplates {
355 private final SnippetInfo arraycopyGenericSnippet = snippet("arraycopyGenericSnippet");
356 private final SnippetInfo arraycopyUnrolledSnippet = snippet("arraycopyUnrolledSnippet");
357 private final SnippetInfo arraycopyExactSnippet = snippet("arraycopyExactSnippet");
358 private final SnippetInfo arraycopyZeroLengthSnippet = snippet("arraycopyZeroLengthSnippet");
359 private final SnippetInfo arraycopyCheckcastSnippet = snippet("arraycopyCheckcastSnippet");
360 private final SnippetInfo arraycopyNativeSnippet = snippet("arraycopyNativeSnippet");
361
362 private final SnippetInfo checkcastArraycopyWithSlowPathWork = snippet("checkcastArraycopyWithSlowPathWork");
363 private final SnippetInfo genericArraycopyWithSlowPathWork = snippet("genericArraycopyWithSlowPathWork");
364
365 private ResolvedJavaMethod originalArraycopy;
366 private final Counters counters;
367
368 public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target) {
369 super(options, factories, providers, providers.getSnippetReflection(), target);
370 this.counters = new Counters(factory);
371 }
372
373 protected SnippetInfo snippet(String methodName) {
374 SnippetInfo info = snippet(ArrayCopySnippets.class, methodName, LocationIdentity.any());
375 info.setOriginalMethod(originalArraycopy());
376 return info;
377 }
378
379 public void lower(ArrayCopyNode arraycopy, LoweringTool tool) {
380 JavaKind elementKind = selectComponentKind(arraycopy);
381 SnippetInfo snippetInfo;
382 ArrayCopyTypeCheck arrayTypeCheck;
383
384 ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp());
385 ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp());
386 if (!canBeArray(srcType) || !canBeArray(destType)) {
387 // at least one of the objects is definitely not an array - use the native call
388 // right away as the copying will fail anyways
389 snippetInfo = arraycopyNativeSnippet;
390 arrayTypeCheck = ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK;
391 } else {
392 ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType();
393 ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType();
394
395 if (arraycopy.isExact()) {
396 // there is a sufficient type match - we don't need any additional type checks
397 snippetInfo = arraycopyExactSnippet;
398 arrayTypeCheck = ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK;
399 } else if (srcComponentType == null && destComponentType == null) {
400 // we don't know anything about the types - use the generic copying
401 snippetInfo = arraycopyGenericSnippet;
402 arrayTypeCheck = ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK;
403 } else if (srcComponentType != null && destComponentType != null) {
404 if (!srcComponentType.isPrimitive() && !destComponentType.isPrimitive()) {
405 // it depends on the array content if the copy succeeds - we need
406 // a type check for every store
407 snippetInfo = arraycopyCheckcastSnippet;
408 arrayTypeCheck = ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK;
409 } else {
410 // one object is an object array, the other one is a primitive array.
411 // this copy will always fail - use the native call right away
412 assert !srcComponentType.equals(destComponentType) : "must be handled by arraycopy.isExact()";
413 snippetInfo = arraycopyNativeSnippet;
414 arrayTypeCheck = ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK;
415 }
416 } else {
417 ResolvedJavaType nonNullComponentType = srcComponentType != null ? srcComponentType : destComponentType;
418 if (nonNullComponentType.isPrimitive()) {
419 // one involved object is a primitive array - we can safely assume that we
420 // are copying primitive arrays
421 snippetInfo = arraycopyExactSnippet;
422 arrayTypeCheck = ArrayCopyTypeCheck.HUB_BASED_ARRAY_TYPE_CHECK;
423 elementKind = nonNullComponentType.getJavaKind();
424 } else {
425 // one involved object is an object array - we can safely assume that we are
426 // copying object arrays that might require a store check
427 snippetInfo = arraycopyCheckcastSnippet;
428 arrayTypeCheck = ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK;
429 }
430 }
431 }
432
433 // a few special cases that are easier to handle when all other variables already have a
434 // value
435 if (arraycopy.getLength().isConstant() && arraycopy.getLength().asJavaConstant().asLong() == 0) {
436 snippetInfo = arraycopyZeroLengthSnippet;
437 } else if (snippetInfo == arraycopyExactSnippet && shouldUnroll(arraycopy.getLength())) {
438 snippetInfo = arraycopyUnrolledSnippet;
439 }
440
441 // create the snippet
442 Arguments args = new Arguments(snippetInfo, arraycopy.graph().getGuardsStage(), tool.getLoweringStage());
443 args.add("src", arraycopy.getSource());
444 args.add("srcPos", arraycopy.getSourcePosition());
445 args.add("dest", arraycopy.getDestination());
446 args.add("destPos", arraycopy.getDestinationPosition());
447 args.add("length", arraycopy.getLength());
448 if (snippetInfo != arraycopyNativeSnippet) {
449 assert arrayTypeCheck != ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK;
450 args.addConst("arrayTypeCheck", arrayTypeCheck);
451 }
452 if (snippetInfo == arraycopyUnrolledSnippet) {
453 args.addConst("elementKind", elementKind != null ? elementKind : JavaKind.Illegal);
454 args.addConst("unrolledLength", arraycopy.getLength().asJavaConstant().asInt());
455 }
456 if (snippetInfo == arraycopyExactSnippet) {
457 assert elementKind != null;
458 args.addConst("elementKind", elementKind);
459 args.addConst("elementKindCounter", counters.arraycopyCallCounters.get(elementKind));
460 args.addConst("elementKindCopiedCounter", counters.arraycopyCallCopiedCounters.get(elementKind));
461 }
462 args.addConst("counters", counters);
463 if (snippetInfo == arraycopyCheckcastSnippet) {
464 args.addConst("workSnippet", checkcastArraycopyWithSlowPathWork);
465 args.addConst("elementKind", JavaKind.Illegal);
466 }
467 if (snippetInfo == arraycopyGenericSnippet) {
468 args.addConst("workSnippet", genericArraycopyWithSlowPathWork);
469 args.addConst("elementKind", JavaKind.Illegal);
470 }
471
472 instantiate(args, arraycopy);
473 }
474
475 public void lower(ArrayCopyWithSlowPathNode arraycopy, LoweringTool tool) {
476 StructuredGraph graph = arraycopy.graph();
477 if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
478 // if an arraycopy contains a slow path, we can't lower it right away
479 return;
480 }
481
482 SnippetInfo snippetInfo = arraycopy.getSnippet();
483 Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage());
484 args.add("src", arraycopy.getSource());
485 args.add("srcPos", arraycopy.getSourcePosition());
486 args.add("dest", arraycopy.getDestination());
487 args.add("destPos", arraycopy.getDestinationPosition());
488 args.add("length", arraycopy.getLength());
489 args.addConst("counters", counters);
490 instantiate(args, arraycopy);
491 }
492
493 private static boolean canBeArray(ResolvedJavaType type) {
494 return type == null || type.isJavaLangObject() || type.isArray();
495 }
496
497 public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy) {
498 ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp());
499 ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp());
500
501 if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) {
502 return null;
503 }
504 if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) {
505 return null;
506 }
507 if (!arraycopy.isExact()) {
508 return null;
509 }
510 return srcType.getComponentType().getJavaKind();
511 }
512
513 private static boolean shouldUnroll(ValueNode length) {
514 return length.isConstant() && length.asJavaConstant().asInt() <= 8 && length.asJavaConstant().asInt() != 0;
515 }
516
517 /**
518 * Instantiate the snippet template and fix up the FrameState of any Invokes of
519 * System.arraycopy and propagate the captured bci in the ArrayCopySlowPathNode.
520 *
521 * @param args
522 * @param arraycopy
523 */
524 private void instantiate(Arguments args, BasicArrayCopyNode arraycopy) {
525 StructuredGraph graph = arraycopy.graph();
526 SnippetTemplate template = template(graph.getDebug(), args);
527 UnmodifiableEconomicMap<Node, Node> replacements = template.instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args, false);
528 for (Node originalNode : replacements.getKeys()) {
529 if (originalNode instanceof Invoke) {
530 Invoke invoke = (Invoke) replacements.get(originalNode);
531 assert invoke.asNode().graph() == graph;
532 CallTargetNode call = invoke.callTarget();
533
534 if (!call.targetMethod().equals(originalArraycopy)) {
535 throw new GraalError("unexpected invoke %s in snippet", call.targetMethod());
536 }
537 // Here we need to fix the bci of the invoke
538 InvokeNode newInvoke = graph.add(new InvokeNode(invoke.callTarget(), arraycopy.getBci()));
539 if (arraycopy.stateDuring() != null) {
540 newInvoke.setStateDuring(arraycopy.stateDuring());
541 } else {
542 assert arraycopy.stateAfter() != null : arraycopy;
543 newInvoke.setStateAfter(arraycopy.stateAfter());
544 }
545 graph.replaceFixedWithFixed((InvokeNode) invoke.asNode(), newInvoke);
546 } else if (originalNode instanceof ArrayCopyWithSlowPathNode) {
547 ArrayCopyWithSlowPathNode slowPath = (ArrayCopyWithSlowPathNode) replacements.get(originalNode);
548 assert arraycopy.stateAfter() != null : arraycopy;
549 assert slowPath.stateAfter() == arraycopy.stateAfter();
550 slowPath.setBci(arraycopy.getBci());
551 }
552 }
553 GraphUtil.killCFG(arraycopy);
554 }
555
556 private ResolvedJavaMethod originalArraycopy() throws GraalError {
557 if (originalArraycopy == null) {
558 Method method;
559 try {
560 method = System.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class);
561 } catch (NoSuchMethodException | SecurityException e) {
562 throw new GraalError(e);
563 }
564 originalArraycopy = providers.getMetaAccess().lookupJavaMethod(method);
565 }
566 return originalArraycopy;
567 }
568 }
569 }