1 /*
2 * Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23 package org.graalvm.compiler.hotspot.replacements.arraycopy;
24
25 import static org.graalvm.compiler.core.common.GraalOptions.SnippetCounters;
26 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
27 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.KLASS_SUPER_CHECK_OFFSET_LOCATION;
28 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION;
29 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset;
30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayClassElementOffset;
31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayIndexScale;
32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperElementTypePrimitiveInPlace;
33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadHub;
34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
35 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.superCheckOffsetOffset;
36 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
37 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
38 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
39
40 import java.lang.reflect.Method;
41 import java.util.EnumMap;
42 import java.util.Map;
43
44 import org.graalvm.compiler.api.directives.GraalDirectives;
45 import org.graalvm.compiler.api.replacements.Fold;
46 import org.graalvm.compiler.api.replacements.Snippet;
47 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
48 import org.graalvm.compiler.core.common.LocationIdentity;
49 import org.graalvm.compiler.debug.GraalError;
50 import org.graalvm.compiler.graph.Node;
51 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
52 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
53 import org.graalvm.compiler.hotspot.word.KlassPointer;
54 import org.graalvm.compiler.nodes.CallTargetNode;
55 import org.graalvm.compiler.nodes.ConstantNode;
56 import org.graalvm.compiler.nodes.DeoptimizeNode;
57 import org.graalvm.compiler.nodes.Invoke;
58 import org.graalvm.compiler.nodes.InvokeNode;
59 import org.graalvm.compiler.nodes.NamedLocationIdentity;
60 import org.graalvm.compiler.nodes.PiNode;
61 import org.graalvm.compiler.nodes.StructuredGraph;
62 import org.graalvm.compiler.nodes.ValueNode;
63 import org.graalvm.compiler.nodes.extended.UnsafeLoadNode;
64 import org.graalvm.compiler.nodes.java.ArrayLengthNode;
65 import org.graalvm.compiler.nodes.spi.LoweringTool;
66 import org.graalvm.compiler.nodes.type.StampTool;
67 import org.graalvm.compiler.replacements.SnippetCounter;
68 import org.graalvm.compiler.replacements.SnippetTemplate;
69 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
70 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
71 import org.graalvm.compiler.replacements.Snippets;
72 import org.graalvm.compiler.replacements.nodes.BasicArrayCopyNode;
73 import org.graalvm.compiler.replacements.nodes.DirectObjectStoreNode;
74 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
75 import org.graalvm.compiler.word.Word;
76
77 import jdk.vm.ci.code.TargetDescription;
78 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
79 import jdk.vm.ci.meta.DeoptimizationAction;
80 import jdk.vm.ci.meta.DeoptimizationReason;
81 import jdk.vm.ci.meta.JavaConstant;
82 import jdk.vm.ci.meta.JavaKind;
83 import jdk.vm.ci.meta.ResolvedJavaMethod;
84 import jdk.vm.ci.meta.ResolvedJavaType;
85
86 public class ArrayCopySnippets implements Snippets {
87
88 private static int checkArrayType(KlassPointer hub) {
89 int layoutHelper = readLayoutHelper(hub);
90 if (probability(SLOW_PATH_PROBABILITY, layoutHelper >= 0)) {
91 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
92 }
93 return layoutHelper;
94 }
95
96 private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length) {
97 if (probability(SLOW_PATH_PROBABILITY, srcPos < 0)) {
98 checkAIOOBECounter.inc();
99 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
100 }
101 if (probability(SLOW_PATH_PROBABILITY, destPos < 0)) {
102 checkAIOOBECounter.inc();
103 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
104 }
105 if (probability(SLOW_PATH_PROBABILITY, length < 0)) {
106 checkAIOOBECounter.inc();
107 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
108 }
109 if (probability(SLOW_PATH_PROBABILITY, srcPos > ArrayLengthNode.arrayLength(src) - length)) {
110 checkAIOOBECounter.inc();
111 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
112 }
113 if (probability(SLOW_PATH_PROBABILITY, destPos > ArrayLengthNode.arrayLength(dest) - length)) {
114 checkAIOOBECounter.inc();
115 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
116 }
117 checkSuccessCounter.inc();
118 }
119
120 @Snippet
121 public static void arraycopyZeroLengthIntrinsic(Object src, int srcPos, Object dest, int destPos, int length) {
122 Object nonNullSrc = GraalDirectives.guardingNonNull(src);
123 Object nonNullDest = GraalDirectives.guardingNonNull(dest);
124 KlassPointer srcHub = loadHub(nonNullSrc);
125 KlassPointer destHub = loadHub(nonNullDest);
126 checkArrayType(srcHub);
127 checkArrayType(destHub);
128 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length);
129 zeroLengthStaticCounter.inc();
130 }
131
132 @Snippet
133 public static void arraycopyExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind, @ConstantParameter SnippetCounter counter,
134 @ConstantParameter SnippetCounter copiedCounter) {
135 Object nonNullSrc = GraalDirectives.guardingNonNull(src);
136 Object nonNullDest = GraalDirectives.guardingNonNull(dest);
137 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length);
138 counter.inc();
139 copiedCounter.add(length);
140 ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind);
141 if (length == 0) {
142 zeroLengthDynamicCounter.inc();
143 } else {
144 nonZeroLengthDynamicCounter.inc();
145 nonZeroLengthDynamicCopiedCounter.add(length);
146 }
147 }
148
149 /**
150 * This intrinsic is useful for the case where we know something statically about one of the
151 * inputs but not the other.
152 */
153 @Snippet
154 public static void arraycopyPredictedExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind,
155 @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter) {
156 Object nonNullSrc = GraalDirectives.guardingNonNull(src);
157 Object nonNullDest = GraalDirectives.guardingNonNull(dest);
158 KlassPointer srcHub = loadHub(nonNullSrc);
159 KlassPointer destHub = loadHub(nonNullDest);
160 if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) {
161 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
162 }
163 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length);
164 counter.inc();
165 copiedCounter.add(length);
166 ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind);
167 if (length == 0) {
168 zeroLengthDynamicCounter.inc();
169 } else {
170 nonZeroLengthDynamicCounter.inc();
171 nonZeroLengthDynamicCopiedCounter.add(length);
172 }
173 }
174
175 @Snippet
176 public static void arraycopyPredictedObjectWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, KlassPointer objectArrayKlass,
177 @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter) {
178 if (length > 0) {
179 KlassPointer srcHub = loadHub(PiNode.asNonNullObject(nonNullSrc));
180 KlassPointer destHub = loadHub(PiNode.asNonNullObject(nonNullDest));
181 if (probability(FAST_PATH_PROBABILITY, srcHub == destHub || destHub == objectArrayKlass)) {
182 counter.inc();
183 copiedCounter.add(length);
184 predictedObjectArrayCopyFastPathCounter.inc();
185 predictedObjectArrayCopyFastPathCopiedCounter.add(length);
186 ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length);
187 } else {
188 predictedObjectArrayCopySlowPathCounter.inc();
189 predictedObjectArrayCopySlowPathCopiedCounter.add(length);
190 System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length);
191 }
192 }
193 }
194
195 /**
196 * This is the basic template for the full arraycopy checks, including a check that the
197 * underlying type is really an array type.
198 */
199 @Snippet
200 public static void arraycopySlowPathIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, KlassPointer predictedKlass, @ConstantParameter JavaKind elementKind,
201 @ConstantParameter SnippetInfo slowPath, @ConstantParameter Object slowPathArgument) {
202 Object nonNullSrc = GraalDirectives.guardingNonNull(src);
203 Object nonNullDest = GraalDirectives.guardingNonNull(dest);
204 KlassPointer srcHub = loadHub(nonNullSrc);
205 KlassPointer destHub = loadHub(nonNullDest);
206 checkArrayType(srcHub);
207 checkArrayType(destHub);
208 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length);
209 if (length == 0) {
210 zeroLengthDynamicCounter.inc();
211 } else {
212 nonZeroLengthDynamicCounter.inc();
213 nonZeroLengthDynamicCopiedCounter.add(length);
214 }
215 ArrayCopySlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, predictedKlass, elementKind, slowPath, slowPathArgument);
216 }
217
218 /**
219 * Snippet for unrolled arraycopy.
220 */
221 @Snippet
222 public static void arraycopyUnrolledIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter int unrolledLength, @ConstantParameter JavaKind elementKind) {
223 Object nonNullSrc = GraalDirectives.guardingNonNull(src);
224 Object nonNullDest = GraalDirectives.guardingNonNull(dest);
225 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length);
226 if (length == 0) {
227 zeroLengthDynamicCounter.inc();
228 } else {
229 nonZeroLengthDynamicCounter.inc();
230 nonZeroLengthDynamicCopiedCounter.add(length);
231 }
232 ArrayCopyUnrollNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, unrolledLength, elementKind);
233 }
234
235 @Snippet
236 public static void checkcastArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length) {
237 if (length > 0) {
238 KlassPointer destKlass = loadHub(nonNullDest);
239 KlassPointer srcKlass = loadHub(nonNullSrc);
240 if (probability(SLOW_PATH_PROBABILITY, srcKlass == destKlass)) {
241 // no storecheck required.
242 objectCheckcastSameTypeCounter.inc();
243 objectCheckcastSameTypeCopiedCounter.add(length);
244 ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length);
245 } else {
246 KlassPointer destElemKlass = destKlass.readKlassPointer(arrayClassElementOffset(INJECTED_VMCONFIG), OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION);
247 Word superCheckOffset = Word.signed(destElemKlass.readInt(superCheckOffsetOffset(INJECTED_VMCONFIG), KLASS_SUPER_CHECK_OFFSET_LOCATION));
248 objectCheckcastCounter.inc();
249 objectCheckcastCopiedCounter.add(length);
250 int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false);
251 if (copiedElements != 0) {
252 /*
253 * the checkcast stub doesn't throw the ArrayStoreException, but returns the
254 * number of copied elements (xor'd with -1).
255 */
256 copiedElements ^= -1;
257 System.arraycopy(nonNullSrc, srcPos + copiedElements, nonNullDest, destPos + copiedElements, length - copiedElements);
258 }
259 }
260 }
261 }
262
263 @Snippet
264 public static void arraycopyGeneric(Object src, int srcPos, Object dest, int destPos, int length) {
265 Object nonNullSrc = GraalDirectives.guardingNonNull(src);
266 Object nonNullDest = GraalDirectives.guardingNonNull(dest);
267 KlassPointer srcHub = loadHub(nonNullSrc);
268 KlassPointer destHub = loadHub(nonNullDest);
269 if (probability(FAST_PATH_PROBABILITY, srcHub.equal(destHub)) && probability(FAST_PATH_PROBABILITY, nonNullSrc != nonNullDest)) {
270 int layoutHelper = checkArrayType(srcHub);
271 final boolean isObjectArray = ((layoutHelper & layoutHelperElementTypePrimitiveInPlace(INJECTED_VMCONFIG)) == 0);
272 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length);
273 if (probability(FAST_PATH_PROBABILITY, isObjectArray)) {
274 genericObjectExactCallCounter.inc();
275 genericObjectExactCallCopiedCounter.add(length);
276 ArrayCopyCallNode.disjointArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, JavaKind.Object);
277 } else {
278 genericPrimitiveCallCounter.inc();
279 genericPrimitiveCallCopiedCounter.add(length);
280 UnsafeArrayCopyNode.arraycopyPrimitive(nonNullSrc, srcPos, nonNullDest, destPos, length, layoutHelper);
281 }
282 } else {
283 SystemArraycopyCounter.inc();
284 SystemArraycopyCopiedCounter.add(length);
285 System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length);
286 }
287 }
288
289 @Fold
290 static LocationIdentity getArrayLocation(JavaKind kind) {
291 return NamedLocationIdentity.getArrayLocation(kind);
292 }
293
294 @Snippet
295 public static void arraycopyUnrolledWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, @ConstantParameter int length, @ConstantParameter JavaKind elementKind) {
296 final int scale = arrayIndexScale(elementKind);
297 int arrayBaseOffset = arrayBaseOffset(elementKind);
298 LocationIdentity arrayLocation = getArrayLocation(elementKind);
299 if (nonNullSrc == nonNullDest && srcPos < destPos) { // bad aliased case
300 long start = (long) (length - 1) * scale;
301 long i = start;
302 ExplodeLoopNode.explodeLoop();
303 for (int iteration = 0; iteration < length; iteration++) {
304 if (i >= 0) {
305 Object a = UnsafeLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation);
306 DirectObjectStoreNode.storeObject(nonNullDest, arrayBaseOffset, i + (long) destPos * scale, a, arrayLocation, elementKind);
307 i -= scale;
308 }
309 }
310 } else {
311 long end = (long) length * scale;
312 long i = 0;
313 ExplodeLoopNode.explodeLoop();
314 for (int iteration = 0; iteration < length; iteration++) {
315 if (i < end) {
316 Object a = UnsafeLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation);
317 DirectObjectStoreNode.storeObject(nonNullDest, arrayBaseOffset, i + (long) destPos * scale, a, arrayLocation, elementKind);
318 i += scale;
319 }
320 }
321 }
322 }
323
324 private static final SnippetCounter.Group checkCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy checkInputs") : null;
325 private static final SnippetCounter checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess");
326 private static final SnippetCounter checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE");
327
328 private static final SnippetCounter.Group counters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy") : null;
329
330 private static final SnippetCounter objectCheckcastCounter = new SnippetCounter(counters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays");
331 private static final SnippetCounter objectCheckcastSameTypeCounter = new SnippetCounter(counters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays");
332 private static final SnippetCounter predictedObjectArrayCopySlowPathCounter = new SnippetCounter(counters, "Object[]{slow-path}", "used System.arraycopy slow path for predicted Object[] arrays");
333 private static final SnippetCounter predictedObjectArrayCopyFastPathCounter = new SnippetCounter(counters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays");
334
335 private static final EnumMap<JavaKind, SnippetCounter> arraycopyCallCounters = new EnumMap<>(JavaKind.class);
336 private static final EnumMap<JavaKind, SnippetCounter> arraycopyCounters = new EnumMap<>(JavaKind.class);
337
338 private static final EnumMap<JavaKind, SnippetCounter> arraycopyCallCopiedCounters = new EnumMap<>(JavaKind.class);
339 private static final EnumMap<JavaKind, SnippetCounter> arraycopyCopiedCounters = new EnumMap<>(JavaKind.class);
340
341 static void createArraycopyCounter(JavaKind kind) {
342 arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays"));
343 arraycopyCounters.put(kind, new SnippetCounter(counters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays"));
344
345 arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays"));
346 arraycopyCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays"));
347 }
348
349 static {
350 createArraycopyCounter(JavaKind.Byte);
351 createArraycopyCounter(JavaKind.Boolean);
352 createArraycopyCounter(JavaKind.Char);
353 createArraycopyCounter(JavaKind.Short);
354 createArraycopyCounter(JavaKind.Int);
355 createArraycopyCounter(JavaKind.Long);
356 createArraycopyCounter(JavaKind.Float);
357 createArraycopyCounter(JavaKind.Double);
358 createArraycopyCounter(JavaKind.Object);
359 }
360
361 private static final SnippetCounter genericPrimitiveCallCounter = new SnippetCounter(counters, "genericPrimitive", "generic arraycopy snippet for primitive arrays");
362 private static final SnippetCounter genericObjectExactCallCounter = new SnippetCounter(counters, "genericObjectExact", "generic arraycopy snippet for special object arrays");
363 private static final SnippetCounter SystemArraycopyCounter = new SnippetCounter(counters, "genericObject", "call to System.arraycopy");
364
365 private static final SnippetCounter.Group lengthCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy 0-length checks") : null;
366
367 private static final SnippetCounter zeroLengthStaticCounter = new SnippetCounter(lengthCounters, "0-lengthcopy static", "arraycopy where the length is statically 0");
368 private static final SnippetCounter zeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "0-lengthcopy dynamically", "arraycopy where the length is dynamically 0");
369 private static final SnippetCounter nonZeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero");
370
371 private static final SnippetCounter.Group copiedCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy copied elements") : null;
372
373 private static final SnippetCounter nonZeroLengthDynamicCopiedCounter = new SnippetCounter(copiedCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero");
374 private static final SnippetCounter genericPrimitiveCallCopiedCounter = new SnippetCounter(copiedCounters, "genericPrimitive", "generic arraycopy snippet for primitive arrays");
375 private static final SnippetCounter genericObjectExactCallCopiedCounter = new SnippetCounter(copiedCounters, "genericObjectExact", "generic arraycopy snippet for special object arrays");
376 private static final SnippetCounter SystemArraycopyCopiedCounter = new SnippetCounter(copiedCounters, "genericObject", "call to System.arraycopy");
377
378 private static final SnippetCounter objectCheckcastCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays");
379 private static final SnippetCounter objectCheckcastSameTypeCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays");
380 private static final SnippetCounter predictedObjectArrayCopySlowPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{slow-path}",
381 "used System.arraycopy slow path for predicted Object[] arrays");
382 private static final SnippetCounter predictedObjectArrayCopyFastPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays");
383
384 public static class Templates extends SnippetTemplate.AbstractTemplates {
385
386 public Templates(HotSpotProviders providers, TargetDescription target) {
387 super(providers, providers.getSnippetReflection(), target);
388 }
389
390 private ResolvedJavaMethod originalArraycopy() throws GraalError {
391 if (originalArraycopy == null) {
392 Method method;
393 try {
394 method = System.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class);
395 } catch (NoSuchMethodException | SecurityException e) {
396 throw new GraalError(e);
397 }
398 originalArraycopy = providers.getMetaAccess().lookupJavaMethod(method);
399 }
400 return originalArraycopy;
401 }
402
403 private ResolvedJavaMethod originalArraycopy;
404
405 private final SnippetInfo checkcastArraycopyWorkSnippet = snippet("checkcastArraycopyWork");
406 private final SnippetInfo arraycopyGenericSnippet = snippet("arraycopyGeneric");
407
408 private final SnippetInfo arraycopySlowPathIntrinsicSnippet = snippet("arraycopySlowPathIntrinsic");
409 private final SnippetInfo arraycopyUnrolledIntrinsicSnippet = snippet("arraycopyUnrolledIntrinsic");
410 private final SnippetInfo arraycopyExactIntrinsicSnippet = snippet("arraycopyExactIntrinsic");
411 private final SnippetInfo arraycopyZeroLengthIntrinsicSnippet = snippet("arraycopyZeroLengthIntrinsic");
412 private final SnippetInfo arraycopyPredictedExactIntrinsicSnippet = snippet("arraycopyPredictedExactIntrinsic");
413 private final SnippetInfo arraycopyPredictedObjectWorkSnippet = snippet("arraycopyPredictedObjectWork");
414
415 private final SnippetInfo arraycopyUnrolledWorkSnippet = snippet("arraycopyUnrolledWork");
416
417 protected SnippetInfo snippet(String methodName) {
418 SnippetInfo info = snippet(ArrayCopySnippets.class, methodName, LocationIdentity.any());
419 info.setOriginalMethod(originalArraycopy());
420 return info;
421 }
422
423 public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy) {
424 return selectComponentKind(arraycopy, true);
425 }
426
427 public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy, boolean exact) {
428 ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp());
429 ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp());
430
431 if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) {
432 if (!exact) {
433 JavaKind component = getComponentKind(srcType);
434 if (component != null) {
435 return component;
436 }
437 return getComponentKind(destType);
438 }
439 return null;
440 }
441 if (exact) {
442 if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) {
443 return null;
444 }
445 if (!arraycopy.isExact()) {
446 return null;
447 }
448 }
449 return srcType.getComponentType().getJavaKind();
450 }
451
452 private static JavaKind getComponentKind(ResolvedJavaType type) {
453 if (type != null && type.isArray()) {
454 return type.getComponentType().getJavaKind();
455 }
456 return null;
457 }
458
459 private static boolean shouldUnroll(ValueNode length) {
460 return length.isConstant() && length.asJavaConstant().asInt() <= 8 && length.asJavaConstant().asInt() != 0;
461 }
462
463 public void lower(ArrayCopyNode arraycopy, LoweringTool tool) {
464 JavaKind componentKind = selectComponentKind(arraycopy);
465 SnippetInfo snippetInfo = null;
466 SnippetInfo slowPathSnippetInfo = null;
467 Object slowPathArgument = null;
468
469 if (arraycopy.getLength().isConstant() && arraycopy.getLength().asJavaConstant().asLong() == 0) {
470 snippetInfo = arraycopyZeroLengthIntrinsicSnippet;
471 } else if (arraycopy.isExact()) {
472 snippetInfo = arraycopyExactIntrinsicSnippet;
473 if (shouldUnroll(arraycopy.getLength())) {
474 snippetInfo = arraycopyUnrolledIntrinsicSnippet;
475 }
476 } else {
477 if (componentKind == JavaKind.Object) {
478 ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp());
479 ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp());
480 ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType();
481 ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType();
482 if (srcComponentType != null && destComponentType != null && !srcComponentType.isPrimitive() && !destComponentType.isPrimitive()) {
483 snippetInfo = arraycopySlowPathIntrinsicSnippet;
484 slowPathSnippetInfo = checkcastArraycopyWorkSnippet;
485 slowPathArgument = LocationIdentity.any();
486 /*
487 * Because this snippet has to use Sysytem.arraycopy as a slow path, we must
488 * pretend to kill any() so clear the componentKind.
489 */
490 componentKind = null;
491 }
492 }
493 if (componentKind == null && snippetInfo == null) {
494 JavaKind predictedKind = selectComponentKind(arraycopy, false);
495 if (predictedKind != null) {
496 /*
497 * At least one array is of a known type requiring no store checks, so
498 * assume the other is of the same type. Generally this is working around
499 * deficiencies in our propagation of type information.
500 */
501 componentKind = predictedKind;
502 if (predictedKind == JavaKind.Object) {
503 snippetInfo = arraycopySlowPathIntrinsicSnippet;
504 slowPathSnippetInfo = arraycopyPredictedObjectWorkSnippet;
505 slowPathArgument = predictedKind;
506 componentKind = null;
507 } else {
508 snippetInfo = arraycopyPredictedExactIntrinsicSnippet;
509 }
510 }
511 }
512 if (snippetInfo == null) {
513 snippetInfo = arraycopyGenericSnippet;
514 }
515 }
516 Arguments args = new Arguments(snippetInfo, arraycopy.graph().getGuardsStage(), tool.getLoweringStage());
517 args.add("src", arraycopy.getSource());
518 args.add("srcPos", arraycopy.getSourcePosition());
519 args.add("dest", arraycopy.getDestination());
520 args.add("destPos", arraycopy.getDestinationPosition());
521 args.add("length", arraycopy.getLength());
522 if (snippetInfo == arraycopyUnrolledIntrinsicSnippet) {
523 args.addConst("unrolledLength", arraycopy.getLength().asJavaConstant().asInt());
524 args.addConst("elementKind", componentKind != null ? componentKind : JavaKind.Illegal);
525 } else if (snippetInfo == arraycopySlowPathIntrinsicSnippet) {
526 ValueNode predictedKlass = null;
527 if (slowPathArgument == arraycopyPredictedObjectWorkSnippet) {
528 HotSpotResolvedObjectType arrayClass = (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(Object[].class);
529 predictedKlass = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayClass.klass(), tool.getMetaAccess(), arraycopy.graph());
530 } else {
531 predictedKlass = ConstantNode.forConstant(KlassPointerStamp.klassAlwaysNull(), JavaConstant.NULL_POINTER, tool.getMetaAccess(), arraycopy.graph());
532 }
533 args.add("predictedKlass", predictedKlass);
534 args.addConst("elementKind", componentKind != null ? componentKind : JavaKind.Illegal);
535 args.addConst("slowPath", slowPathSnippetInfo);
536 assert slowPathArgument != null;
537 args.addConst("slowPathArgument", slowPathArgument);
538 } else if (snippetInfo == arraycopyExactIntrinsicSnippet || snippetInfo == arraycopyPredictedExactIntrinsicSnippet) {
539 assert componentKind != null;
540 args.addConst("elementKind", componentKind);
541 args.addConst("counter", arraycopyCallCounters.get(componentKind));
542 args.addConst("copiedCounter", arraycopyCallCopiedCounters.get(componentKind));
543 }
544 instantiate(args, arraycopy);
545 }
546
547 public void lower(ArrayCopySlowPathNode arraycopy, LoweringTool tool) {
548 StructuredGraph graph = arraycopy.graph();
549 if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
550 // Can't be lowered yet
551 return;
552 }
553 SnippetInfo snippetInfo = arraycopy.getSnippet();
554 Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage());
555 args.add("nonNullSrc", arraycopy.getSource());
556 args.add("srcPos", arraycopy.getSourcePosition());
557 args.add("nonNullDest", arraycopy.getDestination());
558 args.add("destPos", arraycopy.getDestinationPosition());
559 if (snippetInfo == arraycopyUnrolledWorkSnippet) {
560 args.addConst("length", ((Integer) arraycopy.getArgument()).intValue());
561 args.addConst("elementKind", arraycopy.getElementKind());
562 } else {
563 args.add("length", arraycopy.getLength());
564 }
565 if (snippetInfo == arraycopyPredictedObjectWorkSnippet) {
566 args.add("objectArrayKlass", arraycopy.getPredictedKlass());
567 args.addConst("counter", arraycopyCallCounters.get(JavaKind.Object));
568 args.addConst("copiedCounter", arraycopyCallCopiedCounters.get(JavaKind.Object));
569 }
570 instantiate(args, arraycopy);
571 }
572
573 public void lower(ArrayCopyUnrollNode arraycopy, LoweringTool tool) {
574 StructuredGraph graph = arraycopy.graph();
575 if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
576 // Can't be lowered yet
577 return;
578 }
579 SnippetInfo snippetInfo = arraycopyUnrolledWorkSnippet;
580 Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage());
581 args.add("nonNullSrc", arraycopy.getSource());
582 args.add("srcPos", arraycopy.getSourcePosition());
583 args.add("nonNullDest", arraycopy.getDestination());
584 args.add("destPos", arraycopy.getDestinationPosition());
585 args.addConst("length", arraycopy.getUnrollLength());
586 args.addConst("elementKind", arraycopy.getElementKind());
587 template(args).instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args);
588 }
589
590 /**
591 * Instantiate the snippet template and fix up the FrameState of any Invokes of
592 * System.arraycopy and propagate the captured bci in the ArrayCopySlowPathNode.
593 *
594 * @param args
595 * @param arraycopy
596 */
597 private void instantiate(Arguments args, BasicArrayCopyNode arraycopy) {
598 StructuredGraph graph = arraycopy.graph();
599 SnippetTemplate template = template(args);
600 Map<Node, Node> replacements = template.instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args);
601 for (Node originalNode : replacements.keySet()) {
602 if (originalNode instanceof Invoke) {
603 Invoke invoke = (Invoke) replacements.get(originalNode);
604 assert invoke.asNode().graph() == graph;
605 CallTargetNode call = invoke.callTarget();
606
607 if (!call.targetMethod().equals(originalArraycopy)) {
608 throw new GraalError("unexpected invoke %s in snippet", call.targetMethod());
609 }
610 // Here we need to fix the bci of the invoke
611 InvokeNode newInvoke = graph.add(new InvokeNode(invoke.callTarget(), arraycopy.getBci()));
612 if (arraycopy.stateDuring() != null) {
613 newInvoke.setStateDuring(arraycopy.stateDuring());
614 } else {
615 assert arraycopy.stateAfter() != null;
616 newInvoke.setStateAfter(arraycopy.stateAfter());
617 }
618 graph.replaceFixedWithFixed((InvokeNode) invoke.asNode(), newInvoke);
619 } else if (originalNode instanceof ArrayCopySlowPathNode) {
620 ArrayCopySlowPathNode slowPath = (ArrayCopySlowPathNode) replacements.get(originalNode);
621 assert arraycopy.stateAfter() != null;
622 slowPath.setStateAfter(arraycopy.stateAfter());
623 slowPath.setBci(arraycopy.getBci());
624 }
625 }
626 }
627 }
628 }