1 /*
2 * Copyright (c) 2011, 2016, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23 package org.graalvm.compiler.hotspot.replacements.arraycopy;
24
25 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
26 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.KLASS_SUPER_CHECK_OFFSET_LOCATION;
27 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION;
28 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset;
29 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayClassElementOffset;
30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayIndexScale;
31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperElementTypePrimitiveInPlace;
32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadHub;
33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.superCheckOffsetOffset;
35 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
36 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
37 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
38
39 import java.lang.reflect.Method;
40 import java.util.EnumMap;
41
42 import org.graalvm.compiler.api.directives.GraalDirectives;
43 import org.graalvm.compiler.api.replacements.Fold;
44 import org.graalvm.compiler.api.replacements.Snippet;
45 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
46 import org.graalvm.compiler.core.common.LocationIdentity;
47 import org.graalvm.compiler.debug.GraalError;
48 import org.graalvm.compiler.graph.Node;
49 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
50 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
51 import org.graalvm.compiler.hotspot.word.KlassPointer;
52 import org.graalvm.compiler.nodes.CallTargetNode;
53 import org.graalvm.compiler.nodes.ConstantNode;
54 import org.graalvm.compiler.nodes.DeoptimizeNode;
55 import org.graalvm.compiler.nodes.Invoke;
56 import org.graalvm.compiler.nodes.InvokeNode;
57 import org.graalvm.compiler.nodes.NamedLocationIdentity;
58 import org.graalvm.compiler.nodes.PiNode;
59 import org.graalvm.compiler.nodes.StructuredGraph;
60 import org.graalvm.compiler.nodes.ValueNode;
61 import org.graalvm.compiler.nodes.extended.RawLoadNode;
62 import org.graalvm.compiler.nodes.extended.RawStoreNode;
63 import org.graalvm.compiler.nodes.java.ArrayLengthNode;
64 import org.graalvm.compiler.nodes.spi.LoweringTool;
65 import org.graalvm.compiler.nodes.type.StampTool;
66 import org.graalvm.compiler.options.OptionValues;
67 import org.graalvm.compiler.replacements.SnippetCounter;
68 import org.graalvm.compiler.replacements.SnippetCounter.Group;
69 import org.graalvm.compiler.replacements.SnippetTemplate;
70 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
71 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
72 import org.graalvm.compiler.replacements.Snippets;
73 import org.graalvm.compiler.replacements.nodes.BasicArrayCopyNode;
74 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
75 import org.graalvm.compiler.word.Word;
76 import org.graalvm.util.UnmodifiableEconomicMap;
77
78 import jdk.vm.ci.code.TargetDescription;
79 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
80 import jdk.vm.ci.meta.DeoptimizationAction;
81 import jdk.vm.ci.meta.DeoptimizationReason;
82 import jdk.vm.ci.meta.JavaConstant;
83 import jdk.vm.ci.meta.JavaKind;
84 import jdk.vm.ci.meta.ResolvedJavaMethod;
85 import jdk.vm.ci.meta.ResolvedJavaType;
86
87 public class ArrayCopySnippets implements Snippets {
88
89 private static int checkArrayType(KlassPointer nonNullHub) {
90 int layoutHelper = readLayoutHelper(nonNullHub);
91 if (probability(SLOW_PATH_PROBABILITY, layoutHelper >= 0)) {
92 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
93 }
94 return layoutHelper;
95 }
96
97 private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length, Counters counters) {
98 if (probability(SLOW_PATH_PROBABILITY, srcPos < 0)) {
99 counters.checkAIOOBECounter.inc();
100 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
101 }
102 if (probability(SLOW_PATH_PROBABILITY, destPos < 0)) {
103 counters.checkAIOOBECounter.inc();
104 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
105 }
106 if (probability(SLOW_PATH_PROBABILITY, length < 0)) {
107 counters.checkAIOOBECounter.inc();
108 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
109 }
110 if (probability(SLOW_PATH_PROBABILITY, srcPos > ArrayLengthNode.arrayLength(src) - length)) {
111 counters.checkAIOOBECounter.inc();
112 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
113 }
114 if (probability(SLOW_PATH_PROBABILITY, destPos > ArrayLengthNode.arrayLength(dest) - length)) {
115 counters.checkAIOOBECounter.inc();
116 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
117 }
118 counters.checkSuccessCounter.inc();
119 }
120
121 @Snippet
122 public static void arraycopyZeroLengthIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
123 Object nonNullSrc = GraalDirectives.guardingNonNull(src);
124 Object nonNullDest = GraalDirectives.guardingNonNull(dest);
125 KlassPointer srcHub = loadHub(nonNullSrc);
126 KlassPointer destHub = loadHub(nonNullDest);
127 checkArrayType(srcHub);
128 checkArrayType(destHub);
129 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
130 counters.zeroLengthStaticCounter.inc();
131 }
132
133 @Snippet
134 public static void arraycopyExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind, @ConstantParameter SnippetCounter counter,
135 @ConstantParameter SnippetCounter copiedCounter, @ConstantParameter Counters counters) {
136 Object nonNullSrc = GraalDirectives.guardingNonNull(src);
137 Object nonNullDest = GraalDirectives.guardingNonNull(dest);
138 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
139 counter.inc();
140 copiedCounter.add(length);
141 ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind);
142 if (length == 0) {
143 counters.zeroLengthDynamicCounter.inc();
144 } else {
145 counters.nonZeroLengthDynamicCounter.inc();
146 counters.nonZeroLengthDynamicCopiedCounter.add(length);
147 }
148 }
149
150 /**
151 * This intrinsic is useful for the case where we know something statically about one of the
152 * inputs but not the other.
153 */
154 @Snippet
155 public static void arraycopyPredictedExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter JavaKind elementKind,
156 @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter, @ConstantParameter Counters counters) {
157 Object nonNullSrc = GraalDirectives.guardingNonNull(src);
158 Object nonNullDest = GraalDirectives.guardingNonNull(dest);
159 KlassPointer srcHub = loadHub(nonNullSrc);
160 KlassPointer destHub = loadHub(nonNullDest);
161 if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) {
162 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
163 }
164 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
165 counter.inc();
166 copiedCounter.add(length);
167 ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind);
168 if (length == 0) {
169 counters.zeroLengthDynamicCounter.inc();
170 } else {
171 counters.nonZeroLengthDynamicCounter.inc();
172 counters.nonZeroLengthDynamicCopiedCounter.add(length);
173 }
174 }
175
176 @Snippet
177 public static void arraycopyPredictedObjectWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, KlassPointer objectArrayKlass,
178 @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter, @ConstantParameter Counters counters) {
179 if (length > 0) {
180 KlassPointer srcHub = loadHub(PiNode.asNonNullObject(nonNullSrc));
181 KlassPointer destHub = loadHub(PiNode.asNonNullObject(nonNullDest));
182 if (probability(FAST_PATH_PROBABILITY, srcHub == destHub || destHub == objectArrayKlass)) {
183 counter.inc();
184 copiedCounter.add(length);
185 counters.predictedObjectArrayCopyFastPathCounter.inc();
186 counters.predictedObjectArrayCopyFastPathCopiedCounter.add(length);
187 ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length);
188 } else {
189 counters.predictedObjectArrayCopySlowPathCounter.inc();
190 counters.predictedObjectArrayCopySlowPathCopiedCounter.add(length);
191 System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length);
192 }
193 }
194 }
195
196 /**
197 * This is the basic template for the full arraycopy checks, including a check that the
198 * underlying type is really an array type.
199 */
200 @Snippet
201 public static void arraycopySlowPathIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, KlassPointer predictedKlass, @ConstantParameter JavaKind elementKind,
202 @ConstantParameter SnippetInfo slowPath, @ConstantParameter Object slowPathArgument, @ConstantParameter Counters counters) {
203 Object nonNullSrc = GraalDirectives.guardingNonNull(src);
204 Object nonNullDest = GraalDirectives.guardingNonNull(dest);
205 KlassPointer srcHub = loadHub(nonNullSrc);
206 KlassPointer destHub = loadHub(nonNullDest);
207 checkArrayType(srcHub);
208 checkArrayType(destHub);
209 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
210 if (length == 0) {
211 counters.zeroLengthDynamicCounter.inc();
212 } else {
213 counters.nonZeroLengthDynamicCounter.inc();
214 counters.nonZeroLengthDynamicCopiedCounter.add(length);
215 }
216 ArrayCopySlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, predictedKlass, elementKind, slowPath, slowPathArgument);
217 }
218
219 /**
220 * Snippet for unrolled arraycopy.
221 */
222 @Snippet
223 public static void arraycopyUnrolledIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter int unrolledLength, @ConstantParameter JavaKind elementKind,
224 @ConstantParameter Counters counters) {
225 Object nonNullSrc = GraalDirectives.guardingNonNull(src);
226 Object nonNullDest = GraalDirectives.guardingNonNull(dest);
227 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
228 if (length == 0) {
229 counters.zeroLengthDynamicCounter.inc();
230 } else {
231 counters.nonZeroLengthDynamicCounter.inc();
232 counters.nonZeroLengthDynamicCopiedCounter.add(length);
233 }
234 ArrayCopyUnrollNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, unrolledLength, elementKind);
235 }
236
237 @Snippet
238 public static void checkcastArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, @ConstantParameter Counters counters) {
239 if (length > 0) {
240 KlassPointer destKlass = loadHub(nonNullDest);
241 KlassPointer srcKlass = loadHub(nonNullSrc);
242 if (probability(SLOW_PATH_PROBABILITY, srcKlass == destKlass)) {
243 // no storecheck required.
244 counters.objectCheckcastSameTypeCounter.inc();
245 counters.objectCheckcastSameTypeCopiedCounter.add(length);
246 ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length);
247 } else {
248 KlassPointer destElemKlass = destKlass.readKlassPointer(arrayClassElementOffset(INJECTED_VMCONFIG), OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION);
249 Word superCheckOffset = Word.signed(destElemKlass.readInt(superCheckOffsetOffset(INJECTED_VMCONFIG), KLASS_SUPER_CHECK_OFFSET_LOCATION));
250 counters.objectCheckcastCounter.inc();
251 counters.objectCheckcastCopiedCounter.add(length);
252 int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false);
253 if (copiedElements != 0) {
254 /*
255 * the checkcast stub doesn't throw the ArrayStoreException, but returns the
256 * number of copied elements (xor'd with -1).
257 */
258 copiedElements ^= -1;
259 System.arraycopy(nonNullSrc, srcPos + copiedElements, nonNullDest, destPos + copiedElements, length - copiedElements);
260 }
261 }
262 }
263 }
264
265 @Snippet
266 public static void arraycopyGeneric(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Counters counters) {
267 Object nonNullSrc = GraalDirectives.guardingNonNull(src);
268 Object nonNullDest = GraalDirectives.guardingNonNull(dest);
269 KlassPointer srcHub = loadHub(nonNullSrc);
270 KlassPointer destHub = loadHub(nonNullDest);
271 if (probability(FAST_PATH_PROBABILITY, srcHub.equal(destHub)) && probability(FAST_PATH_PROBABILITY, nonNullSrc != nonNullDest)) {
272 int layoutHelper = checkArrayType(srcHub);
273 final boolean isObjectArray = ((layoutHelper & layoutHelperElementTypePrimitiveInPlace(INJECTED_VMCONFIG)) == 0);
274 checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length, counters);
275 if (probability(FAST_PATH_PROBABILITY, isObjectArray)) {
276 counters.genericObjectExactCallCounter.inc();
277 counters.genericObjectExactCallCopiedCounter.add(length);
278 ArrayCopyCallNode.disjointArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, JavaKind.Object);
279 } else {
280 counters.genericPrimitiveCallCounter.inc();
281 counters.genericPrimitiveCallCopiedCounter.add(length);
282 UnsafeArrayCopyNode.arraycopyPrimitive(nonNullSrc, srcPos, nonNullDest, destPos, length, layoutHelper);
283 }
284 } else {
285 counters.systemArraycopyCounter.inc();
286 counters.systemArraycopyCopiedCounter.add(length);
287 System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length);
288 }
289 }
290
291 @Fold
292 static LocationIdentity getArrayLocation(JavaKind kind) {
293 return NamedLocationIdentity.getArrayLocation(kind);
294 }
295
296 @Snippet
297 public static void arraycopyUnrolledWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, @ConstantParameter int length, @ConstantParameter JavaKind elementKind) {
298 final int scale = arrayIndexScale(elementKind);
299 int arrayBaseOffset = arrayBaseOffset(elementKind);
300 LocationIdentity arrayLocation = getArrayLocation(elementKind);
301 if (nonNullSrc == nonNullDest && srcPos < destPos) { // bad aliased case
302 long start = (long) (length - 1) * scale;
303 long i = start;
304 ExplodeLoopNode.explodeLoop();
305 for (int iteration = 0; iteration < length; iteration++) {
306 if (i >= 0) {
307 Object a = RawLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation);
308 RawStoreNode.storeObject(nonNullDest, arrayBaseOffset + i + (long) destPos * scale, a, elementKind, arrayLocation, false);
309 i -= scale;
310 }
311 }
312 } else {
313 long end = (long) length * scale;
314 long i = 0;
315 ExplodeLoopNode.explodeLoop();
316 for (int iteration = 0; iteration < length; iteration++) {
317 if (i < end) {
318 Object a = RawLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation);
319 RawStoreNode.storeObject(nonNullDest, arrayBaseOffset + i + (long) destPos * scale, a, elementKind, arrayLocation, false);
320 i += scale;
321 }
322 }
323 }
324 }
325
326 static class Counters {
327 final SnippetCounter checkSuccessCounter;
328 final SnippetCounter checkAIOOBECounter;
329
330 final SnippetCounter objectCheckcastCounter;
331 final SnippetCounter objectCheckcastSameTypeCounter;
332 final SnippetCounter predictedObjectArrayCopySlowPathCounter;
333 final SnippetCounter predictedObjectArrayCopyFastPathCounter;
334
335 final SnippetCounter genericPrimitiveCallCounter;
336 final SnippetCounter genericObjectExactCallCounter;
337 final SnippetCounter systemArraycopyCounter;
338
339 final SnippetCounter zeroLengthStaticCounter;
340 final SnippetCounter zeroLengthDynamicCounter;
341 final SnippetCounter nonZeroLengthDynamicCounter;
342
343 final SnippetCounter nonZeroLengthDynamicCopiedCounter;
344 final SnippetCounter genericPrimitiveCallCopiedCounter;
345 final SnippetCounter genericObjectExactCallCopiedCounter;
346 final SnippetCounter systemArraycopyCopiedCounter;
347
348 final SnippetCounter objectCheckcastCopiedCounter;
349 final SnippetCounter objectCheckcastSameTypeCopiedCounter;
350 final SnippetCounter predictedObjectArrayCopySlowPathCopiedCounter;
351 final SnippetCounter predictedObjectArrayCopyFastPathCopiedCounter;
352
353 final EnumMap<JavaKind, SnippetCounter> arraycopyCallCounters = new EnumMap<>(JavaKind.class);
354 final EnumMap<JavaKind, SnippetCounter> arraycopyCounters = new EnumMap<>(JavaKind.class);
355
356 final EnumMap<JavaKind, SnippetCounter> arraycopyCallCopiedCounters = new EnumMap<>(JavaKind.class);
357 final EnumMap<JavaKind, SnippetCounter> arraycopyCopiedCounters = new EnumMap<>(JavaKind.class);
358
359 Counters(SnippetCounter.Group.Factory factory) {
360 final Group checkCounters = factory.createSnippetCounterGroup("System.arraycopy checkInputs");
361 final Group counters = factory.createSnippetCounterGroup("System.arraycopy");
362 final Group copiedCounters = factory.createSnippetCounterGroup("System.arraycopy copied elements");
363 final Group lengthCounters = factory.createSnippetCounterGroup("System.arraycopy 0-length checks");
364
365 checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess");
366 checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE");
367
368 objectCheckcastCounter = new SnippetCounter(counters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays");
369 objectCheckcastSameTypeCounter = new SnippetCounter(counters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays");
370 predictedObjectArrayCopySlowPathCounter = new SnippetCounter(counters, "Object[]{slow-path}", "used System.arraycopy slow path for predicted Object[] arrays");
371 predictedObjectArrayCopyFastPathCounter = new SnippetCounter(counters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays");
372 genericPrimitiveCallCounter = new SnippetCounter(counters, "genericPrimitive", "generic arraycopy snippet for primitive arrays");
373 genericObjectExactCallCounter = new SnippetCounter(counters, "genericObjectExact", "generic arraycopy snippet for special object arrays");
374 systemArraycopyCounter = new SnippetCounter(counters, "genericObject", "call to System.arraycopy");
375
376 zeroLengthStaticCounter = new SnippetCounter(lengthCounters, "0-lengthcopy static", "arraycopy where the length is statically 0");
377 zeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "0-lengthcopy dynamically", "arraycopy where the length is dynamically 0");
378 nonZeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero");
379
380 nonZeroLengthDynamicCopiedCounter = new SnippetCounter(copiedCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero");
381 genericPrimitiveCallCopiedCounter = new SnippetCounter(copiedCounters, "genericPrimitive", "generic arraycopy snippet for primitive arrays");
382 genericObjectExactCallCopiedCounter = new SnippetCounter(copiedCounters, "genericObjectExact", "generic arraycopy snippet for special object arrays");
383 systemArraycopyCopiedCounter = new SnippetCounter(copiedCounters, "genericObject", "call to System.arraycopy");
384
385 objectCheckcastCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays");
386 objectCheckcastSameTypeCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays");
387 predictedObjectArrayCopySlowPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{slow-path}",
388 "used System.arraycopy slow path for predicted Object[] arrays");
389 predictedObjectArrayCopyFastPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays");
390 createArraycopyCounter(JavaKind.Byte, counters, copiedCounters);
391 createArraycopyCounter(JavaKind.Boolean, counters, copiedCounters);
392 createArraycopyCounter(JavaKind.Char, counters, copiedCounters);
393 createArraycopyCounter(JavaKind.Short, counters, copiedCounters);
394 createArraycopyCounter(JavaKind.Int, counters, copiedCounters);
395 createArraycopyCounter(JavaKind.Long, counters, copiedCounters);
396 createArraycopyCounter(JavaKind.Float, counters, copiedCounters);
397 createArraycopyCounter(JavaKind.Double, counters, copiedCounters);
398 createArraycopyCounter(JavaKind.Object, counters, copiedCounters);
399 }
400
401 void createArraycopyCounter(JavaKind kind, Group counters, Group copiedCounters) {
402 arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays"));
403 arraycopyCounters.put(kind, new SnippetCounter(counters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays"));
404
405 arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays"));
406 arraycopyCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays"));
407 }
408 }
409
410 public static class Templates extends SnippetTemplate.AbstractTemplates {
411
412 public Templates(OptionValues options, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target) {
413 super(options, providers, providers.getSnippetReflection(), target);
414 this.counters = new Counters(factory);
415 }
416
417 private ResolvedJavaMethod originalArraycopy() throws GraalError {
418 if (originalArraycopy == null) {
419 Method method;
420 try {
421 method = System.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class);
422 } catch (NoSuchMethodException | SecurityException e) {
423 throw new GraalError(e);
424 }
425 originalArraycopy = providers.getMetaAccess().lookupJavaMethod(method);
426 }
427 return originalArraycopy;
428 }
429
430 private ResolvedJavaMethod originalArraycopy;
431
432 private final SnippetInfo checkcastArraycopyWorkSnippet = snippet("checkcastArraycopyWork");
433 private final SnippetInfo arraycopyGenericSnippet = snippet("arraycopyGeneric");
434
435 private final SnippetInfo arraycopySlowPathIntrinsicSnippet = snippet("arraycopySlowPathIntrinsic");
436 private final SnippetInfo arraycopyUnrolledIntrinsicSnippet = snippet("arraycopyUnrolledIntrinsic");
437 private final SnippetInfo arraycopyExactIntrinsicSnippet = snippet("arraycopyExactIntrinsic");
438 private final SnippetInfo arraycopyZeroLengthIntrinsicSnippet = snippet("arraycopyZeroLengthIntrinsic");
439 private final SnippetInfo arraycopyPredictedExactIntrinsicSnippet = snippet("arraycopyPredictedExactIntrinsic");
440 private final SnippetInfo arraycopyPredictedObjectWorkSnippet = snippet("arraycopyPredictedObjectWork");
441
442 private final SnippetInfo arraycopyUnrolledWorkSnippet = snippet("arraycopyUnrolledWork");
443
444 private final Counters counters;
445
446 protected SnippetInfo snippet(String methodName) {
447 SnippetInfo info = snippet(ArrayCopySnippets.class, methodName, LocationIdentity.any());
448 info.setOriginalMethod(originalArraycopy());
449 return info;
450 }
451
452 public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy) {
453 return selectComponentKind(arraycopy, true);
454 }
455
456 public static JavaKind selectComponentKind(BasicArrayCopyNode arraycopy, boolean exact) {
457 ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp());
458 ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp());
459
460 if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) {
461 if (!exact) {
462 JavaKind component = getComponentKind(srcType);
463 if (component != null) {
464 return component;
465 }
466 return getComponentKind(destType);
467 }
468 return null;
469 }
470 if (exact) {
471 if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) {
472 return null;
473 }
474 if (!arraycopy.isExact()) {
475 return null;
476 }
477 }
478 return srcType.getComponentType().getJavaKind();
479 }
480
481 private static JavaKind getComponentKind(ResolvedJavaType type) {
482 if (type != null && type.isArray()) {
483 return type.getComponentType().getJavaKind();
484 }
485 return null;
486 }
487
488 private static boolean shouldUnroll(ValueNode length) {
489 return length.isConstant() && length.asJavaConstant().asInt() <= 8 && length.asJavaConstant().asInt() != 0;
490 }
491
492 public void lower(ArrayCopyNode arraycopy, LoweringTool tool) {
493 JavaKind componentKind = selectComponentKind(arraycopy);
494 SnippetInfo snippetInfo = null;
495 SnippetInfo slowPathSnippetInfo = null;
496 Object slowPathArgument = null;
497
498 if (arraycopy.getLength().isConstant() && arraycopy.getLength().asJavaConstant().asLong() == 0) {
499 snippetInfo = arraycopyZeroLengthIntrinsicSnippet;
500 } else if (arraycopy.isExact()) {
501 snippetInfo = arraycopyExactIntrinsicSnippet;
502 if (shouldUnroll(arraycopy.getLength())) {
503 snippetInfo = arraycopyUnrolledIntrinsicSnippet;
504 }
505 } else {
506 if (componentKind == JavaKind.Object) {
507 ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp());
508 ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp());
509 ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType();
510 ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType();
511 if (srcComponentType != null && destComponentType != null && !srcComponentType.isPrimitive() && !destComponentType.isPrimitive()) {
512 snippetInfo = arraycopySlowPathIntrinsicSnippet;
513 slowPathSnippetInfo = checkcastArraycopyWorkSnippet;
514 slowPathArgument = LocationIdentity.any();
515 /*
516 * Because this snippet has to use Sysytem.arraycopy as a slow path, we must
517 * pretend to kill any() so clear the componentKind.
518 */
519 componentKind = null;
520 }
521 }
522 if (componentKind == null && snippetInfo == null) {
523 JavaKind predictedKind = selectComponentKind(arraycopy, false);
524 if (predictedKind != null) {
525 /*
526 * At least one array is of a known type requiring no store checks, so
527 * assume the other is of the same type. Generally this is working around
528 * deficiencies in our propagation of type information.
529 */
530 componentKind = predictedKind;
531 if (predictedKind == JavaKind.Object) {
532 snippetInfo = arraycopySlowPathIntrinsicSnippet;
533 slowPathSnippetInfo = arraycopyPredictedObjectWorkSnippet;
534 slowPathArgument = predictedKind;
535 componentKind = null;
536 } else {
537 snippetInfo = arraycopyPredictedExactIntrinsicSnippet;
538 }
539 }
540 }
541 if (snippetInfo == null) {
542 snippetInfo = arraycopyGenericSnippet;
543 }
544 }
545 Arguments args = new Arguments(snippetInfo, arraycopy.graph().getGuardsStage(), tool.getLoweringStage());
546 args.add("src", arraycopy.getSource());
547 args.add("srcPos", arraycopy.getSourcePosition());
548 args.add("dest", arraycopy.getDestination());
549 args.add("destPos", arraycopy.getDestinationPosition());
550 args.add("length", arraycopy.getLength());
551 if (snippetInfo == arraycopyUnrolledIntrinsicSnippet) {
552 args.addConst("unrolledLength", arraycopy.getLength().asJavaConstant().asInt());
553 args.addConst("elementKind", componentKind != null ? componentKind : JavaKind.Illegal);
554 } else if (snippetInfo == arraycopySlowPathIntrinsicSnippet) {
555 ValueNode predictedKlass = null;
556 if (slowPathArgument == arraycopyPredictedObjectWorkSnippet) {
557 HotSpotResolvedObjectType arrayClass = (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(Object[].class);
558 predictedKlass = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayClass.klass(), tool.getMetaAccess(), arraycopy.graph());
559 } else {
560 predictedKlass = ConstantNode.forConstant(KlassPointerStamp.klassAlwaysNull(), JavaConstant.NULL_POINTER, tool.getMetaAccess(), arraycopy.graph());
561 }
562 args.add("predictedKlass", predictedKlass);
563 args.addConst("elementKind", componentKind != null ? componentKind : JavaKind.Illegal);
564 args.addConst("slowPath", slowPathSnippetInfo);
565 assert slowPathArgument != null;
566 args.addConst("slowPathArgument", slowPathArgument);
567 } else if (snippetInfo == arraycopyExactIntrinsicSnippet || snippetInfo == arraycopyPredictedExactIntrinsicSnippet) {
568 assert componentKind != null;
569 args.addConst("elementKind", componentKind);
570 args.addConst("counter", counters.arraycopyCallCounters.get(componentKind));
571 args.addConst("copiedCounter", counters.arraycopyCallCopiedCounters.get(componentKind));
572 }
573 args.addConst("counters", counters);
574 instantiate(args, arraycopy);
575 }
576
577 public void lower(ArrayCopySlowPathNode arraycopy, LoweringTool tool) {
578 StructuredGraph graph = arraycopy.graph();
579 if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
580 // Can't be lowered yet
581 return;
582 }
583 SnippetInfo snippetInfo = arraycopy.getSnippet();
584 Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage());
585 args.add("nonNullSrc", arraycopy.getSource());
586 args.add("srcPos", arraycopy.getSourcePosition());
587 args.add("nonNullDest", arraycopy.getDestination());
588 args.add("destPos", arraycopy.getDestinationPosition());
589 if (snippetInfo == arraycopyUnrolledWorkSnippet) {
590 args.addConst("length", ((Integer) arraycopy.getArgument()).intValue());
591 args.addConst("elementKind", arraycopy.getElementKind());
592 } else {
593 args.add("length", arraycopy.getLength());
594 }
595 if (snippetInfo == arraycopyPredictedObjectWorkSnippet) {
596 args.add("objectArrayKlass", arraycopy.getPredictedKlass());
597 args.addConst("counter", counters.arraycopyCallCounters.get(JavaKind.Object));
598 args.addConst("copiedCounter", counters.arraycopyCallCopiedCounters.get(JavaKind.Object));
599 args.addConst("counters", counters);
600 }
601 instantiate(args, arraycopy);
602 }
603
604 public void lower(ArrayCopyUnrollNode arraycopy, LoweringTool tool) {
605 StructuredGraph graph = arraycopy.graph();
606 if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
607 // Can't be lowered yet
608 return;
609 }
610 SnippetInfo snippetInfo = arraycopyUnrolledWorkSnippet;
611 Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage());
612 args.add("nonNullSrc", arraycopy.getSource());
613 args.add("srcPos", arraycopy.getSourcePosition());
614 args.add("nonNullDest", arraycopy.getDestination());
615 args.add("destPos", arraycopy.getDestinationPosition());
616 args.addConst("length", arraycopy.getUnrollLength());
617 args.addConst("elementKind", arraycopy.getElementKind());
618 template(args).instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args);
619 }
620
621 /**
622 * Instantiate the snippet template and fix up the FrameState of any Invokes of
623 * System.arraycopy and propagate the captured bci in the ArrayCopySlowPathNode.
624 *
625 * @param args
626 * @param arraycopy
627 */
628 private void instantiate(Arguments args, BasicArrayCopyNode arraycopy) {
629 StructuredGraph graph = arraycopy.graph();
630 SnippetTemplate template = template(args);
631 UnmodifiableEconomicMap<Node, Node> replacements = template.instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args);
632 for (Node originalNode : replacements.getKeys()) {
633 if (originalNode instanceof Invoke) {
634 Invoke invoke = (Invoke) replacements.get(originalNode);
635 assert invoke.asNode().graph() == graph;
636 CallTargetNode call = invoke.callTarget();
637
638 if (!call.targetMethod().equals(originalArraycopy)) {
639 throw new GraalError("unexpected invoke %s in snippet", call.targetMethod());
640 }
641 // Here we need to fix the bci of the invoke
642 InvokeNode newInvoke = graph.add(new InvokeNode(invoke.callTarget(), arraycopy.getBci()));
643 if (arraycopy.stateDuring() != null) {
644 newInvoke.setStateDuring(arraycopy.stateDuring());
645 } else {
646 assert arraycopy.stateAfter() != null;
647 newInvoke.setStateAfter(arraycopy.stateAfter());
648 }
649 graph.replaceFixedWithFixed((InvokeNode) invoke.asNode(), newInvoke);
650 } else if (originalNode instanceof ArrayCopySlowPathNode) {
651 ArrayCopySlowPathNode slowPath = (ArrayCopySlowPathNode) replacements.get(originalNode);
652 assert arraycopy.stateAfter() != null;
653 slowPath.setStateAfter(arraycopy.stateAfter());
654 slowPath.setBci(arraycopy.getBci());
655 }
656 }
657 }
658 }
659 }