13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24
25 package org.graalvm.compiler.replacements.arraycopy;
26
27 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
28 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.LIKELY_PROBABILITY;
29 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY;
30 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
31 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
32
33 import java.lang.reflect.Method;
34 import java.util.EnumMap;
35
36 import jdk.internal.vm.compiler.collections.UnmodifiableEconomicMap;
37 import org.graalvm.compiler.api.directives.GraalDirectives;
38 import org.graalvm.compiler.api.replacements.Fold;
39 import org.graalvm.compiler.api.replacements.Fold.InjectedParameter;
40 import org.graalvm.compiler.api.replacements.Snippet;
41 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
42 import org.graalvm.compiler.api.replacements.SnippetReflectionProvider;
43 import org.graalvm.compiler.debug.DebugHandlersFactory;
44 import org.graalvm.compiler.debug.GraalError;
45 import org.graalvm.compiler.graph.Node;
46 import org.graalvm.compiler.nodes.CallTargetNode;
47 import org.graalvm.compiler.nodes.DeoptimizeNode;
48 import org.graalvm.compiler.nodes.InvokeNode;
49 import org.graalvm.compiler.nodes.InvokeWithExceptionNode;
50 import org.graalvm.compiler.nodes.NamedLocationIdentity;
51 import org.graalvm.compiler.nodes.NodeView;
52 import org.graalvm.compiler.nodes.PiNode;
53 import org.graalvm.compiler.nodes.StructuredGraph;
273 }
274 counters.checkSuccessCounter.inc();
275 }
276
277 private void checkArrayTypes(Object nonNullSrc, Object nonNullDest, ArrayCopyTypeCheck arrayTypeCheck) {
278 if (arrayTypeCheck == ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK) {
279 // nothing to do
280 } else if (arrayTypeCheck == ArrayCopyTypeCheck.HUB_BASED_ARRAY_TYPE_CHECK) {
281 Pointer srcHub = loadHub(nonNullSrc);
282 Pointer destHub = loadHub(nonNullDest);
283 if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) {
284 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
285 }
286 } else if (arrayTypeCheck == ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK) {
287 Pointer srcHub = loadHub(nonNullSrc);
288 Pointer destHub = loadHub(nonNullDest);
289 if (probability(SLOW_PATH_PROBABILITY, getReadLayoutHelper(srcHub) != getReadLayoutHelper(destHub))) {
290 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
291 }
292 } else {
293 ReplacementsUtil.staticAssert(false, "unknown array type check");
294 }
295 }
296
297 static class Counters {
298 final SnippetCounter checkSuccessCounter;
299 final SnippetCounter checkAIOOBECounter;
300
301 final SnippetCounter zeroLengthStaticCounter;
302 final SnippetIntegerHistogram lengthHistogram;
303
304 final SnippetCounter systemArraycopyCounter;
305 final SnippetCounter systemArraycopyCopiedCounter;
306
307 final SnippetCounter genericArraycopyDifferentTypeCopiedCounter;
308 final SnippetCounter genericArraycopyDifferentTypeCounter;
309
310 final SnippetCounter objectCheckcastSameTypeCopiedCounter;
311 final SnippetCounter objectCheckcastSameTypeCounter;
312 final SnippetCounter objectCheckcastDifferentTypeCopiedCounter;
313 final SnippetCounter objectCheckcastDifferentTypeCounter;
367
368 private ResolvedJavaMethod originalArraycopy;
369 private final Counters counters;
370
371 public Templates(ArrayCopySnippets receiver, OptionValues options, Iterable<DebugHandlersFactory> factories, Factory factory, Providers providers,
372 SnippetReflectionProvider snippetReflection, TargetDescription target) {
373 super(options, factories, providers, snippetReflection, target);
374 this.counters = new Counters(factory);
375
376 arraycopyGenericSnippet = snippet(receiver, "arraycopyGenericSnippet");
377 arraycopyUnrolledSnippet = snippet(receiver, "arraycopyUnrolledSnippet");
378 arraycopyExactSnippet = snippet(receiver, "arraycopyExactSnippet");
379 arraycopyZeroLengthSnippet = snippet(receiver, "arraycopyZeroLengthSnippet");
380 arraycopyCheckcastSnippet = snippet(receiver, "arraycopyCheckcastSnippet");
381 arraycopyNativeSnippet = snippet(null, "arraycopyNativeSnippet");
382 checkcastArraycopyWithSlowPathWork = snippet(receiver, "checkcastArraycopyWithSlowPathWork");
383 genericArraycopyWithSlowPathWork = snippet(receiver, "genericArraycopyWithSlowPathWork");
384 }
385
386 protected SnippetInfo snippet(ArrayCopySnippets receiver, String methodName) {
387 SnippetInfo info = snippet(ArrayCopySnippets.class, methodName, receiver, LocationIdentity.any());
388 info.setOriginalMethod(originalArraycopy());
389 return info;
390 }
391
392 public void lower(ArrayCopyNode arraycopy, LoweringTool tool) {
393 JavaKind elementKind = selectComponentKind(arraycopy);
394 SnippetInfo snippetInfo;
395 ArrayCopyTypeCheck arrayTypeCheck;
396
397 ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp(NodeView.DEFAULT));
398 ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp(NodeView.DEFAULT));
399 if (!canBeArray(srcType) || !canBeArray(destType)) {
400 // at least one of the objects is definitely not an array - use the native call
401 // right away as the copying will fail anyways
402 snippetInfo = arraycopyNativeSnippet;
403 arrayTypeCheck = ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK;
404 } else {
405 ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType();
406 ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType();
407
408 if (arraycopy.isExact()) {
560 if (arraycopy.stateDuring() != null) {
561 newInvoke.setStateDuring(arraycopy.stateDuring());
562 } else {
563 assert arraycopy.stateAfter() != null : arraycopy;
564 newInvoke.setStateAfter(arraycopy.stateAfter());
565 }
566 } else if (originalNode instanceof InvokeWithExceptionNode) {
567 throw new GraalError("unexpected invoke with exception %s in snippet", originalNode);
568 } else if (originalNode instanceof ArrayCopyWithSlowPathNode) {
569 ArrayCopyWithSlowPathNode slowPath = (ArrayCopyWithSlowPathNode) replacements.get(originalNode);
570 assert arraycopy.stateAfter() != null : arraycopy;
571 assert slowPath.stateAfter() == arraycopy.stateAfter();
572 slowPath.setBci(arraycopy.getBci());
573 }
574 }
575 GraphUtil.killCFG(arraycopy);
576 }
577
578 private ResolvedJavaMethod originalArraycopy() throws GraalError {
579 if (originalArraycopy == null) {
580 Method method;
581 try {
582 method = System.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class);
583 } catch (NoSuchMethodException | SecurityException e) {
584 throw new GraalError(e);
585 }
586 originalArraycopy = providers.getMetaAccess().lookupJavaMethod(method);
587 }
588 return originalArraycopy;
589 }
590 }
591 }
|
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24
25 package org.graalvm.compiler.replacements.arraycopy;
26
27 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FREQUENT_PROBABILITY;
28 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.LIKELY_PROBABILITY;
29 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY;
30 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
31 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
32
33 import java.util.EnumMap;
34
35 import jdk.internal.vm.compiler.collections.UnmodifiableEconomicMap;
36 import org.graalvm.compiler.api.directives.GraalDirectives;
37 import org.graalvm.compiler.api.replacements.Fold;
38 import org.graalvm.compiler.api.replacements.Fold.InjectedParameter;
39 import org.graalvm.compiler.api.replacements.Snippet;
40 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
41 import org.graalvm.compiler.api.replacements.SnippetReflectionProvider;
42 import org.graalvm.compiler.debug.DebugHandlersFactory;
43 import org.graalvm.compiler.debug.GraalError;
44 import org.graalvm.compiler.graph.Node;
45 import org.graalvm.compiler.nodes.CallTargetNode;
46 import org.graalvm.compiler.nodes.DeoptimizeNode;
47 import org.graalvm.compiler.nodes.InvokeNode;
48 import org.graalvm.compiler.nodes.InvokeWithExceptionNode;
49 import org.graalvm.compiler.nodes.NamedLocationIdentity;
50 import org.graalvm.compiler.nodes.NodeView;
51 import org.graalvm.compiler.nodes.PiNode;
52 import org.graalvm.compiler.nodes.StructuredGraph;
272 }
273 counters.checkSuccessCounter.inc();
274 }
275
276 private void checkArrayTypes(Object nonNullSrc, Object nonNullDest, ArrayCopyTypeCheck arrayTypeCheck) {
277 if (arrayTypeCheck == ArrayCopyTypeCheck.NO_ARRAY_TYPE_CHECK) {
278 // nothing to do
279 } else if (arrayTypeCheck == ArrayCopyTypeCheck.HUB_BASED_ARRAY_TYPE_CHECK) {
280 Pointer srcHub = loadHub(nonNullSrc);
281 Pointer destHub = loadHub(nonNullDest);
282 if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) {
283 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
284 }
285 } else if (arrayTypeCheck == ArrayCopyTypeCheck.LAYOUT_HELPER_BASED_ARRAY_TYPE_CHECK) {
286 Pointer srcHub = loadHub(nonNullSrc);
287 Pointer destHub = loadHub(nonNullDest);
288 if (probability(SLOW_PATH_PROBABILITY, getReadLayoutHelper(srcHub) != getReadLayoutHelper(destHub))) {
289 DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
290 }
291 } else {
292 ReplacementsUtil.staticAssert(false, "unknown array type check ", arrayTypeCheck);
293 }
294 }
295
296 static class Counters {
297 final SnippetCounter checkSuccessCounter;
298 final SnippetCounter checkAIOOBECounter;
299
300 final SnippetCounter zeroLengthStaticCounter;
301 final SnippetIntegerHistogram lengthHistogram;
302
303 final SnippetCounter systemArraycopyCounter;
304 final SnippetCounter systemArraycopyCopiedCounter;
305
306 final SnippetCounter genericArraycopyDifferentTypeCopiedCounter;
307 final SnippetCounter genericArraycopyDifferentTypeCounter;
308
309 final SnippetCounter objectCheckcastSameTypeCopiedCounter;
310 final SnippetCounter objectCheckcastSameTypeCounter;
311 final SnippetCounter objectCheckcastDifferentTypeCopiedCounter;
312 final SnippetCounter objectCheckcastDifferentTypeCounter;
366
367 private ResolvedJavaMethod originalArraycopy;
368 private final Counters counters;
369
370 public Templates(ArrayCopySnippets receiver, OptionValues options, Iterable<DebugHandlersFactory> factories, Factory factory, Providers providers,
371 SnippetReflectionProvider snippetReflection, TargetDescription target) {
372 super(options, factories, providers, snippetReflection, target);
373 this.counters = new Counters(factory);
374
375 arraycopyGenericSnippet = snippet(receiver, "arraycopyGenericSnippet");
376 arraycopyUnrolledSnippet = snippet(receiver, "arraycopyUnrolledSnippet");
377 arraycopyExactSnippet = snippet(receiver, "arraycopyExactSnippet");
378 arraycopyZeroLengthSnippet = snippet(receiver, "arraycopyZeroLengthSnippet");
379 arraycopyCheckcastSnippet = snippet(receiver, "arraycopyCheckcastSnippet");
380 arraycopyNativeSnippet = snippet(null, "arraycopyNativeSnippet");
381 checkcastArraycopyWithSlowPathWork = snippet(receiver, "checkcastArraycopyWithSlowPathWork");
382 genericArraycopyWithSlowPathWork = snippet(receiver, "genericArraycopyWithSlowPathWork");
383 }
384
385 protected SnippetInfo snippet(ArrayCopySnippets receiver, String methodName) {
386 SnippetInfo info = snippet(ArrayCopySnippets.class, methodName, originalArraycopy(), receiver, LocationIdentity.any());
387 return info;
388 }
389
390 public void lower(ArrayCopyNode arraycopy, LoweringTool tool) {
391 JavaKind elementKind = selectComponentKind(arraycopy);
392 SnippetInfo snippetInfo;
393 ArrayCopyTypeCheck arrayTypeCheck;
394
395 ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp(NodeView.DEFAULT));
396 ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp(NodeView.DEFAULT));
397 if (!canBeArray(srcType) || !canBeArray(destType)) {
398 // at least one of the objects is definitely not an array - use the native call
399 // right away as the copying will fail anyways
400 snippetInfo = arraycopyNativeSnippet;
401 arrayTypeCheck = ArrayCopyTypeCheck.UNDEFINED_ARRAY_TYPE_CHECK;
402 } else {
403 ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType();
404 ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType();
405
406 if (arraycopy.isExact()) {
558 if (arraycopy.stateDuring() != null) {
559 newInvoke.setStateDuring(arraycopy.stateDuring());
560 } else {
561 assert arraycopy.stateAfter() != null : arraycopy;
562 newInvoke.setStateAfter(arraycopy.stateAfter());
563 }
564 } else if (originalNode instanceof InvokeWithExceptionNode) {
565 throw new GraalError("unexpected invoke with exception %s in snippet", originalNode);
566 } else if (originalNode instanceof ArrayCopyWithSlowPathNode) {
567 ArrayCopyWithSlowPathNode slowPath = (ArrayCopyWithSlowPathNode) replacements.get(originalNode);
568 assert arraycopy.stateAfter() != null : arraycopy;
569 assert slowPath.stateAfter() == arraycopy.stateAfter();
570 slowPath.setBci(arraycopy.getBci());
571 }
572 }
573 GraphUtil.killCFG(arraycopy);
574 }
575
576 private ResolvedJavaMethod originalArraycopy() throws GraalError {
577 if (originalArraycopy == null) {
578 try {
579 originalArraycopy = findMethod(providers.getMetaAccess(), System.class, "arraycopy");
580 } catch (SecurityException e) {
581 throw new GraalError(e);
582 }
583 }
584 return originalArraycopy;
585 }
586 }
587 }
|