26 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.KLASS_SUPER_CHECK_OFFSET_LOCATION;
27 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION;
28 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset;
29 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayClassElementOffset;
30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayIndexScale;
31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperElementTypePrimitiveInPlace;
32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadHub;
33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.superCheckOffsetOffset;
35 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
36 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
37 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
38
39 import java.lang.reflect.Method;
40 import java.util.EnumMap;
41
42 import org.graalvm.compiler.api.directives.GraalDirectives;
43 import org.graalvm.compiler.api.replacements.Fold;
44 import org.graalvm.compiler.api.replacements.Snippet;
45 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
46 import org.graalvm.compiler.debug.GraalError;
47 import org.graalvm.compiler.graph.Node;
48 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
49 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
50 import org.graalvm.compiler.hotspot.word.KlassPointer;
51 import org.graalvm.compiler.nodes.CallTargetNode;
52 import org.graalvm.compiler.nodes.ConstantNode;
53 import org.graalvm.compiler.nodes.DeoptimizeNode;
54 import org.graalvm.compiler.nodes.Invoke;
55 import org.graalvm.compiler.nodes.InvokeNode;
56 import org.graalvm.compiler.nodes.NamedLocationIdentity;
57 import org.graalvm.compiler.nodes.PiNode;
58 import org.graalvm.compiler.nodes.StructuredGraph;
59 import org.graalvm.compiler.nodes.ValueNode;
60 import org.graalvm.compiler.nodes.extended.RawLoadNode;
61 import org.graalvm.compiler.nodes.extended.RawStoreNode;
62 import org.graalvm.compiler.nodes.java.ArrayLengthNode;
63 import org.graalvm.compiler.nodes.spi.LoweringTool;
64 import org.graalvm.compiler.nodes.type.StampTool;
65 import org.graalvm.compiler.nodes.util.GraphUtil;
394 createArraycopyCounter(JavaKind.Char, counters, copiedCounters);
395 createArraycopyCounter(JavaKind.Short, counters, copiedCounters);
396 createArraycopyCounter(JavaKind.Int, counters, copiedCounters);
397 createArraycopyCounter(JavaKind.Long, counters, copiedCounters);
398 createArraycopyCounter(JavaKind.Float, counters, copiedCounters);
399 createArraycopyCounter(JavaKind.Double, counters, copiedCounters);
400 createArraycopyCounter(JavaKind.Object, counters, copiedCounters);
401 }
402
403 void createArraycopyCounter(JavaKind kind, Group counters, Group copiedCounters) {
404 arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays"));
405 arraycopyCounters.put(kind, new SnippetCounter(counters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays"));
406
407 arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays"));
408 arraycopyCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays"));
409 }
410 }
411
412 public static class Templates extends SnippetTemplate.AbstractTemplates {
413
414 public Templates(OptionValues options, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target) {
415 super(options, providers, providers.getSnippetReflection(), target);
416 this.counters = new Counters(factory);
417 }
418
419 private ResolvedJavaMethod originalArraycopy() throws GraalError {
420 if (originalArraycopy == null) {
421 Method method;
422 try {
423 method = System.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class);
424 } catch (NoSuchMethodException | SecurityException e) {
425 throw new GraalError(e);
426 }
427 originalArraycopy = providers.getMetaAccess().lookupJavaMethod(method);
428 }
429 return originalArraycopy;
430 }
431
432 private ResolvedJavaMethod originalArraycopy;
433
434 private final SnippetInfo checkcastArraycopyWorkSnippet = snippet("checkcastArraycopyWork");
435 private final SnippetInfo arraycopyGenericSnippet = snippet("arraycopyGeneric");
600 args.addConst("copiedCounter", counters.arraycopyCallCopiedCounters.get(JavaKind.Object));
601 args.addConst("counters", counters);
602 }
603 instantiate(args, arraycopy);
604 }
605
606 public void lower(ArrayCopyUnrollNode arraycopy, LoweringTool tool) {
607 StructuredGraph graph = arraycopy.graph();
608 if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
609 // Can't be lowered yet
610 return;
611 }
612 SnippetInfo snippetInfo = arraycopyUnrolledWorkSnippet;
613 Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage());
614 args.add("nonNullSrc", arraycopy.getSource());
615 args.add("srcPos", arraycopy.getSourcePosition());
616 args.add("nonNullDest", arraycopy.getDestination());
617 args.add("destPos", arraycopy.getDestinationPosition());
618 args.addConst("length", arraycopy.getUnrollLength());
619 args.addConst("elementKind", arraycopy.getElementKind());
620 template(args).instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args);
621 }
622
623 /**
624 * Instantiate the snippet template and fix up the FrameState of any Invokes of
625 * System.arraycopy and propagate the captured bci in the ArrayCopySlowPathNode.
626 *
627 * @param args
628 * @param arraycopy
629 */
630 private void instantiate(Arguments args, BasicArrayCopyNode arraycopy) {
631 StructuredGraph graph = arraycopy.graph();
632 SnippetTemplate template = template(args);
633 UnmodifiableEconomicMap<Node, Node> replacements = template.instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args, false);
634 for (Node originalNode : replacements.getKeys()) {
635 if (originalNode instanceof Invoke) {
636 Invoke invoke = (Invoke) replacements.get(originalNode);
637 assert invoke.asNode().graph() == graph;
638 CallTargetNode call = invoke.callTarget();
639
640 if (!call.targetMethod().equals(originalArraycopy)) {
641 throw new GraalError("unexpected invoke %s in snippet", call.targetMethod());
642 }
643 // Here we need to fix the bci of the invoke
644 InvokeNode newInvoke = graph.add(new InvokeNode(invoke.callTarget(), arraycopy.getBci()));
645 if (arraycopy.stateDuring() != null) {
646 newInvoke.setStateDuring(arraycopy.stateDuring());
647 } else {
648 assert arraycopy.stateAfter() != null : arraycopy;
649 newInvoke.setStateAfter(arraycopy.stateAfter());
650 }
651 graph.replaceFixedWithFixed((InvokeNode) invoke.asNode(), newInvoke);
652 } else if (originalNode instanceof ArrayCopySlowPathNode) {
|
26 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.KLASS_SUPER_CHECK_OFFSET_LOCATION;
27 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION;
28 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayBaseOffset;
29 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayClassElementOffset;
30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.arrayIndexScale;
31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.layoutHelperElementTypePrimitiveInPlace;
32 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.loadHub;
33 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.readLayoutHelper;
34 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.superCheckOffsetOffset;
35 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.FAST_PATH_PROBABILITY;
36 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
37 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
38
39 import java.lang.reflect.Method;
40 import java.util.EnumMap;
41
42 import org.graalvm.compiler.api.directives.GraalDirectives;
43 import org.graalvm.compiler.api.replacements.Fold;
44 import org.graalvm.compiler.api.replacements.Snippet;
45 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
46 import org.graalvm.compiler.debug.DebugHandlersFactory;
47 import org.graalvm.compiler.debug.GraalError;
48 import org.graalvm.compiler.graph.Node;
49 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
50 import org.graalvm.compiler.hotspot.nodes.type.KlassPointerStamp;
51 import org.graalvm.compiler.hotspot.word.KlassPointer;
52 import org.graalvm.compiler.nodes.CallTargetNode;
53 import org.graalvm.compiler.nodes.ConstantNode;
54 import org.graalvm.compiler.nodes.DeoptimizeNode;
55 import org.graalvm.compiler.nodes.Invoke;
56 import org.graalvm.compiler.nodes.InvokeNode;
57 import org.graalvm.compiler.nodes.NamedLocationIdentity;
58 import org.graalvm.compiler.nodes.PiNode;
59 import org.graalvm.compiler.nodes.StructuredGraph;
60 import org.graalvm.compiler.nodes.ValueNode;
61 import org.graalvm.compiler.nodes.extended.RawLoadNode;
62 import org.graalvm.compiler.nodes.extended.RawStoreNode;
63 import org.graalvm.compiler.nodes.java.ArrayLengthNode;
64 import org.graalvm.compiler.nodes.spi.LoweringTool;
65 import org.graalvm.compiler.nodes.type.StampTool;
66 import org.graalvm.compiler.nodes.util.GraphUtil;
395 createArraycopyCounter(JavaKind.Char, counters, copiedCounters);
396 createArraycopyCounter(JavaKind.Short, counters, copiedCounters);
397 createArraycopyCounter(JavaKind.Int, counters, copiedCounters);
398 createArraycopyCounter(JavaKind.Long, counters, copiedCounters);
399 createArraycopyCounter(JavaKind.Float, counters, copiedCounters);
400 createArraycopyCounter(JavaKind.Double, counters, copiedCounters);
401 createArraycopyCounter(JavaKind.Object, counters, copiedCounters);
402 }
403
404 void createArraycopyCounter(JavaKind kind, Group counters, Group copiedCounters) {
405 arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays"));
406 arraycopyCounters.put(kind, new SnippetCounter(counters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays"));
407
408 arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays"));
409 arraycopyCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays"));
410 }
411 }
412
413 public static class Templates extends SnippetTemplate.AbstractTemplates {
414
415 public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, SnippetCounter.Group.Factory factory, HotSpotProviders providers, TargetDescription target) {
416 super(options, factories, providers, providers.getSnippetReflection(), target);
417 this.counters = new Counters(factory);
418 }
419
420 private ResolvedJavaMethod originalArraycopy() throws GraalError {
421 if (originalArraycopy == null) {
422 Method method;
423 try {
424 method = System.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class);
425 } catch (NoSuchMethodException | SecurityException e) {
426 throw new GraalError(e);
427 }
428 originalArraycopy = providers.getMetaAccess().lookupJavaMethod(method);
429 }
430 return originalArraycopy;
431 }
432
433 private ResolvedJavaMethod originalArraycopy;
434
435 private final SnippetInfo checkcastArraycopyWorkSnippet = snippet("checkcastArraycopyWork");
436 private final SnippetInfo arraycopyGenericSnippet = snippet("arraycopyGeneric");
601 args.addConst("copiedCounter", counters.arraycopyCallCopiedCounters.get(JavaKind.Object));
602 args.addConst("counters", counters);
603 }
604 instantiate(args, arraycopy);
605 }
606
607 public void lower(ArrayCopyUnrollNode arraycopy, LoweringTool tool) {
608 StructuredGraph graph = arraycopy.graph();
609 if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
610 // Can't be lowered yet
611 return;
612 }
613 SnippetInfo snippetInfo = arraycopyUnrolledWorkSnippet;
614 Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage());
615 args.add("nonNullSrc", arraycopy.getSource());
616 args.add("srcPos", arraycopy.getSourcePosition());
617 args.add("nonNullDest", arraycopy.getDestination());
618 args.add("destPos", arraycopy.getDestinationPosition());
619 args.addConst("length", arraycopy.getUnrollLength());
620 args.addConst("elementKind", arraycopy.getElementKind());
621 template(graph.getDebug(), args).instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args);
622 }
623
624 /**
625 * Instantiate the snippet template and fix up the FrameState of any Invokes of
626 * System.arraycopy and propagate the captured bci in the ArrayCopySlowPathNode.
627 *
628 * @param args
629 * @param arraycopy
630 */
631 private void instantiate(Arguments args, BasicArrayCopyNode arraycopy) {
632 StructuredGraph graph = arraycopy.graph();
633 SnippetTemplate template = template(graph.getDebug(), args);
634 UnmodifiableEconomicMap<Node, Node> replacements = template.instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args, false);
635 for (Node originalNode : replacements.getKeys()) {
636 if (originalNode instanceof Invoke) {
637 Invoke invoke = (Invoke) replacements.get(originalNode);
638 assert invoke.asNode().graph() == graph;
639 CallTargetNode call = invoke.callTarget();
640
641 if (!call.targetMethod().equals(originalArraycopy)) {
642 throw new GraalError("unexpected invoke %s in snippet", call.targetMethod());
643 }
644 // Here we need to fix the bci of the invoke
645 InvokeNode newInvoke = graph.add(new InvokeNode(invoke.callTarget(), arraycopy.getBci()));
646 if (arraycopy.stateDuring() != null) {
647 newInvoke.setStateDuring(arraycopy.stateDuring());
648 } else {
649 assert arraycopy.stateAfter() != null : arraycopy;
650 newInvoke.setStateAfter(arraycopy.stateAfter());
651 }
652 graph.replaceFixedWithFixed((InvokeNode) invoke.asNode(), newInvoke);
653 } else if (originalNode instanceof ArrayCopySlowPathNode) {
|