parameterUsages = calleeInfo.inline(new Providers(context));
+ canonicalizedNodes.addAll(parameterUsages);
+ counterInliningRuns.increment();
+ Debug.dump(Debug.INFO_LOG_LEVEL, callerGraph, "after %s", calleeInfo);
+
+ Graph.Mark markBeforeCanonicalization = callerGraph.getMark();
+
+ canonicalizer.applyIncremental(callerGraph, context, canonicalizedNodes);
+
+ // process invokes that are possibly created during canonicalization
+ for (Node newNode : callerGraph.getNewNodes(markBeforeCanonicalization)) {
+ if (newNode instanceof Invoke) {
+ callerCallsiteHolder.pushInvoke((Invoke) newNode);
+ }
+ }
+
+ callerCallsiteHolder.computeProbabilities();
+
+ counterInliningPerformed.increment();
+ }
+ } catch (BailoutException bailout) {
+ throw bailout;
+ } catch (AssertionError | RuntimeException e) {
+ throw new GraalError(e).addContext(calleeInfo.toString());
+ } catch (GraalError e) {
+ throw e.addContext(calleeInfo.toString());
+ } catch (Throwable e) {
+ throw Debug.handle(e);
+ }
+ }
+
+ /**
+ *
+ * This method attempts:
+ *
+ * - to inline at the callsite given by
calleeInvocation, where that callsite
+ * belongs to the {@link CallsiteHolderExplorable} at the top of the {@link #graphQueue}
+ * maintained in this class.
+ * - otherwise, to devirtualize the callsite in question.
+ *
+ *
+ * @return true iff inlining was actually performed
+ */
+ private boolean tryToInline(MethodInvocation calleeInvocation, int inliningDepth) {
+ CallsiteHolderExplorable callerCallsiteHolder = (CallsiteHolderExplorable) currentGraph();
+ InlineInfo calleeInfo = calleeInvocation.callee();
+ assert callerCallsiteHolder.containsInvoke(calleeInfo.invoke());
+ counterInliningConsidered.increment();
+
+ if (inliningPolicy.isWorthInlining(context.getReplacements(), calleeInvocation, inliningDepth, true)) {
+ doInline(callerCallsiteHolder, calleeInvocation);
+ return true;
+ }
+
+ if (context.getOptimisticOptimizations().devirtualizeInvokes()) {
+ calleeInfo.tryToDevirtualizeInvoke(new Providers(context));
+ }
+
+ return false;
+ }
+
+ /**
+ * This method picks one of the callsites belonging to the current
+ * {@link CallsiteHolderExplorable}. Provided the callsite qualifies to be analyzed for
+ * inlining, this method prepares a new stack top in {@link InliningData} for such callsite,
+ * which comprises:
+ *
+ * - preparing a summary of feasible targets, ie preparing an {@link InlineInfo}
+ * - based on it, preparing the stack top proper which consists of:
+ *
+ * - one {@link MethodInvocation}
+ * - a {@link CallsiteHolder} for each feasible target
+ *
+ *
+ *
+ *
+ * The thus prepared "stack top" is needed by {@link #moveForward()} to explore the space of
+ * inlining decisions (each decision one of: backtracking, delving, inlining).
+ *
+ *
+ *
+ * The {@link InlineInfo} used to get things rolling is kept around in the
+ * {@link MethodInvocation}, it will be needed in case of inlining, see
+ * {@link InlineInfo#inline(Providers)}
+ *
+ */
+ private void processNextInvoke() {
+ CallsiteHolderExplorable callsiteHolder = (CallsiteHolderExplorable) currentGraph();
+ Invoke invoke = callsiteHolder.popInvoke();
+ InlineInfo info = getInlineInfo(invoke);
+
+ if (info != null) {
+ info.populateInlinableElements(context, currentGraph().graph(), canonicalizer);
+ double invokeProbability = callsiteHolder.invokeProbability(invoke);
+ double invokeRelevance = callsiteHolder.invokeRelevance(invoke);
+ MethodInvocation methodInvocation = new MethodInvocation(info, invokeProbability, invokeRelevance, freshlyInstantiatedArguments(invoke, callsiteHolder.getFixedParams()));
+ pushInvocationAndGraphs(methodInvocation);
+ }
+ }
+
+ /**
+ * Gets the freshly instantiated arguments.
+ *
+ * A freshly instantiated argument is either:
+ *
+ * - an {@link InliningData#isFreshInstantiation(org.graalvm.compiler.nodes.ValueNode)}
+ * - a fixed-param, ie a {@link ParameterNode} receiving a freshly instantiated argument
+ *
+ *
+ *
+ * @return the positions of freshly instantiated arguments in the argument list of the
+ * invoke, or null if no such positions exist.
+ */
+ public static BitSet freshlyInstantiatedArguments(Invoke invoke, Set fixedParams) {
+ assert fixedParams != null;
+ assert paramsAndInvokeAreInSameGraph(invoke, fixedParams);
+ BitSet result = null;
+ int argIdx = 0;
+ for (ValueNode arg : invoke.callTarget().arguments()) {
+ assert arg != null;
+ if (isFreshInstantiation(arg) || fixedParams.contains(arg)) {
+ if (result == null) {
+ result = new BitSet();
+ }
+ result.set(argIdx);
+ }
+ argIdx++;
+ }
+ return result;
+ }
+
+ private static boolean paramsAndInvokeAreInSameGraph(Invoke invoke, Set fixedParams) {
+ if (fixedParams.isEmpty()) {
+ return true;
+ }
+ for (ParameterNode p : fixedParams) {
+ if (p.graph() != invoke.asNode().graph()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public int graphCount() {
+ return graphQueue.size();
+ }
+
+ public boolean hasUnprocessedGraphs() {
+ return !graphQueue.isEmpty();
+ }
+
+ private CallsiteHolder currentGraph() {
+ return graphQueue.peek();
+ }
+
+ private void popGraph() {
+ graphQueue.pop();
+ assert graphQueue.size() <= maxGraphs;
+ }
+
+ private void popGraphs(int count) {
+ assert count >= 0;
+ for (int i = 0; i < count; i++) {
+ graphQueue.pop();
+ }
+ }
+
+ private static final Object[] NO_CONTEXT = {};
+
+ /**
+ * Gets the call hierarchy of this inlining from outer most call to inner most callee.
+ */
+ private Object[] inliningContext() {
+ if (!Debug.isDumpEnabled(Debug.INFO_LOG_LEVEL)) {
+ return NO_CONTEXT;
+ }
+ Object[] result = new Object[graphQueue.size()];
+ int i = 0;
+ for (CallsiteHolder g : graphQueue) {
+ result[i++] = g.method();
+ }
+ return result;
+ }
+
+ private MethodInvocation currentInvocation() {
+ return invocationQueue.peekFirst();
+ }
+
+ private void pushInvocationAndGraphs(MethodInvocation methodInvocation) {
+ invocationQueue.addFirst(methodInvocation);
+ InlineInfo info = methodInvocation.callee();
+ maxGraphs += info.numberOfMethods();
+ assert graphQueue.size() <= maxGraphs;
+ for (int i = 0; i < info.numberOfMethods(); i++) {
+ CallsiteHolder ch = methodInvocation.buildCallsiteHolderForElement(i);
+ assert !contains(ch.graph());
+ graphQueue.push(ch);
+ assert graphQueue.size() <= maxGraphs;
+ }
+ }
+
+ private void popInvocation() {
+ maxGraphs -= invocationQueue.peekFirst().callee().numberOfMethods();
+ assert graphQueue.size() <= maxGraphs;
+ invocationQueue.removeFirst();
+ }
+
+ public int countRecursiveInlining(ResolvedJavaMethod method) {
+ int count = 0;
+ for (CallsiteHolder callsiteHolder : graphQueue) {
+ if (method.equals(callsiteHolder.method())) {
+ count++;
+ }
+ }
+ return count;
+ }
+
+ public int inliningDepth() {
+ assert invocationQueue.size() > 0;
+ return invocationQueue.size() - 1;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder result = new StringBuilder("Invocations: ");
+
+ for (MethodInvocation invocation : invocationQueue) {
+ if (invocation.callee() != null) {
+ result.append(invocation.callee().numberOfMethods());
+ result.append("x ");
+ result.append(invocation.callee().invoke());
+ result.append("; ");
+ }
+ }
+
+ result.append("\nGraphs: ");
+ for (CallsiteHolder graph : graphQueue) {
+ result.append(graph.graph());
+ result.append("; ");
+ }
+
+ return result.toString();
+ }
+
+ /**
+ * Gets a stack trace representing the current inlining stack represented by this object.
+ */
+ public Collection getInvocationStackTrace() {
+ List result = new ArrayList<>();
+ for (CallsiteHolder graph : graphQueue) {
+ result.add(graph.method().asStackTraceElement(0));
+ }
+
+ return result;
+ }
+
+ private boolean contains(StructuredGraph graph) {
+ assert graph != null;
+ for (CallsiteHolder info : graphQueue) {
+ if (info.graph() == graph) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ *
+ * The stack realized by {@link InliningData} grows and shrinks as choices are made among the
+ * alternatives below:
+ *
+ * - not worth inlining: pop stack top, which comprises:
+ *
+ * - pop any remaining graphs not yet delved into
+ * - pop the current invocation
+ *
+ *
+ * - {@link #processNextInvoke() delve} into one of the callsites hosted in the current graph,
+ * such callsite is explored next by {@link #moveForward()}
+ * - {@link #tryToInline(MethodInvocation, int) try to inline}: move past the current graph
+ * (remove it from the topmost element).
+ *
+ * - If that was the last one then {@link #tryToInline(MethodInvocation, int) try to inline}
+ * the callsite under consideration (ie, the "current invocation").
+ * - Whether inlining occurs or not, that callsite is removed from the top of
+ * {@link InliningData} .
+ *
+ *
+ *
+ *
+ *
+ *
+ * Some facts about the alternatives above:
+ *
+ * - the first step amounts to backtracking, the 2nd one to depth-search, and the 3rd one also
+ * involves backtracking (however possibly after inlining).
+ * - the choice of abandon-and-backtrack or delve-into depends on
+ * {@link InliningPolicy#isWorthInlining} and {@link InliningPolicy#continueInlining}.
+ * - the 3rd choice is picked whenever none of the previous choices are made
+ *
+ *
+ *
+ * @return true iff inlining was actually performed
+ */
+ @SuppressWarnings("try")
+ public boolean moveForward() {
+
+ final MethodInvocation currentInvocation = currentInvocation();
+
+ final boolean backtrack = (!currentInvocation.isRoot() && !inliningPolicy.isWorthInlining(context.getReplacements(), currentInvocation, inliningDepth(), false));
+ if (backtrack) {
+ int remainingGraphs = currentInvocation.totalGraphs() - currentInvocation.processedGraphs();
+ assert remainingGraphs > 0;
+ popGraphs(remainingGraphs);
+ popInvocation();
+ return false;
+ }
+
+ final boolean delve = currentGraph().hasRemainingInvokes() && inliningPolicy.continueInlining(currentGraph().graph());
+ if (delve) {
+ processNextInvoke();
+ return false;
+ }
+
+ popGraph();
+ if (currentInvocation.isRoot()) {
+ return false;
+ }
+
+ // try to inline
+ assert currentInvocation.callee().invoke().asNode().isAlive();
+ currentInvocation.incrementProcessedGraphs();
+ if (currentInvocation.processedGraphs() == currentInvocation.totalGraphs()) {
+ /*
+ * "all of currentInvocation's graphs processed" amounts to
+ * "all concrete methods that come into question already had the callees they contain analyzed for inlining"
+ */
+ popInvocation();
+ try (Debug.Scope s = Debug.scope("Inlining", inliningContext())) {
+ if (tryToInline(currentInvocation, inliningDepth() + 1)) {
+ // Report real progress only if we inline into the root graph
+ return currentGraph().graph() == rootGraph;
+ }
+ return false;
+ } catch (Throwable e) {
+ throw Debug.handle(e);
+ }
+ }
+
+ return false;
+ }
+
+ /**
+ * Checks an invariant that {@link #moveForward()} must maintain: "the top invocation records
+ * how many concrete target methods (for it) remain on the {@link #graphQueue}; those targets
+ * 'belong' to the current invocation in question.
+ */
+ private boolean topGraphsForTopInvocation() {
+ if (invocationQueue.isEmpty()) {
+ assert graphQueue.isEmpty();
+ return true;
+ }
+ if (currentInvocation().isRoot()) {
+ if (!graphQueue.isEmpty()) {
+ assert graphQueue.size() == 1;
+ }
+ return true;
+ }
+ final int remainingGraphs = currentInvocation().totalGraphs() - currentInvocation().processedGraphs();
+ final Iterator iter = graphQueue.iterator();
+ for (int i = (remainingGraphs - 1); i >= 0; i--) {
+ if (!iter.hasNext()) {
+ assert false;
+ return false;
+ }
+ CallsiteHolder queuedTargetCH = iter.next();
+ Inlineable targetIE = currentInvocation().callee().inlineableElementAt(i);
+ InlineableGraph targetIG = (InlineableGraph) targetIE;
+ assert queuedTargetCH.method().equals(targetIG.getGraph().method());
+ }
+ return true;
+ }
+
+ /**
+ * This method checks invariants for this class. Named after shorthand for "internal
+ * representation is ok".
+ */
+ public boolean repOK() {
+ assert topGraphsForTopInvocation();
+ return true;
+ }
+}