--- /dev/null 2017-01-22 10:16:57.869617664 -0800 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/walker/ComputeInliningRelevance.java 2017-02-15 17:08:20.705100842 -0800 @@ -0,0 +1,337 @@ +/* + * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.graalvm.compiler.phases.common.inlining.walker; + +import java.util.ArrayList; +import java.util.Map; +import java.util.function.ToDoubleFunction; + +import org.graalvm.compiler.core.common.SuppressFBWarnings; +import org.graalvm.compiler.graph.Node; +import org.graalvm.compiler.graph.NodeWorkList; +import org.graalvm.compiler.nodes.AbstractBeginNode; +import org.graalvm.compiler.nodes.AbstractMergeNode; +import org.graalvm.compiler.nodes.ControlSinkNode; +import org.graalvm.compiler.nodes.ControlSplitNode; +import org.graalvm.compiler.nodes.EndNode; +import org.graalvm.compiler.nodes.FixedNode; +import org.graalvm.compiler.nodes.FixedWithNextNode; +import org.graalvm.compiler.nodes.Invoke; +import org.graalvm.compiler.nodes.LoopBeginNode; +import org.graalvm.compiler.nodes.LoopEndNode; +import org.graalvm.compiler.nodes.LoopExitNode; +import org.graalvm.compiler.nodes.MergeNode; +import org.graalvm.compiler.nodes.StartNode; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.phases.common.inlining.InliningUtil; + +public class ComputeInliningRelevance { + + private static final double EPSILON = 1d / Integer.MAX_VALUE; + private static final double UNINITIALIZED = -1D; + + private static final int EXPECTED_MIN_INVOKE_COUNT = 3; + private static final int EXPECTED_INVOKE_RATIO = 20; + private static final int EXPECTED_LOOP_COUNT = 3; + + private final StructuredGraph graph; + private final ToDoubleFunction nodeProbabilities; + + /** + * Node relevances are pre-computed for all invokes if the graph contains loops. If there are no + * loops, the computation happens lazily based on {@link #rootScope}. + */ + private Map nodeRelevances; + /** + * This scope is non-null if (and only if) there are no loops in the graph. In this case, the + * root scope is used to compute invoke relevances on the fly. + */ + private Scope rootScope; + + public ComputeInliningRelevance(StructuredGraph graph, ToDoubleFunction nodeProbabilities) { + this.graph = graph; + this.nodeProbabilities = nodeProbabilities; + } + + /** + * Initializes or updates the relevance computation. If there are no loops within the graph, + * most computation happens lazily. + */ + public void compute() { + rootScope = null; + if (!graph.hasLoops()) { + // fast path for the frequent case of no loops + rootScope = new Scope(graph.start(), null); + } else { + if (nodeRelevances == null) { + nodeRelevances = Node.newIdentityMap(EXPECTED_MIN_INVOKE_COUNT + InliningUtil.getNodeCount(graph) / EXPECTED_INVOKE_RATIO); + } + NodeWorkList workList = graph.createNodeWorkList(); + Map loops = Node.newIdentityMap(EXPECTED_LOOP_COUNT); + + loops.put(null, new Scope(graph.start(), null)); + for (LoopBeginNode loopBegin : graph.getNodes(LoopBeginNode.TYPE)) { + createLoopScope(loopBegin, loops); + } + + for (Scope scope : loops.values()) { + scope.process(workList); + } + } + } + + public double getRelevance(Invoke invoke) { + if (rootScope != null) { + return rootScope.computeInvokeRelevance(invoke); + } + assert nodeRelevances != null : "uninitialized relevance"; + return nodeRelevances.get(invoke); + } + + /** + * Determines the parent of the given loop and creates a {@link Scope} object for each one. This + * method will call itself recursively if no {@link Scope} for the parent loop exists. + */ + private Scope createLoopScope(LoopBeginNode loopBegin, Map loops) { + Scope scope = loops.get(loopBegin); + if (scope == null) { + final Scope parent; + // look for the parent scope + FixedNode current = loopBegin.forwardEnd(); + while (true) { + if (current.predecessor() == null) { + if (current instanceof LoopBeginNode) { + // if we reach a LoopBeginNode then we're within this loop + parent = createLoopScope((LoopBeginNode) current, loops); + break; + } else if (current instanceof StartNode) { + // we're within the outermost scope + parent = loops.get(null); + break; + } else { + assert current instanceof MergeNode : current; + // follow any path upwards - it doesn't matter which one + current = ((AbstractMergeNode) current).forwardEndAt(0); + } + } else if (current instanceof LoopExitNode) { + // if we reach a loop exit then we follow this loop and have the same parent + parent = createLoopScope(((LoopExitNode) current).loopBegin(), loops).parent; + break; + } else { + current = (FixedNode) current.predecessor(); + } + } + scope = new Scope(loopBegin, parent); + loops.put(loopBegin, scope); + } + return scope; + } + + /** + * A scope holds information for the contents of one loop or of the root of the method. It does + * not include child loops, i.e., the iteration in {@link #process(NodeWorkList)} explicitly + * excludes the nodes of child loops. + */ + private class Scope { + public final FixedNode start; + public final Scope parent; // can be null for the outermost scope + + /** + * The minimum probability along the most probable path in this scope. Computed lazily. + */ + private double fastPathMinProbability = UNINITIALIZED; + /** + * A measure of how important this scope is within its parent scope. Computed lazily. + */ + private double scopeRelevanceWithinParent = UNINITIALIZED; + + Scope(FixedNode start, Scope parent) { + this.start = start; + this.parent = parent; + } + + @SuppressFBWarnings(value = "FE_FLOATING_POINT_EQUALITY", justification = "comparing against -1D is accurate") + public double getFastPathMinProbability() { + if (fastPathMinProbability == UNINITIALIZED) { + fastPathMinProbability = Math.max(EPSILON, computeFastPathMinProbability(start)); + } + return fastPathMinProbability; + } + + /** + * Computes the ratio between the probabilities of the current scope's entry point and the + * parent scope's fastPathMinProbability. + */ + @SuppressFBWarnings(value = "FE_FLOATING_POINT_EQUALITY", justification = "comparing against -1D is accurate") + public double getScopeRelevanceWithinParent() { + if (scopeRelevanceWithinParent == UNINITIALIZED) { + if (start instanceof LoopBeginNode) { + assert parent != null; + double scopeEntryProbability = nodeProbabilities.applyAsDouble(((LoopBeginNode) start).forwardEnd()); + + scopeRelevanceWithinParent = scopeEntryProbability / parent.getFastPathMinProbability(); + } else { + scopeRelevanceWithinParent = 1D; + } + } + return scopeRelevanceWithinParent; + } + + /** + * Processes all invokes in this scope by starting at the scope's start node and iterating + * all fixed nodes. Child loops are skipped by going from loop entries directly to the loop + * exits. Processing stops at loop exits of the current loop. + */ + public void process(NodeWorkList workList) { + assert !(start instanceof Invoke); + workList.addAll(start.successors()); + + for (Node current : workList) { + assert current.isAlive(); + + if (current instanceof Invoke) { + // process the invoke and queue its successors + nodeRelevances.put((FixedNode) current, computeInvokeRelevance((Invoke) current)); + workList.addAll(current.successors()); + } else if (current instanceof LoopBeginNode) { + // skip child loops by advancing over the loop exits + ((LoopBeginNode) current).loopExits().forEach(exit -> workList.add(exit.next())); + } else if (current instanceof LoopEndNode) { + // nothing to do + } else if (current instanceof LoopExitNode) { + // nothing to do + } else if (current instanceof FixedWithNextNode) { + workList.add(((FixedWithNextNode) current).next()); + } else if (current instanceof EndNode) { + workList.add(((EndNode) current).merge()); + } else if (current instanceof ControlSinkNode) { + // nothing to do + } else if (current instanceof ControlSplitNode) { + workList.addAll(current.successors()); + } else { + assert false : current; + } + } + } + + /** + * The relevance of an invoke is the ratio between the invoke's probability and the current + * scope's fastPathMinProbability, adjusted by scopeRelevanceWithinParent. + */ + public double computeInvokeRelevance(Invoke invoke) { + double invokeProbability = nodeProbabilities.applyAsDouble(invoke.asNode()); + assert !Double.isNaN(invokeProbability); + + double relevance = (invokeProbability / getFastPathMinProbability()) * Math.min(1.0, getScopeRelevanceWithinParent()); + assert !Double.isNaN(relevance) : invoke + ": " + relevance + " / " + invokeProbability + " / " + getFastPathMinProbability() + " / " + getScopeRelevanceWithinParent(); + return relevance; + } + } + + /** + * Computes the minimum probability along the most probable path within the scope. During + * iteration, the method returns immediately once a loop exit is discovered. + */ + private double computeFastPathMinProbability(FixedNode scopeStart) { + ArrayList pathBeginNodes = new ArrayList<>(); + pathBeginNodes.add(scopeStart); + double minPathProbability = nodeProbabilities.applyAsDouble(scopeStart); + boolean isLoopScope = scopeStart instanceof LoopBeginNode; + + do { + Node current = pathBeginNodes.remove(pathBeginNodes.size() - 1); + do { + if (isLoopScope && current instanceof LoopExitNode && ((LoopBeginNode) scopeStart).loopExits().contains((LoopExitNode) current)) { + return minPathProbability; + } else if (current instanceof LoopBeginNode && current != scopeStart) { + current = getMaxProbabilityLoopExit((LoopBeginNode) current, pathBeginNodes); + minPathProbability = getMinPathProbability((FixedNode) current, minPathProbability); + } else if (current instanceof ControlSplitNode) { + current = getMaxProbabilitySux((ControlSplitNode) current, pathBeginNodes); + minPathProbability = getMinPathProbability((FixedNode) current, minPathProbability); + } else { + assert current.successors().count() <= 1; + current = current.successors().first(); + } + } while (current != null); + } while (!pathBeginNodes.isEmpty()); + + return minPathProbability; + } + + private double getMinPathProbability(FixedNode current, double minPathProbability) { + return current == null ? minPathProbability : Math.min(minPathProbability, nodeProbabilities.applyAsDouble(current)); + } + + /** + * Returns the most probable successor. If multiple successors share the maximum probability, + * one is returned and the others are enqueued in pathBeginNodes. + */ + private static Node getMaxProbabilitySux(ControlSplitNode controlSplit, ArrayList pathBeginNodes) { + Node maxSux = null; + double maxProbability = 0.0; + int pathBeginCount = pathBeginNodes.size(); + + for (Node sux : controlSplit.successors()) { + double probability = controlSplit.probability((AbstractBeginNode) sux); + if (probability > maxProbability) { + maxProbability = probability; + maxSux = sux; + truncate(pathBeginNodes, pathBeginCount); + } else if (probability == maxProbability) { + pathBeginNodes.add((FixedNode) sux); + } + } + + return maxSux; + } + + /** + * Returns the most probable loop exit. If multiple successors share the maximum probability, + * one is returned and the others are enqueued in pathBeginNodes. + */ + private Node getMaxProbabilityLoopExit(LoopBeginNode loopBegin, ArrayList pathBeginNodes) { + Node maxSux = null; + double maxProbability = 0.0; + int pathBeginCount = pathBeginNodes.size(); + + for (LoopExitNode sux : loopBegin.loopExits()) { + double probability = nodeProbabilities.applyAsDouble(sux); + if (probability > maxProbability) { + maxProbability = probability; + maxSux = sux; + truncate(pathBeginNodes, pathBeginCount); + } else if (probability == maxProbability) { + pathBeginNodes.add(sux); + } + } + + return maxSux; + } + + private static void truncate(ArrayList pathBeginNodes, int pathBeginCount) { + for (int i = pathBeginNodes.size() - pathBeginCount; i > 0; i--) { + pathBeginNodes.remove(pathBeginNodes.size() - 1); + } + } +}