--- /dev/null 2017-01-22 10:16:57.869617664 -0800 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.phases.common/src/org/graalvm/compiler/phases/common/inlining/info/MultiTypeGuardInlineInfo.java 2017-02-15 17:08:17.155945454 -0800 @@ -0,0 +1,489 @@ +/* + * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.graalvm.compiler.phases.common.inlining.info; + +import static org.graalvm.compiler.core.common.GraalOptions.UseGraalInstrumentation; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.graalvm.compiler.core.common.type.StampFactory; +import org.graalvm.compiler.debug.Debug; +import org.graalvm.compiler.graph.Node; +import org.graalvm.compiler.nodes.AbstractBeginNode; +import org.graalvm.compiler.nodes.AbstractMergeNode; +import org.graalvm.compiler.nodes.BeginNode; +import org.graalvm.compiler.nodes.CallTargetNode.InvokeKind; +import org.graalvm.compiler.nodes.DeoptimizeNode; +import org.graalvm.compiler.nodes.EndNode; +import org.graalvm.compiler.nodes.FixedNode; +import org.graalvm.compiler.nodes.FixedWithNextNode; +import org.graalvm.compiler.nodes.FrameState; +import org.graalvm.compiler.nodes.GuardedValueNode; +import org.graalvm.compiler.nodes.Invoke; +import org.graalvm.compiler.nodes.InvokeWithExceptionNode; +import org.graalvm.compiler.nodes.MergeNode; +import org.graalvm.compiler.nodes.PhiNode; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.ValuePhiNode; +import org.graalvm.compiler.nodes.extended.LoadHubNode; +import org.graalvm.compiler.nodes.java.ExceptionObjectNode; +import org.graalvm.compiler.nodes.java.MethodCallTargetNode; +import org.graalvm.compiler.nodes.java.TypeSwitchNode; +import org.graalvm.compiler.nodes.spi.StampProvider; +import org.graalvm.compiler.nodes.util.GraphUtil; +import org.graalvm.compiler.phases.common.inlining.InliningUtil; +import org.graalvm.compiler.phases.common.inlining.info.elem.Inlineable; +import org.graalvm.compiler.phases.util.Providers; + +import jdk.vm.ci.meta.ConstantReflectionProvider; +import jdk.vm.ci.meta.DeoptimizationAction; +import jdk.vm.ci.meta.DeoptimizationReason; +import jdk.vm.ci.meta.JavaKind; +import jdk.vm.ci.meta.JavaTypeProfile.ProfiledType; +import jdk.vm.ci.meta.ResolvedJavaMethod; +import jdk.vm.ci.meta.ResolvedJavaType; + +/** + * Polymorphic inlining of m methods with n type checks (n ≥ m) in case that the profiling + * information suggests a reasonable amount of different receiver types and different methods. If an + * unknown type is encountered a deoptimization is triggered. + */ +public class MultiTypeGuardInlineInfo extends AbstractInlineInfo { + + private final List concretes; + private final double[] methodProbabilities; + private final double maximumMethodProbability; + private final ArrayList typesToConcretes; + private final ArrayList ptypes; + private final double notRecordedTypeProbability; + private final Inlineable[] inlineableElements; + + public MultiTypeGuardInlineInfo(Invoke invoke, ArrayList concretes, ArrayList ptypes, ArrayList typesToConcretes, double notRecordedTypeProbability) { + super(invoke); + assert concretes.size() > 0 : "must have at least one method"; + assert ptypes.size() == typesToConcretes.size() : "array lengths must match"; + + this.concretes = concretes; + this.ptypes = ptypes; + this.typesToConcretes = typesToConcretes; + this.notRecordedTypeProbability = notRecordedTypeProbability; + this.inlineableElements = new Inlineable[concretes.size()]; + this.methodProbabilities = computeMethodProbabilities(); + this.maximumMethodProbability = maximumMethodProbability(); + assert maximumMethodProbability > 0; + assert assertUniqueTypes(ptypes); + } + + private static boolean assertUniqueTypes(ArrayList ptypes) { + Set set = new HashSet<>(); + for (ProfiledType ptype : ptypes) { + set.add(ptype.getType()); + } + return set.size() == ptypes.size(); + } + + private double[] computeMethodProbabilities() { + double[] result = new double[concretes.size()]; + for (int i = 0; i < typesToConcretes.size(); i++) { + int concrete = typesToConcretes.get(i); + double probability = ptypes.get(i).getProbability(); + result[concrete] += probability; + } + return result; + } + + private double maximumMethodProbability() { + double max = 0; + for (int i = 0; i < methodProbabilities.length; i++) { + max = Math.max(max, methodProbabilities[i]); + } + return max; + } + + @Override + public int numberOfMethods() { + return concretes.size(); + } + + @Override + public ResolvedJavaMethod methodAt(int index) { + assert index >= 0 && index < concretes.size(); + return concretes.get(index); + } + + @Override + public Inlineable inlineableElementAt(int index) { + assert index >= 0 && index < concretes.size(); + return inlineableElements[index]; + } + + @Override + public double probabilityAt(int index) { + return methodProbabilities[index]; + } + + @Override + public double relevanceAt(int index) { + return probabilityAt(index) / maximumMethodProbability; + } + + @Override + public void setInlinableElement(int index, Inlineable inlineableElement) { + assert index >= 0 && index < concretes.size(); + inlineableElements[index] = inlineableElement; + } + + @Override + public Collection inline(Providers providers) { + if (hasSingleMethod()) { + return inlineSingleMethod(graph(), providers.getStampProvider(), providers.getConstantReflection()); + } else { + return inlineMultipleMethods(graph(), providers); + } + } + + @Override + public boolean shouldInline() { + for (ResolvedJavaMethod method : concretes) { + if (method.shouldBeInlined()) { + return true; + } + } + return false; + } + + private boolean hasSingleMethod() { + return concretes.size() == 1 && !shouldFallbackToInvoke(); + } + + private boolean shouldFallbackToInvoke() { + return notRecordedTypeProbability > 0; + } + + private Collection inlineMultipleMethods(StructuredGraph graph, Providers providers) { + int numberOfMethods = concretes.size(); + FixedNode continuation = invoke.next(); + + // setup merge and phi nodes for results and exceptions + AbstractMergeNode returnMerge = graph.add(new MergeNode()); + returnMerge.setStateAfter(invoke.stateAfter()); + + PhiNode returnValuePhi = null; + if (invoke.asNode().getStackKind() != JavaKind.Void) { + returnValuePhi = graph.addWithoutUnique(new ValuePhiNode(invoke.asNode().stamp().unrestricted(), returnMerge)); + } + + AbstractMergeNode exceptionMerge = null; + PhiNode exceptionObjectPhi = null; + if (invoke instanceof InvokeWithExceptionNode) { + InvokeWithExceptionNode invokeWithException = (InvokeWithExceptionNode) invoke; + ExceptionObjectNode exceptionEdge = (ExceptionObjectNode) invokeWithException.exceptionEdge(); + + exceptionMerge = graph.add(new MergeNode()); + + FixedNode exceptionSux = exceptionEdge.next(); + graph.addBeforeFixed(exceptionSux, exceptionMerge); + exceptionObjectPhi = graph.addWithoutUnique(new ValuePhiNode(StampFactory.forKind(JavaKind.Object), exceptionMerge)); + exceptionMerge.setStateAfter(exceptionEdge.stateAfter().duplicateModified(invoke.stateAfter().bci, true, JavaKind.Object, new JavaKind[]{JavaKind.Object}, + new ValueNode[]{exceptionObjectPhi})); + } + + // create one separate block for each invoked method + AbstractBeginNode[] successors = new AbstractBeginNode[numberOfMethods + 1]; + for (int i = 0; i < numberOfMethods; i++) { + successors[i] = createInvocationBlock(graph, invoke, returnMerge, returnValuePhi, exceptionMerge, exceptionObjectPhi, true); + } + + // create the successor for an unknown type + FixedNode unknownTypeSux; + if (shouldFallbackToInvoke()) { + unknownTypeSux = createInvocationBlock(graph, invoke, returnMerge, returnValuePhi, exceptionMerge, exceptionObjectPhi, false); + } else { + unknownTypeSux = graph.add(new DeoptimizeNode(DeoptimizationAction.InvalidateReprofile, DeoptimizationReason.TypeCheckedInliningViolated)); + } + successors[successors.length - 1] = BeginNode.begin(unknownTypeSux); + + // replace the invoke exception edge + if (invoke instanceof InvokeWithExceptionNode) { + InvokeWithExceptionNode invokeWithExceptionNode = (InvokeWithExceptionNode) invoke; + ExceptionObjectNode exceptionEdge = (ExceptionObjectNode) invokeWithExceptionNode.exceptionEdge(); + exceptionEdge.replaceAtUsages(exceptionObjectPhi); + exceptionEdge.setNext(null); + GraphUtil.killCFG(invokeWithExceptionNode.exceptionEdge()); + } + + assert invoke.asNode().isAlive(); + + // replace the invoke with a switch on the type of the actual receiver + boolean methodDispatch = createDispatchOnTypeBeforeInvoke(graph, successors, false, providers.getStampProvider(), providers.getConstantReflection()); + + assert invoke.next() == continuation; + invoke.setNext(null); + returnMerge.setNext(continuation); + if (UseGraalInstrumentation.getValue()) { + InliningUtil.detachInstrumentation(invoke); + } + if (returnValuePhi != null) { + invoke.asNode().replaceAtUsages(returnValuePhi); + } + invoke.asNode().safeDelete(); + + ArrayList replacementNodes = new ArrayList<>(); + + // prepare the anchors for the invokes + for (int i = 0; i < numberOfMethods; i++) { + AbstractBeginNode node = successors[i]; + Invoke invokeForInlining = (Invoke) node.next(); + + ResolvedJavaType commonType; + if (methodDispatch) { + commonType = concretes.get(i).getDeclaringClass(); + } else { + commonType = getLeastCommonType(i); + } + + ValueNode receiver = ((MethodCallTargetNode) invokeForInlining.callTarget()).receiver(); + boolean exact = (getTypeCount(i) == 1 && !methodDispatch); + GuardedValueNode anchoredReceiver = InliningUtil.createAnchoredReceiver(graph, node, commonType, receiver, exact); + invokeForInlining.callTarget().replaceFirstInput(receiver, anchoredReceiver); + + assert !anchoredReceiver.isDeleted() : anchoredReceiver; + replacementNodes.add(anchoredReceiver); + } + if (shouldFallbackToInvoke()) { + replacementNodes.add(null); + } + + Collection canonicalizeNodes = new ArrayList<>(); + // do the actual inlining for every invoke + for (int i = 0; i < numberOfMethods; i++) { + Invoke invokeForInlining = (Invoke) successors[i].next(); + canonicalizeNodes.addAll(inline(invokeForInlining, methodAt(i), inlineableElementAt(i), false)); + } + if (returnValuePhi != null) { + canonicalizeNodes.add(returnValuePhi); + } + return canonicalizeNodes; + } + + private int getTypeCount(int concreteMethodIndex) { + int count = 0; + for (int i = 0; i < typesToConcretes.size(); i++) { + if (typesToConcretes.get(i) == concreteMethodIndex) { + count++; + } + } + return count; + } + + private ResolvedJavaType getLeastCommonType(int concreteMethodIndex) { + ResolvedJavaType commonType = null; + for (int i = 0; i < typesToConcretes.size(); i++) { + if (typesToConcretes.get(i) == concreteMethodIndex) { + if (commonType == null) { + commonType = ptypes.get(i).getType(); + } else { + commonType = commonType.findLeastCommonAncestor(ptypes.get(i).getType()); + } + } + } + assert commonType != null; + return commonType; + } + + private ResolvedJavaType getLeastCommonType() { + ResolvedJavaType result = getLeastCommonType(0); + for (int i = 1; i < concretes.size(); i++) { + result = result.findLeastCommonAncestor(getLeastCommonType(i)); + } + return result; + } + + private Collection inlineSingleMethod(StructuredGraph graph, StampProvider stampProvider, ConstantReflectionProvider constantReflection) { + assert concretes.size() == 1 && inlineableElements.length == 1 && ptypes.size() > 1 && !shouldFallbackToInvoke() && notRecordedTypeProbability == 0; + + AbstractBeginNode calleeEntryNode = graph.add(new BeginNode()); + + AbstractBeginNode unknownTypeSux = createUnknownTypeSuccessor(graph); + AbstractBeginNode[] successors = new AbstractBeginNode[]{calleeEntryNode, unknownTypeSux}; + createDispatchOnTypeBeforeInvoke(graph, successors, false, stampProvider, constantReflection); + + calleeEntryNode.setNext(invoke.asNode()); + + return inline(invoke, methodAt(0), inlineableElementAt(0), false); + } + + private boolean createDispatchOnTypeBeforeInvoke(StructuredGraph graph, AbstractBeginNode[] successors, boolean invokeIsOnlySuccessor, StampProvider stampProvider, + ConstantReflectionProvider constantReflection) { + assert ptypes.size() >= 1; + ValueNode nonNullReceiver = InliningUtil.nonNullReceiver(invoke); + LoadHubNode hub = graph.unique(new LoadHubNode(stampProvider, nonNullReceiver)); + + Debug.log("Type switch with %d types", concretes.size()); + + ResolvedJavaType[] keys = new ResolvedJavaType[ptypes.size()]; + double[] keyProbabilities = new double[ptypes.size() + 1]; + int[] keySuccessors = new int[ptypes.size() + 1]; + double totalProbability = notRecordedTypeProbability; + for (int i = 0; i < ptypes.size(); i++) { + keys[i] = ptypes.get(i).getType(); + keyProbabilities[i] = ptypes.get(i).getProbability(); + totalProbability += keyProbabilities[i]; + keySuccessors[i] = invokeIsOnlySuccessor ? 0 : typesToConcretes.get(i); + assert keySuccessors[i] < successors.length - 1 : "last successor is the unknownTypeSux"; + } + keyProbabilities[keyProbabilities.length - 1] = notRecordedTypeProbability; + keySuccessors[keySuccessors.length - 1] = successors.length - 1; + + // Normalize the probabilities. + for (int i = 0; i < keyProbabilities.length; i++) { + keyProbabilities[i] /= totalProbability; + } + + TypeSwitchNode typeSwitch = graph.add(new TypeSwitchNode(hub, successors, keys, keyProbabilities, keySuccessors, constantReflection)); + FixedWithNextNode pred = (FixedWithNextNode) invoke.asNode().predecessor(); + pred.setNext(typeSwitch); + return false; + } + + private static AbstractBeginNode createInvocationBlock(StructuredGraph graph, Invoke invoke, AbstractMergeNode returnMerge, PhiNode returnValuePhi, AbstractMergeNode exceptionMerge, + PhiNode exceptionObjectPhi, boolean useForInlining) { + Invoke duplicatedInvoke = duplicateInvokeForInlining(graph, invoke, exceptionMerge, exceptionObjectPhi, useForInlining); + AbstractBeginNode calleeEntryNode = graph.add(new BeginNode()); + calleeEntryNode.setNext(duplicatedInvoke.asNode()); + + EndNode endNode = graph.add(new EndNode()); + duplicatedInvoke.setNext(endNode); + returnMerge.addForwardEnd(endNode); + + if (returnValuePhi != null) { + returnValuePhi.addInput(duplicatedInvoke.asNode()); + } + return calleeEntryNode; + } + + private static Invoke duplicateInvokeForInlining(StructuredGraph graph, Invoke invoke, AbstractMergeNode exceptionMerge, PhiNode exceptionObjectPhi, boolean useForInlining) { + Invoke result = (Invoke) invoke.asNode().copyWithInputs(); + Node callTarget = result.callTarget().copyWithInputs(); + result.asNode().replaceFirstInput(result.callTarget(), callTarget); + result.setUseForInlining(useForInlining); + + JavaKind kind = invoke.asNode().getStackKind(); + if (kind != JavaKind.Void) { + FrameState stateAfter = invoke.stateAfter(); + stateAfter = stateAfter.duplicate(stateAfter.bci); + stateAfter.replaceFirstInput(invoke.asNode(), result.asNode()); + result.setStateAfter(stateAfter); + } + + if (invoke instanceof InvokeWithExceptionNode) { + assert exceptionMerge != null && exceptionObjectPhi != null; + + InvokeWithExceptionNode invokeWithException = (InvokeWithExceptionNode) invoke; + ExceptionObjectNode exceptionEdge = (ExceptionObjectNode) invokeWithException.exceptionEdge(); + FrameState stateAfterException = exceptionEdge.stateAfter(); + + ExceptionObjectNode newExceptionEdge = (ExceptionObjectNode) exceptionEdge.copyWithInputs(); + // set new state (pop old exception object, push new one) + newExceptionEdge.setStateAfter(stateAfterException.duplicateModified(JavaKind.Object, JavaKind.Object, newExceptionEdge)); + + EndNode endNode = graph.add(new EndNode()); + newExceptionEdge.setNext(endNode); + exceptionMerge.addForwardEnd(endNode); + exceptionObjectPhi.addInput(newExceptionEdge); + + ((InvokeWithExceptionNode) result).setExceptionEdge(newExceptionEdge); + } + return result; + } + + @Override + public void tryToDevirtualizeInvoke(Providers providers) { + if (hasSingleMethod()) { + devirtualizeWithTypeSwitch(graph(), InvokeKind.Special, concretes.get(0), providers.getStampProvider(), providers.getConstantReflection()); + } else { + tryToDevirtualizeMultipleMethods(graph(), providers.getStampProvider(), providers.getConstantReflection()); + } + } + + private void tryToDevirtualizeMultipleMethods(StructuredGraph graph, StampProvider stampProvider, ConstantReflectionProvider constantReflection) { + MethodCallTargetNode methodCallTarget = (MethodCallTargetNode) invoke.callTarget(); + if (methodCallTarget.invokeKind() == InvokeKind.Interface) { + ResolvedJavaMethod targetMethod = methodCallTarget.targetMethod(); + ResolvedJavaType leastCommonType = getLeastCommonType(); + ResolvedJavaType contextType = invoke.getContextType(); + // check if we have a common base type that implements the interface -> in that case + // we have a vtable entry for the interface method and can use a less expensive + // virtual call + if (!leastCommonType.isInterface() && targetMethod.getDeclaringClass().isAssignableFrom(leastCommonType)) { + ResolvedJavaMethod baseClassTargetMethod = leastCommonType.resolveConcreteMethod(targetMethod, contextType); + if (baseClassTargetMethod != null) { + devirtualizeWithTypeSwitch(graph, InvokeKind.Virtual, leastCommonType.resolveConcreteMethod(targetMethod, contextType), stampProvider, constantReflection); + } + } + } + } + + private void devirtualizeWithTypeSwitch(StructuredGraph graph, InvokeKind kind, ResolvedJavaMethod target, StampProvider stampProvider, ConstantReflectionProvider constantReflection) { + AbstractBeginNode invocationEntry = graph.add(new BeginNode()); + AbstractBeginNode unknownTypeSux = createUnknownTypeSuccessor(graph); + AbstractBeginNode[] successors = new AbstractBeginNode[]{invocationEntry, unknownTypeSux}; + createDispatchOnTypeBeforeInvoke(graph, successors, true, stampProvider, constantReflection); + + invocationEntry.setNext(invoke.asNode()); + ValueNode receiver = ((MethodCallTargetNode) invoke.callTarget()).receiver(); + GuardedValueNode anchoredReceiver = InliningUtil.createAnchoredReceiver(graph, invocationEntry, target.getDeclaringClass(), receiver, false); + invoke.callTarget().replaceFirstInput(receiver, anchoredReceiver); + InliningUtil.replaceInvokeCallTarget(invoke, graph, kind, target); + } + + private static AbstractBeginNode createUnknownTypeSuccessor(StructuredGraph graph) { + return BeginNode.begin(graph.add(new DeoptimizeNode(DeoptimizationAction.InvalidateReprofile, DeoptimizationReason.TypeCheckedInliningViolated))); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(shouldFallbackToInvoke() ? "megamorphic" : "polymorphic"); + builder.append(", "); + builder.append(concretes.size()); + builder.append(" methods [ "); + for (int i = 0; i < concretes.size(); i++) { + builder.append(concretes.get(i).format(" %H.%n(%p):%r")); + } + builder.append(" ], "); + builder.append(ptypes.size()); + builder.append(" type checks [ "); + for (int i = 0; i < ptypes.size(); i++) { + builder.append(" "); + builder.append(ptypes.get(i).getType().getName()); + builder.append(ptypes.get(i).getProbability()); + } + builder.append(" ]"); + return builder.toString(); + } +}