--- /dev/null 2017-01-22 10:16:57.869617664 -0800 +++ new/src/jdk.internal.vm.compiler/share/classes/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/PEGraphDecoder.java 2017-02-15 17:09:08.476192338 -0800 @@ -0,0 +1,901 @@ +/* + * Copyright (c) 2015, 2016, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.graalvm.compiler.replacements; + +import static org.graalvm.compiler.debug.GraalError.unimplemented; +import static org.graalvm.compiler.java.BytecodeParserOptions.DumpDuringGraphBuilding; +import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_IGNORED; +import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_IGNORED; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.graalvm.compiler.bytecode.Bytecode; +import org.graalvm.compiler.bytecode.BytecodeProvider; +import org.graalvm.compiler.common.PermanentBailoutException; +import org.graalvm.compiler.core.common.cfg.CFGVerifier; +import org.graalvm.compiler.core.common.spi.ConstantFieldProvider; +import org.graalvm.compiler.core.common.type.StampFactory; +import org.graalvm.compiler.core.common.type.StampPair; +import org.graalvm.compiler.debug.Debug; +import org.graalvm.compiler.debug.DebugCloseable; +import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.graph.Node; +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.graph.NodeSourcePosition; +import org.graalvm.compiler.graph.spi.Canonicalizable; +import org.graalvm.compiler.java.GraphBuilderPhase; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.AbstractBeginNode; +import org.graalvm.compiler.nodes.AbstractMergeNode; +import org.graalvm.compiler.nodes.CallTargetNode; +import org.graalvm.compiler.nodes.CallTargetNode.InvokeKind; +import org.graalvm.compiler.nodes.DeoptimizeNode; +import org.graalvm.compiler.nodes.EncodedGraph; +import org.graalvm.compiler.nodes.FixedNode; +import org.graalvm.compiler.nodes.FixedWithNextNode; +import org.graalvm.compiler.nodes.FrameState; +import org.graalvm.compiler.nodes.IfNode; +import org.graalvm.compiler.nodes.Invoke; +import org.graalvm.compiler.nodes.InvokeWithExceptionNode; +import org.graalvm.compiler.nodes.MergeNode; +import org.graalvm.compiler.nodes.ParameterNode; +import org.graalvm.compiler.nodes.ReturnNode; +import org.graalvm.compiler.nodes.SimplifyingGraphDecoder; +import org.graalvm.compiler.nodes.StateSplit; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.UnwindNode; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.cfg.ControlFlowGraph; +import org.graalvm.compiler.nodes.extended.ForeignCallNode; +import org.graalvm.compiler.nodes.extended.IntegerSwitchNode; +import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext; +import org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin; +import org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin.InlineInfo; +import org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext; +import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin; +import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins; +import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins.InvocationPluginReceiver; +import org.graalvm.compiler.nodes.graphbuilderconf.LoopExplosionPlugin; +import org.graalvm.compiler.nodes.graphbuilderconf.LoopExplosionPlugin.LoopExplosionKind; +import org.graalvm.compiler.nodes.graphbuilderconf.ParameterPlugin; +import org.graalvm.compiler.nodes.java.MethodCallTargetNode; +import org.graalvm.compiler.nodes.java.MonitorIdNode; +import org.graalvm.compiler.nodes.spi.StampProvider; +import org.graalvm.compiler.nodes.util.GraphUtil; +import org.graalvm.compiler.options.Option; +import org.graalvm.compiler.options.OptionType; +import org.graalvm.compiler.options.OptionValue; +import org.graalvm.compiler.phases.common.inlining.InliningUtil; + +import jdk.vm.ci.code.Architecture; +import jdk.vm.ci.code.BailoutException; +import jdk.vm.ci.code.BytecodeFrame; +import jdk.vm.ci.meta.ConstantReflectionProvider; +import jdk.vm.ci.meta.DeoptimizationAction; +import jdk.vm.ci.meta.DeoptimizationReason; +import jdk.vm.ci.meta.JavaConstant; +import jdk.vm.ci.meta.JavaKind; +import jdk.vm.ci.meta.JavaType; +import jdk.vm.ci.meta.MetaAccessProvider; +import jdk.vm.ci.meta.ResolvedJavaMethod; + +/** + * A graph decoder that performs partial evaluation, i.e., that performs method inlining and + * canonicalization/simplification of nodes during decoding. + * + * Inlining and loop explosion are configured via the plugin mechanism also used by the + * {@link GraphBuilderPhase}. However, not all callback methods defined in + * {@link GraphBuilderContext} are available since decoding is more limited than graph building. + * + * The standard {@link Canonicalizable#canonical node canonicalization} interface is used to + * canonicalize nodes during decoding. Additionally, {@link IfNode branches} and + * {@link IntegerSwitchNode switches} with constant conditions are simplified. + */ +public abstract class PEGraphDecoder extends SimplifyingGraphDecoder { + + public static class Options { + @Option(help = "Maximum inlining depth during partial evaluation before reporting an infinite recursion")// + public static final OptionValue InliningDepthError = new OptionValue<>(1000); + + @Option(help = "Max number of loop explosions per method.", type = OptionType.Debug)// + public static final OptionValue MaximumLoopExplosionCount = new OptionValue<>(10000); + + @Option(help = "Do not bail out but throw an exception on failed loop explosion.", type = OptionType.Debug) // + public static final OptionValue FailedLoopExplosionIsFatal = new OptionValue<>(false); + } + + protected class PEMethodScope extends MethodScope { + /** The state of the caller method. Only non-null during method inlining. */ + protected final PEMethodScope caller; + protected final ResolvedJavaMethod method; + protected final InvokeData invokeData; + protected final int inliningDepth; + + protected final LoopExplosionPlugin loopExplosionPlugin; + protected final InvocationPlugins invocationPlugins; + protected final InlineInvokePlugin[] inlineInvokePlugins; + protected final ParameterPlugin parameterPlugin; + protected final ValueNode[] arguments; + + protected FrameState outerState; + protected FrameState exceptionState; + protected ExceptionPlaceholderNode exceptionPlaceholderNode; + protected NodeSourcePosition callerBytecodePosition; + + protected PEMethodScope(StructuredGraph targetGraph, PEMethodScope caller, LoopScope callerLoopScope, EncodedGraph encodedGraph, ResolvedJavaMethod method, InvokeData invokeData, + int inliningDepth, LoopExplosionPlugin loopExplosionPlugin, InvocationPlugins invocationPlugins, InlineInvokePlugin[] inlineInvokePlugins, ParameterPlugin parameterPlugin, + ValueNode[] arguments) { + super(callerLoopScope, targetGraph, encodedGraph, loopExplosionKind(method, loopExplosionPlugin)); + + this.caller = caller; + this.method = method; + this.invokeData = invokeData; + this.inliningDepth = inliningDepth; + this.loopExplosionPlugin = loopExplosionPlugin; + this.invocationPlugins = invocationPlugins; + this.inlineInvokePlugins = inlineInvokePlugins; + this.parameterPlugin = parameterPlugin; + this.arguments = arguments; + } + + public boolean isInlinedMethod() { + return caller != null; + } + + public NodeSourcePosition getCallerBytecodePosition() { + if (caller == null) { + return null; + } + if (callerBytecodePosition == null) { + JavaConstant constantReceiver = caller.invokeData == null ? null : caller.invokeData.constantReceiver; + NodeSourcePosition callerPosition = caller.getCallerBytecodePosition(); + NodeSourcePosition invokePosition = invokeData.invoke.asNode().getNodeSourcePosition(); + callerBytecodePosition = invokePosition != null ? invokePosition.addCaller(constantReceiver, callerPosition) : callerPosition; + } + return callerBytecodePosition; + } + } + + protected class PENonAppendGraphBuilderContext implements GraphBuilderContext { + protected final PEMethodScope methodScope; + protected final Invoke invoke; + + public PENonAppendGraphBuilderContext(PEMethodScope methodScope, Invoke invoke) { + this.methodScope = methodScope; + this.invoke = invoke; + } + + @Override + public BailoutException bailout(String string) { + BailoutException bailout = new PermanentBailoutException(string); + throw GraphUtil.createBailoutException(string, bailout, GraphUtil.approxSourceStackTraceElement(methodScope.getCallerBytecodePosition())); + } + + @Override + public StampProvider getStampProvider() { + return stampProvider; + } + + @Override + public MetaAccessProvider getMetaAccess() { + return metaAccess; + } + + @Override + public ConstantReflectionProvider getConstantReflection() { + return constantReflection; + } + + @Override + public ConstantFieldProvider getConstantFieldProvider() { + return constantFieldProvider; + } + + @Override + public StructuredGraph getGraph() { + return methodScope.graph; + } + + @Override + public int getDepth() { + return methodScope.inliningDepth; + } + + @Override + public IntrinsicContext getIntrinsic() { + return null; + } + + @Override + public T append(T value) { + throw unimplemented(); + } + + @Override + public T recursiveAppend(T value) { + throw unimplemented(); + } + + @Override + public void push(JavaKind kind, ValueNode value) { + throw unimplemented(); + } + + @Override + public void handleReplacedInvoke(InvokeKind invokeKind, ResolvedJavaMethod targetMethod, ValueNode[] args, boolean inlineEverything) { + throw unimplemented(); + } + + @Override + public boolean intrinsify(BytecodeProvider bytecodeProvider, ResolvedJavaMethod targetMethod, ResolvedJavaMethod substitute, InvocationPlugin.Receiver receiver, ValueNode[] args) { + return false; + } + + @Override + public void setStateAfter(StateSplit stateSplit) { + throw unimplemented(); + } + + @Override + public GraphBuilderContext getParent() { + throw unimplemented(); + } + + @Override + public Bytecode getCode() { + throw unimplemented(); + } + + @Override + public ResolvedJavaMethod getMethod() { + throw unimplemented(); + } + + @Override + public int bci() { + return invoke.bci(); + } + + @Override + public InvokeKind getInvokeKind() { + throw unimplemented(); + } + + @Override + public JavaType getInvokeReturnType() { + throw unimplemented(); + } + } + + protected class PEAppendGraphBuilderContext extends PENonAppendGraphBuilderContext { + protected FixedWithNextNode lastInstr; + protected ValueNode pushedNode; + + public PEAppendGraphBuilderContext(PEMethodScope inlineScope, FixedWithNextNode lastInstr) { + super(inlineScope, inlineScope.invokeData.invoke); + this.lastInstr = lastInstr; + } + + @Override + public void push(JavaKind kind, ValueNode value) { + if (pushedNode != null) { + throw unimplemented("Only one push is supported"); + } + pushedNode = value; + } + + @Override + public void setStateAfter(StateSplit stateSplit) { + Node stateAfter = decodeFloatingNode(methodScope.caller, methodScope.callerLoopScope, methodScope.invokeData.stateAfterOrderId); + getGraph().add(stateAfter); + FrameState fs = (FrameState) handleFloatingNodeAfterAdd(methodScope.caller, methodScope.callerLoopScope, stateAfter); + stateSplit.setStateAfter(fs); + } + + @SuppressWarnings("try") + @Override + public T append(T v) { + if (v.graph() != null) { + return v; + } + try (DebugCloseable position = withNodeSoucePosition()) { + T added = getGraph().addOrUnique(v); + if (added == v) { + updateLastInstruction(v); + } + return added; + } + } + + private DebugCloseable withNodeSoucePosition() { + if (getGraph().mayHaveNodeSourcePosition()) { + return getGraph().withNodeSourcePosition(methodScope.getCallerBytecodePosition()); + } + return null; + } + + @SuppressWarnings("try") + @Override + public T recursiveAppend(T v) { + if (v.graph() != null) { + return v; + } + try (DebugCloseable position = withNodeSoucePosition()) { + T added = getGraph().addOrUniqueWithInputs(v); + if (added == v) { + updateLastInstruction(v); + } + return added; + } + } + + private void updateLastInstruction(T v) { + if (v instanceof FixedNode) { + FixedNode fixedNode = (FixedNode) v; + lastInstr.setNext(fixedNode); + if (fixedNode instanceof FixedWithNextNode) { + FixedWithNextNode fixedWithNextNode = (FixedWithNextNode) fixedNode; + assert fixedWithNextNode.next() == null : "cannot append instruction to instruction which isn't end"; + lastInstr = fixedWithNextNode; + } else { + lastInstr = null; + } + } + } + } + + @NodeInfo(cycles = CYCLES_IGNORED, size = SIZE_IGNORED) + static class ExceptionPlaceholderNode extends ValueNode { + public static final NodeClass TYPE = NodeClass.create(ExceptionPlaceholderNode.class); + + protected ExceptionPlaceholderNode() { + super(TYPE, StampFactory.object()); + } + } + + public PEGraphDecoder(MetaAccessProvider metaAccess, ConstantReflectionProvider constantReflection, ConstantFieldProvider constantFieldProvider, StampProvider stampProvider, + Architecture architecture) { + super(metaAccess, constantReflection, constantFieldProvider, stampProvider, true, architecture); + } + + protected static LoopExplosionKind loopExplosionKind(ResolvedJavaMethod method, LoopExplosionPlugin loopExplosionPlugin) { + if (loopExplosionPlugin == null) { + return LoopExplosionKind.NONE; + } else { + return loopExplosionPlugin.loopExplosionKind(method); + } + } + + public void decode(StructuredGraph targetGraph, ResolvedJavaMethod method, LoopExplosionPlugin loopExplosionPlugin, InvocationPlugins invocationPlugins, InlineInvokePlugin[] inlineInvokePlugins, + ParameterPlugin parameterPlugin) { + PEMethodScope methodScope = new PEMethodScope(targetGraph, null, null, lookupEncodedGraph(method, null), method, null, 0, loopExplosionPlugin, invocationPlugins, inlineInvokePlugins, + parameterPlugin, null); + decode(createInitialLoopScope(methodScope, null)); + cleanupGraph(methodScope); + + Debug.dump(Debug.VERBOSE_LOG_LEVEL, methodScope.graph, "After graph cleanup"); + assert methodScope.graph.verify(); + + try { + /* Check that the control flow graph can be computed, to catch problems early. */ + assert CFGVerifier.verify(ControlFlowGraph.compute(methodScope.graph, true, true, true, true)); + } catch (Throwable ex) { + throw GraalError.shouldNotReachHere("Control flow graph not valid after partial evaluation"); + } + } + + @Override + protected void cleanupGraph(MethodScope methodScope) { + super.cleanupGraph(methodScope); + + for (FrameState frameState : methodScope.graph.getNodes(FrameState.TYPE)) { + if (frameState.bci == BytecodeFrame.UNWIND_BCI) { + /* + * handleMissingAfterExceptionFrameState is called during graph decoding from + * InliningUtil.processFrameState - but during graph decoding it does not do + * anything because the usages of the frameState are not available yet. So we need + * to call it again. + */ + InliningUtil.handleMissingAfterExceptionFrameState(frameState); + + /* + * The frameState must be gone now, because it is not a valid deoptimization point. + */ + assert frameState.isDeleted(); + } + } + } + + @Override + protected void checkLoopExplosionIteration(MethodScope s, LoopScope loopScope) { + PEMethodScope methodScope = (PEMethodScope) s; + + if (loopScope.loopIteration > Options.MaximumLoopExplosionCount.getValue()) { + throw tooManyLoopExplosionIterations(methodScope); + } + } + + private static RuntimeException tooManyLoopExplosionIterations(PEMethodScope methodScope) { + String message = "too many loop explosion iterations - does the explosion not terminate for method " + methodScope.method + "?"; + RuntimeException bailout = Options.FailedLoopExplosionIsFatal.getValue() ? new RuntimeException(message) : new PermanentBailoutException(message); + throw GraphUtil.createBailoutException(message, bailout, GraphUtil.approxSourceStackTraceElement(methodScope.getCallerBytecodePosition())); + } + + @Override + protected LoopScope handleInvoke(MethodScope s, LoopScope loopScope, InvokeData invokeData) { + PEMethodScope methodScope = (PEMethodScope) s; + + /* + * Decode the call target, but do not add it to the graph yet. This avoids adding usages for + * all the arguments, which are expensive to remove again when we can inline the method. + */ + assert invokeData.invoke.callTarget() == null : "callTarget edge is ignored during decoding of Invoke"; + CallTargetNode callTarget = (CallTargetNode) decodeFloatingNode(methodScope, loopScope, invokeData.callTargetOrderId); + if (callTarget instanceof MethodCallTargetNode) { + MethodCallTargetNode methodCall = (MethodCallTargetNode) callTarget; + if (methodCall.invokeKind().hasReceiver()) { + invokeData.constantReceiver = methodCall.arguments().get(0).asJavaConstant(); + } + LoopScope inlineLoopScope = trySimplifyInvoke(methodScope, loopScope, invokeData, (MethodCallTargetNode) callTarget); + if (inlineLoopScope != null) { + return inlineLoopScope; + } + } + + /* We know that we need an invoke, so now we can add the call target to the graph. */ + methodScope.graph.add(callTarget); + registerNode(loopScope, invokeData.callTargetOrderId, callTarget, false, false); + return super.handleInvoke(methodScope, loopScope, invokeData); + } + + protected LoopScope trySimplifyInvoke(PEMethodScope methodScope, LoopScope loopScope, InvokeData invokeData, MethodCallTargetNode callTarget) { + // attempt to devirtualize the call + ResolvedJavaMethod specialCallTarget = MethodCallTargetNode.findSpecialCallTarget(callTarget.invokeKind(), callTarget.receiver(), callTarget.targetMethod(), invokeData.contextType); + if (specialCallTarget != null) { + callTarget.setTargetMethod(specialCallTarget); + callTarget.setInvokeKind(InvokeKind.Special); + } + + if (tryInvocationPlugin(methodScope, loopScope, invokeData, callTarget)) { + /* + * The invocation plugin handled the call, so decoding continues in the calling method. + */ + return loopScope; + } + LoopScope inlineLoopScope = tryInline(methodScope, loopScope, invokeData, callTarget); + if (inlineLoopScope != null) { + /* + * We can inline the call, so decoding continues in the inlined method. + */ + return inlineLoopScope; + } + + for (InlineInvokePlugin plugin : methodScope.inlineInvokePlugins) { + plugin.notifyNotInlined(new PENonAppendGraphBuilderContext(methodScope, invokeData.invoke), callTarget.targetMethod(), invokeData.invoke); + } + return null; + } + + protected boolean tryInvocationPlugin(PEMethodScope methodScope, LoopScope loopScope, InvokeData invokeData, MethodCallTargetNode callTarget) { + if (methodScope.invocationPlugins == null) { + return false; + } + + Invoke invoke = invokeData.invoke; + + ResolvedJavaMethod targetMethod = callTarget.targetMethod(); + InvocationPlugin invocationPlugin = methodScope.invocationPlugins.lookupInvocation(targetMethod); + if (invocationPlugin == null) { + return false; + } + + ValueNode[] arguments = callTarget.arguments().toArray(new ValueNode[0]); + FixedWithNextNode invokePredecessor = (FixedWithNextNode) invoke.asNode().predecessor(); + + /* + * Remove invoke from graph so that invocation plugin can append nodes to the predecessor. + */ + invoke.asNode().replaceAtPredecessor(null); + + PEMethodScope inlineScope = new PEMethodScope(methodScope.graph, methodScope, loopScope, null, targetMethod, invokeData, methodScope.inliningDepth + 1, methodScope.loopExplosionPlugin, + methodScope.invocationPlugins, methodScope.inlineInvokePlugins, null, arguments); + PEAppendGraphBuilderContext graphBuilderContext = new PEAppendGraphBuilderContext(inlineScope, invokePredecessor); + InvocationPluginReceiver invocationPluginReceiver = new InvocationPluginReceiver(graphBuilderContext); + + if (invocationPlugin.execute(graphBuilderContext, targetMethod, invocationPluginReceiver.init(targetMethod, arguments), arguments)) { + + if (graphBuilderContext.lastInstr != null) { + registerNode(loopScope, invokeData.invokeOrderId, graphBuilderContext.pushedNode, true, true); + invoke.asNode().replaceAtUsages(graphBuilderContext.pushedNode); + graphBuilderContext.lastInstr.setNext(nodeAfterInvoke(methodScope, loopScope, invokeData, AbstractBeginNode.prevBegin(graphBuilderContext.lastInstr))); + } else { + assert graphBuilderContext.pushedNode == null : "Why push a node when the invoke does not return anyway?"; + invoke.asNode().replaceAtUsages(null); + } + + deleteInvoke(invoke); + return true; + + } else { + /* Intrinsification failed, restore original state: invoke is in Graph. */ + invokePredecessor.setNext(invoke.asNode()); + return false; + } + } + + protected LoopScope tryInline(PEMethodScope methodScope, LoopScope loopScope, InvokeData invokeData, MethodCallTargetNode callTarget) { + if (!callTarget.invokeKind().isDirect()) { + return null; + } + + ResolvedJavaMethod targetMethod = callTarget.targetMethod(); + if (!targetMethod.canBeInlined()) { + return null; + } + + ValueNode[] arguments = callTarget.arguments().toArray(new ValueNode[0]); + GraphBuilderContext graphBuilderContext = new PENonAppendGraphBuilderContext(methodScope, invokeData.invoke); + + for (InlineInvokePlugin plugin : methodScope.inlineInvokePlugins) { + InlineInfo inlineInfo = plugin.shouldInlineInvoke(graphBuilderContext, targetMethod, arguments); + if (inlineInfo != null) { + if (inlineInfo.getMethodToInline() == null) { + return null; + } else { + return doInline(methodScope, loopScope, invokeData, inlineInfo, arguments); + } + } + } + return null; + } + + protected LoopScope doInline(PEMethodScope methodScope, LoopScope loopScope, InvokeData invokeData, InlineInfo inlineInfo, ValueNode[] arguments) { + ResolvedJavaMethod inlineMethod = inlineInfo.getMethodToInline(); + EncodedGraph graphToInline = lookupEncodedGraph(inlineMethod, inlineInfo.getIntrinsicBytecodeProvider()); + if (graphToInline == null) { + return null; + } + + if (methodScope.inliningDepth > Options.InliningDepthError.getValue()) { + throw tooDeepInlining(methodScope); + } + + for (InlineInvokePlugin plugin : methodScope.inlineInvokePlugins) { + plugin.notifyBeforeInline(inlineMethod); + } + + Invoke invoke = invokeData.invoke; + FixedNode invokeNode = invoke.asNode(); + FixedWithNextNode predecessor = (FixedWithNextNode) invokeNode.predecessor(); + invokeNode.replaceAtPredecessor(null); + + PEMethodScope inlineScope = new PEMethodScope(methodScope.graph, methodScope, loopScope, graphToInline, inlineMethod, invokeData, methodScope.inliningDepth + 1, + methodScope.loopExplosionPlugin, methodScope.invocationPlugins, methodScope.inlineInvokePlugins, null, arguments); + + /* + * After decoding all the nodes of the inlined method, we need to re-wire the return and + * unwind nodes. Since inlining is non-recursive, this cannot be done at the end of this + * method, but must be registered as a cleanup task that runs when all nodes of the inlined + * methods have been decoded. + */ + inlineScope.cleanupTasks.add(() -> finishInlining(methodScope, loopScope, invokeData, inlineMethod, inlineScope)); + + /* + * Do the actual inlining by returning the initial loop scope for the inlined method scope. + */ + return createInitialLoopScope(inlineScope, predecessor); + } + + protected void finishInlining(PEMethodScope methodScope, LoopScope loopScope, InvokeData invokeData, ResolvedJavaMethod inlineMethod, PEMethodScope inlineScope) { + Invoke invoke = invokeData.invoke; + FixedNode invokeNode = invoke.asNode(); + + ValueNode exceptionValue = null; + List unwindNodes = inlineScope.unwindNodes; + Iterator iter = unwindNodes.iterator(); + while (iter.hasNext()) { + if (iter.next().isDeleted()) { + iter.remove(); + } + } + + if (!unwindNodes.isEmpty()) { + FixedNode unwindReplacement; + if (invoke instanceof InvokeWithExceptionNode) { + /* Decoding continues for the exception handler. */ + unwindReplacement = makeStubNode(methodScope, loopScope, invokeData.exceptionNextOrderId); + } else { + /* No exception handler available, so the only thing we can do is deoptimize. */ + unwindReplacement = methodScope.graph.add(new DeoptimizeNode(DeoptimizationAction.InvalidateRecompile, DeoptimizationReason.NotCompiledExceptionHandler)); + } + + if (unwindNodes.size() == 1) { + /* Only one UnwindNode, we can use the exception directly. */ + UnwindNode unwindNode = unwindNodes.get(0); + exceptionValue = unwindNode.exception(); + unwindNode.replaceAndDelete(unwindReplacement); + + } else { + /* + * More than one UnwindNode. This can happen with the loop explosion strategy + * FULL_EXPLODE_UNTIL_RETURN, where we keep exploding after the loop and therefore + * also explode exception paths. Merge the exception in a similar way as multiple + * return values. + */ + MergeNode unwindMergeNode = methodScope.graph.add(new MergeNode()); + exceptionValue = InliningUtil.mergeValueProducers(unwindMergeNode, unwindNodes, null, unwindNode -> unwindNode.exception()); + unwindMergeNode.setNext(unwindReplacement); + + ensureExceptionStateDecoded(inlineScope); + unwindMergeNode.setStateAfter(inlineScope.exceptionState.duplicateModified(JavaKind.Object, JavaKind.Object, exceptionValue)); + } + } + + assert invoke.next() == null; + assert !(invoke instanceof InvokeWithExceptionNode) || ((InvokeWithExceptionNode) invoke).exceptionEdge() == null; + + ValueNode returnValue; + List returnNodes = inlineScope.returnNodes; + if (!returnNodes.isEmpty()) { + if (returnNodes.size() == 1) { + ReturnNode returnNode = returnNodes.get(0); + returnValue = returnNode.result(); + FixedNode n = nodeAfterInvoke(methodScope, loopScope, invokeData, AbstractBeginNode.prevBegin(returnNode)); + returnNode.replaceAndDelete(n); + } else { + AbstractMergeNode merge = methodScope.graph.add(new MergeNode()); + merge.setStateAfter((FrameState) ensureNodeCreated(methodScope, loopScope, invokeData.stateAfterOrderId)); + returnValue = InliningUtil.mergeReturns(merge, returnNodes, null); + FixedNode n = nodeAfterInvoke(methodScope, loopScope, invokeData, merge); + merge.setNext(n); + } + } else { + returnValue = null; + } + invokeNode.replaceAtUsages(returnValue); + + /* + * Usage the handles that we have on the return value and the exception to update the + * orderId->Node table. + */ + registerNode(loopScope, invokeData.invokeOrderId, returnValue, true, true); + if (invoke instanceof InvokeWithExceptionNode) { + registerNode(loopScope, invokeData.exceptionOrderId, exceptionValue, true, true); + } + if (inlineScope.exceptionPlaceholderNode != null) { + inlineScope.exceptionPlaceholderNode.replaceAtUsagesAndDelete(exceptionValue); + } + deleteInvoke(invoke); + + for (InlineInvokePlugin plugin : methodScope.inlineInvokePlugins) { + plugin.notifyAfterInline(inlineMethod); + } + + if (Debug.isDumpEnabled(Debug.INFO_LOG_LEVEL) && DumpDuringGraphBuilding.getValue()) { + Debug.dump(Debug.INFO_LOG_LEVEL, methodScope.graph, "Inline finished: %s.%s", inlineMethod.getDeclaringClass().getUnqualifiedName(), inlineMethod.getName()); + } + } + + private static RuntimeException tooDeepInlining(PEMethodScope methodScope) { + HashMap methodCounts = new HashMap<>(); + for (PEMethodScope cur = methodScope; cur != null; cur = cur.caller) { + Integer oldCount = methodCounts.get(cur.method); + methodCounts.put(cur.method, oldCount == null ? 1 : oldCount + 1); + } + + List> methods = new ArrayList<>(methodCounts.entrySet()); + methods.sort((e1, e2) -> -Integer.compare(e1.getValue(), e2.getValue())); + + StringBuilder msg = new StringBuilder("Too deep inlining, probably caused by recursive inlining.").append(System.lineSeparator()).append("== Inlined methods ordered by inlining frequency:"); + for (Map.Entry entry : methods) { + msg.append(System.lineSeparator()).append(entry.getKey().format("%H.%n(%p) [")).append(entry.getValue()).append("]"); + } + msg.append(System.lineSeparator()).append("== Complete stack trace of inlined methods:"); + int lastBci = 0; + for (PEMethodScope cur = methodScope; cur != null; cur = cur.caller) { + msg.append(System.lineSeparator()).append(cur.method.asStackTraceElement(lastBci)); + if (cur.invokeData != null) { + lastBci = cur.invokeData.invoke.bci(); + } else { + lastBci = 0; + } + } + + throw new PermanentBailoutException(msg.toString()); + } + + public FixedNode nodeAfterInvoke(PEMethodScope methodScope, LoopScope loopScope, InvokeData invokeData, AbstractBeginNode lastBlock) { + assert lastBlock.isAlive(); + FixedNode n; + if (invokeData.invoke instanceof InvokeWithExceptionNode) { + registerNode(loopScope, invokeData.nextOrderId, lastBlock, false, false); + n = makeStubNode(methodScope, loopScope, invokeData.nextNextOrderId); + } else { + n = makeStubNode(methodScope, loopScope, invokeData.nextOrderId); + } + return n; + } + + private static void deleteInvoke(Invoke invoke) { + /* + * Clean up unused nodes. We cannot just call killCFG on the invoke node because that can + * kill too much: nodes that are decoded later can use values that appear unused by now. + */ + FrameState frameState = invoke.stateAfter(); + invoke.asNode().safeDelete(); + assert invoke.callTarget() == null : "must not have been added to the graph yet"; + if (frameState != null && frameState.hasNoUsages()) { + frameState.safeDelete(); + } + } + + protected abstract EncodedGraph lookupEncodedGraph(ResolvedJavaMethod method, BytecodeProvider intrinsicBytecodeProvider); + + @SuppressWarnings("try") + @Override + protected void handleFixedNode(MethodScope s, LoopScope loopScope, int nodeOrderId, FixedNode node) { + PEMethodScope methodScope = (PEMethodScope) s; + if (node instanceof ForeignCallNode) { + ForeignCallNode foreignCall = (ForeignCallNode) node; + if (foreignCall.getBci() == BytecodeFrame.UNKNOWN_BCI && methodScope.invokeData != null) { + foreignCall.setBci(methodScope.invokeData.invoke.bci()); + } + } + + NodeSourcePosition pos = node.getNodeSourcePosition(); + if (pos != null && methodScope.isInlinedMethod()) { + NodeSourcePosition newPosition = pos.addCaller(methodScope.getCallerBytecodePosition()); + try (DebugCloseable scope = node.graph().withoutNodeSourcePosition()) { + super.handleFixedNode(s, loopScope, nodeOrderId, node); + } + if (node.isAlive()) { + node.setNodeSourcePosition(newPosition); + } + } else { + super.handleFixedNode(s, loopScope, nodeOrderId, node); + } + + } + + @Override + protected Node handleFloatingNodeBeforeAdd(MethodScope s, LoopScope loopScope, Node node) { + PEMethodScope methodScope = (PEMethodScope) s; + + if (node instanceof ParameterNode) { + ParameterNode param = (ParameterNode) node; + if (methodScope.arguments != null) { + Node result = methodScope.arguments[param.index()]; + assert result != null; + return result; + + } else if (methodScope.parameterPlugin != null) { + GraphBuilderContext graphBuilderContext = new PENonAppendGraphBuilderContext(methodScope, null); + Node result = methodScope.parameterPlugin.interceptParameter(graphBuilderContext, param.index(), + StampPair.create(param.stamp(), param.uncheckedStamp())); + if (result != null) { + return result; + } + } + + } + + return super.handleFloatingNodeBeforeAdd(methodScope, loopScope, node); + } + + protected void ensureOuterStateDecoded(PEMethodScope methodScope) { + if (methodScope.outerState == null && methodScope.caller != null) { + FrameState stateAtReturn = methodScope.invokeData.invoke.stateAfter(); + if (stateAtReturn == null) { + stateAtReturn = (FrameState) decodeFloatingNode(methodScope.caller, methodScope.callerLoopScope, methodScope.invokeData.stateAfterOrderId); + } + + JavaKind invokeReturnKind = methodScope.invokeData.invoke.asNode().getStackKind(); + FrameState outerState = stateAtReturn.duplicateModified(methodScope.graph, methodScope.invokeData.invoke.bci(), stateAtReturn.rethrowException(), true, invokeReturnKind, null, null); + + /* + * When the encoded graph has methods inlining, we can already have a proper caller + * state. If not, we set the caller state here. + */ + if (outerState.outerFrameState() == null && methodScope.caller != null) { + ensureOuterStateDecoded(methodScope.caller); + outerState.setOuterFrameState(methodScope.caller.outerState); + } + methodScope.outerState = outerState; + } + } + + protected void ensureStateAfterDecoded(PEMethodScope methodScope) { + if (methodScope.invokeData.invoke.stateAfter() == null) { + methodScope.invokeData.invoke.setStateAfter((FrameState) ensureNodeCreated(methodScope.caller, methodScope.callerLoopScope, methodScope.invokeData.stateAfterOrderId)); + } + } + + protected void ensureExceptionStateDecoded(PEMethodScope methodScope) { + if (methodScope.exceptionState == null && methodScope.caller != null && methodScope.invokeData.invoke instanceof InvokeWithExceptionNode) { + ensureStateAfterDecoded(methodScope); + + assert methodScope.exceptionPlaceholderNode == null; + methodScope.exceptionPlaceholderNode = methodScope.graph.add(new ExceptionPlaceholderNode()); + registerNode(methodScope.callerLoopScope, methodScope.invokeData.exceptionOrderId, methodScope.exceptionPlaceholderNode, false, false); + FrameState exceptionState = (FrameState) ensureNodeCreated(methodScope.caller, methodScope.callerLoopScope, methodScope.invokeData.exceptionStateOrderId); + + if (exceptionState.outerFrameState() == null && methodScope.caller != null) { + ensureOuterStateDecoded(methodScope.caller); + exceptionState.setOuterFrameState(methodScope.caller.outerState); + } + methodScope.exceptionState = exceptionState; + } + } + + @Override + protected Node addFloatingNode(MethodScope s, Node node) { + Node addedNode = super.addFloatingNode(s, node); + PEMethodScope methodScope = (PEMethodScope) s; + NodeSourcePosition pos = node.getNodeSourcePosition(); + if (methodScope.isInlinedMethod()) { + if (pos != null) { + NodeSourcePosition bytecodePosition = methodScope.getCallerBytecodePosition(); + node.setNodeSourcePosition(pos.addCaller(bytecodePosition)); + } + } + return addedNode; + } + + @Override + protected Node handleFloatingNodeAfterAdd(MethodScope s, LoopScope loopScope, Node node) { + PEMethodScope methodScope = (PEMethodScope) s; + + if (methodScope.isInlinedMethod()) { + if (node instanceof FrameState) { + FrameState frameState = (FrameState) node; + + ensureOuterStateDecoded(methodScope); + if (frameState.bci < 0) { + ensureExceptionStateDecoded(methodScope); + } + List invokeArgsList = null; + if (frameState.bci == BytecodeFrame.BEFORE_BCI) { + /* + * We know that the argument list is only used in this case, so avoid the List + * allocation for "normal" bcis. + */ + invokeArgsList = Arrays.asList(methodScope.arguments); + } + return InliningUtil.processFrameState(frameState, methodScope.invokeData.invoke, methodScope.method, methodScope.exceptionState, methodScope.outerState, true, methodScope.method, + invokeArgsList); + + } else if (node instanceof MonitorIdNode) { + ensureOuterStateDecoded(methodScope); + InliningUtil.processMonitorId(methodScope.outerState, (MonitorIdNode) node); + return node; + } + } + + return node; + } +}