/* * Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 only, as * published by the Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that * accompanied this code). * * You should have received a copy of the GNU General Public License version * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA * or visit www.oracle.com if you need additional information or have any * questions. */ package org.graalvm.compiler.replacements; import static org.graalvm.compiler.core.common.CompilationIdentifier.INVALID_COMPILATION_ID; import static org.graalvm.compiler.core.common.GraalOptions.UseGraalInstrumentation; import static org.graalvm.compiler.core.common.LocationIdentity.ANY_LOCATION; import static org.graalvm.compiler.core.common.LocationIdentity.any; import static org.graalvm.compiler.debug.Debug.applyFormattingFlagsAndWidth; import static org.graalvm.compiler.graph.iterators.NodePredicates.isNotA; import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_IGNORED; import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_IGNORED; import static org.graalvm.compiler.phases.common.DeadCodeEliminationPhase.Optionality.Required; import static java.util.FormattableFlags.ALTERNATE; import java.lang.reflect.Array; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Formattable; import java.util.Formatter; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Predicate; import java.util.stream.Collectors; import org.graalvm.compiler.api.replacements.Snippet; import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter; import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter; import org.graalvm.compiler.api.replacements.SnippetReflectionProvider; import org.graalvm.compiler.core.common.GraalOptions; import org.graalvm.compiler.core.common.LocationIdentity; import org.graalvm.compiler.core.common.type.Stamp; import org.graalvm.compiler.core.common.type.StampFactory; import org.graalvm.compiler.core.common.type.StampPair; import org.graalvm.compiler.core.common.type.TypeReference; import org.graalvm.compiler.debug.Debug; import org.graalvm.compiler.debug.Debug.Scope; import org.graalvm.compiler.debug.DebugCloseable; import org.graalvm.compiler.debug.DebugCounter; import org.graalvm.compiler.debug.DebugTimer; import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.graph.Graph.Mark; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.graph.NodeClass; import org.graalvm.compiler.graph.Position; import org.graalvm.compiler.loop.LoopEx; import org.graalvm.compiler.loop.LoopsData; import org.graalvm.compiler.loop.phases.LoopTransformations; import org.graalvm.compiler.nodeinfo.InputType; import org.graalvm.compiler.nodeinfo.NodeInfo; import org.graalvm.compiler.nodes.AbstractBeginNode; import org.graalvm.compiler.nodes.AbstractMergeNode; import org.graalvm.compiler.nodes.ConstantNode; import org.graalvm.compiler.nodes.ControlSinkNode; import org.graalvm.compiler.nodes.DeoptimizingNode; import org.graalvm.compiler.nodes.FixedNode; import org.graalvm.compiler.nodes.FixedWithNextNode; import org.graalvm.compiler.nodes.FrameState; import org.graalvm.compiler.nodes.LoopBeginNode; import org.graalvm.compiler.nodes.MergeNode; import org.graalvm.compiler.nodes.ParameterNode; import org.graalvm.compiler.nodes.PhiNode; import org.graalvm.compiler.nodes.ReturnNode; import org.graalvm.compiler.nodes.StartNode; import org.graalvm.compiler.nodes.StateSplit; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.StructuredGraph.AllowAssumptions; import org.graalvm.compiler.nodes.StructuredGraph.GuardsStage; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.ValueNodeUtil; import org.graalvm.compiler.nodes.calc.FloatingNode; import org.graalvm.compiler.nodes.debug.instrumentation.InstrumentationNode; import org.graalvm.compiler.nodes.java.LoadIndexedNode; import org.graalvm.compiler.nodes.java.StoreIndexedNode; import org.graalvm.compiler.nodes.memory.MemoryAccess; import org.graalvm.compiler.nodes.memory.MemoryAnchorNode; import org.graalvm.compiler.nodes.memory.MemoryCheckpoint; import org.graalvm.compiler.nodes.memory.MemoryMap; import org.graalvm.compiler.nodes.memory.MemoryMapNode; import org.graalvm.compiler.nodes.memory.MemoryNode; import org.graalvm.compiler.nodes.memory.MemoryPhiNode; import org.graalvm.compiler.nodes.spi.ArrayLengthProvider; import org.graalvm.compiler.nodes.spi.LoweringTool; import org.graalvm.compiler.nodes.spi.MemoryProxy; import org.graalvm.compiler.nodes.util.GraphUtil; import org.graalvm.compiler.options.Option; import org.graalvm.compiler.options.OptionValue; import org.graalvm.compiler.phases.common.CanonicalizerPhase; import org.graalvm.compiler.phases.common.DeadCodeEliminationPhase; import org.graalvm.compiler.phases.common.FloatingReadPhase; import org.graalvm.compiler.phases.common.FloatingReadPhase.MemoryMapImpl; import org.graalvm.compiler.phases.common.GuardLoweringPhase; import org.graalvm.compiler.phases.common.LoweringPhase; import org.graalvm.compiler.phases.common.inlining.InliningUtil; import org.graalvm.compiler.phases.tiers.PhaseContext; import org.graalvm.compiler.phases.util.Providers; import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode; import org.graalvm.compiler.replacements.nodes.LoadSnippetVarargParameterNode; import org.graalvm.compiler.word.WordBase; import jdk.vm.ci.code.TargetDescription; import jdk.vm.ci.meta.Constant; import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.Local; import jdk.vm.ci.meta.LocalVariableTable; import jdk.vm.ci.meta.MetaAccessProvider; import jdk.vm.ci.meta.ResolvedJavaMethod; import jdk.vm.ci.meta.ResolvedJavaType; import jdk.vm.ci.meta.Signature; /** * A snippet template is a graph created by parsing a snippet method and then specialized by binding * constants to the snippet's {@link ConstantParameter} parameters. * * Snippet templates can be managed in a cache maintained by {@link AbstractTemplates}. */ public class SnippetTemplate { private boolean mayRemoveLocation = false; /** * Holds the {@link ResolvedJavaMethod} of the snippet, together with some information about the * method that needs to be computed only once. The {@link SnippetInfo} should be created once * per snippet and then cached. */ public abstract static class SnippetInfo { protected final ResolvedJavaMethod method; protected ResolvedJavaMethod original; protected final LocationIdentity[] privateLocations; /** * Lazily constructed parts of {@link SnippetInfo}. */ static class Lazy { Lazy(ResolvedJavaMethod method) { int count = method.getSignature().getParameterCount(false); constantParameters = new boolean[count]; varargsParameters = new boolean[count]; for (int i = 0; i < count; i++) { constantParameters[i] = method.getParameterAnnotation(ConstantParameter.class, i) != null; varargsParameters[i] = method.getParameterAnnotation(VarargsParameter.class, i) != null; assert !constantParameters[i] || !varargsParameters[i] : "Parameter cannot be annotated with both @" + ConstantParameter.class.getSimpleName() + " and @" + VarargsParameter.class.getSimpleName(); } // Retrieve the names only when assertions are turned on. assert initNames(method, count); } final boolean[] constantParameters; final boolean[] varargsParameters; /** * The parameter names, taken from the local variables table. Only used for assertion * checking, so use only within an assert statement. */ String[] names; private boolean initNames(ResolvedJavaMethod method, int parameterCount) { names = new String[parameterCount]; int slotIdx = 0; LocalVariableTable localVariableTable = method.getLocalVariableTable(); if (localVariableTable != null) { for (int i = 0; i < names.length; i++) { Local local = localVariableTable.getLocal(slotIdx, 0); if (local != null) { names[i] = local.getName(); } JavaKind kind = method.getSignature().getParameterKind(i); slotIdx += kind.getSlotCount(); } } return true; } } /** * Times instantiations of all templates derived form this snippet. * * @see SnippetTemplate#instantiationTimer */ private final DebugTimer instantiationTimer; /** * Counts instantiations of all templates derived from this snippet. * * @see SnippetTemplate#instantiationCounter */ private final DebugCounter instantiationCounter; protected abstract Lazy lazy(); protected SnippetInfo(ResolvedJavaMethod method, LocationIdentity[] privateLocations) { this.method = method; this.privateLocations = SnippetCounterNode.addSnippetCounters(privateLocations); instantiationCounter = Debug.counter("SnippetInstantiationCount[%s]", method.getName()); instantiationTimer = Debug.timer("SnippetInstantiationTime[%s]", method.getName()); assert method.isStatic() : "snippet method must be static: " + method.format("%H.%n"); } public ResolvedJavaMethod getMethod() { return method; } public int getParameterCount() { return lazy().constantParameters.length; } public void setOriginalMethod(ResolvedJavaMethod original) { this.original = original; } public boolean isConstantParameter(int paramIdx) { return lazy().constantParameters[paramIdx]; } public boolean isVarargsParameter(int paramIdx) { return lazy().varargsParameters[paramIdx]; } public String getParameterName(int paramIdx) { String[] names = lazy().names; if (names != null) { return names[paramIdx]; } return null; } @Override public String toString() { return getClass().getSimpleName() + ":" + method.format("%h.%n"); } } protected static class LazySnippetInfo extends SnippetInfo { protected final AtomicReference lazy = new AtomicReference<>(null); protected LazySnippetInfo(ResolvedJavaMethod method, LocationIdentity[] privateLocations) { super(method, privateLocations); } @Override protected Lazy lazy() { if (lazy.get() == null) { lazy.compareAndSet(null, new Lazy(method)); } return lazy.get(); } } protected static class EagerSnippetInfo extends SnippetInfo { protected final Lazy lazy; protected EagerSnippetInfo(ResolvedJavaMethod method, LocationIdentity[] privateLocations) { super(method, privateLocations); lazy = new Lazy(method); } @Override protected Lazy lazy() { return lazy; } } /** * Values that are bound to the snippet method parameters. The methods {@link #add}, * {@link #addConst}, and {@link #addVarargs} must be called in the same order as in the * signature of the snippet method. The parameter name is passed to the add methods for * assertion checking, i.e., to enforce that the order matches. Which method needs to be called * depends on the annotation of the snippet method parameter: * */ public static class Arguments implements Formattable { protected final SnippetInfo info; protected final CacheKey cacheKey; protected final Object[] values; protected final Stamp[] constStamps; protected boolean cacheable; protected int nextParamIdx; public Arguments(SnippetInfo info, GuardsStage guardsStage, LoweringTool.LoweringStage loweringStage) { this.info = info; this.cacheKey = new CacheKey(info, guardsStage, loweringStage); this.values = new Object[info.getParameterCount()]; this.constStamps = new Stamp[info.getParameterCount()]; this.cacheable = true; } public Arguments add(String name, Object value) { assert check(name, false, false); values[nextParamIdx] = value; nextParamIdx++; return this; } public Arguments addConst(String name, Object value) { assert value != null; return addConst(name, value, null); } public Arguments addConst(String name, Object value, Stamp stamp) { assert check(name, true, false); values[nextParamIdx] = value; constStamps[nextParamIdx] = stamp; cacheKey.setParam(nextParamIdx, value); nextParamIdx++; return this; } public Arguments addVarargs(String name, Class componentType, Stamp argStamp, Object value) { assert check(name, false, true); Varargs varargs = new Varargs(componentType, argStamp, value); values[nextParamIdx] = varargs; // A separate template is necessary for every distinct array length cacheKey.setParam(nextParamIdx, varargs.length); nextParamIdx++; return this; } public void setCacheable(boolean cacheable) { this.cacheable = cacheable; } private boolean check(String name, boolean constParam, boolean varargsParam) { assert nextParamIdx < info.getParameterCount() : "too many parameters: " + name + " " + this; assert info.getParameterName(nextParamIdx) == null || info.getParameterName(nextParamIdx).equals(name) : "wrong parameter name: " + name + " " + this; assert constParam == info.isConstantParameter(nextParamIdx) : "Parameter " + (constParam ? "not " : "") + "annotated with @" + ConstantParameter.class.getSimpleName() + ": " + name + " " + this; assert varargsParam == info.isVarargsParameter(nextParamIdx) : "Parameter " + (varargsParam ? "not " : "") + "annotated with @" + VarargsParameter.class.getSimpleName() + ": " + name + " " + this; return true; } @Override public String toString() { StringBuilder result = new StringBuilder(); result.append("Parameters<").append(info.method.format("%h.%n")).append(" ["); String sep = ""; for (int i = 0; i < info.getParameterCount(); i++) { result.append(sep); if (info.isConstantParameter(i)) { result.append("const "); } else if (info.isVarargsParameter(i)) { result.append("varargs "); } result.append(info.getParameterName(i)).append(" = ").append(values[i]); sep = ", "; } result.append(">"); return result.toString(); } @Override public void formatTo(Formatter formatter, int flags, int width, int precision) { if ((flags & ALTERNATE) == 0) { formatter.format(applyFormattingFlagsAndWidth(toString(), flags, width)); } else { StringBuilder sb = new StringBuilder(); sb.append(info.method.getName()).append('('); String sep = ""; for (int i = 0; i < info.getParameterCount(); i++) { if (info.isConstantParameter(i)) { sb.append(sep); if (info.getParameterName(i) != null) { sb.append(info.getParameterName(i)); } else { sb.append(i); } sb.append('=').append(values[i]); sep = ", "; } } sb.append(")"); String string = sb.toString(); if (string.indexOf('%') != -1) { // Quote any % signs string = string.replace("%", "%%"); } formatter.format(applyFormattingFlagsAndWidth(string, flags & ~ALTERNATE, width)); } } } /** * Wrapper for the prototype value of a {@linkplain VarargsParameter varargs} parameter. */ static class Varargs { protected final Class componentType; protected final Stamp stamp; protected final Object value; protected final int length; protected Varargs(Class componentType, Stamp stamp, Object value) { this.componentType = componentType; this.stamp = stamp; this.value = value; if (value instanceof List) { this.length = ((List) value).size(); } else { this.length = Array.getLength(value); } } @Override public String toString() { if (value instanceof boolean[]) { return Arrays.toString((boolean[]) value); } if (value instanceof byte[]) { return Arrays.toString((byte[]) value); } if (value instanceof char[]) { return Arrays.toString((char[]) value); } if (value instanceof short[]) { return Arrays.toString((short[]) value); } if (value instanceof int[]) { return Arrays.toString((int[]) value); } if (value instanceof long[]) { return Arrays.toString((long[]) value); } if (value instanceof float[]) { return Arrays.toString((float[]) value); } if (value instanceof double[]) { return Arrays.toString((double[]) value); } if (value instanceof Object[]) { return Arrays.toString((Object[]) value); } return String.valueOf(value); } } @NodeInfo(cycles = CYCLES_IGNORED, size = SIZE_IGNORED) static final class VarargsPlaceholderNode extends FloatingNode implements ArrayLengthProvider { public static final NodeClass TYPE = NodeClass.create(VarargsPlaceholderNode.class); protected final Varargs varargs; protected VarargsPlaceholderNode(Varargs varargs, MetaAccessProvider metaAccess) { super(TYPE, StampFactory.objectNonNull(TypeReference.createExactTrusted(metaAccess.lookupJavaType(varargs.componentType).getArrayClass()))); this.varargs = varargs; } @Override public ValueNode length() { return ConstantNode.forInt(varargs.length); } } static class CacheKey { private final ResolvedJavaMethod method; private final Object[] values; private final GuardsStage guardsStage; private final LoweringTool.LoweringStage loweringStage; private int hash; protected CacheKey(SnippetInfo info, GuardsStage guardsStage, LoweringTool.LoweringStage loweringStage) { this.method = info.method; this.guardsStage = guardsStage; this.loweringStage = loweringStage; this.values = new Object[info.getParameterCount()]; this.hash = info.method.hashCode() + 31 * guardsStage.ordinal(); } protected void setParam(int paramIdx, Object value) { values[paramIdx] = value; hash = (hash * 31) ^ (value == null ? 0 : value.hashCode()); } @Override public boolean equals(Object obj) { if (!(obj instanceof CacheKey)) { return false; } CacheKey other = (CacheKey) obj; if (!method.equals(other.method)) { return false; } if (guardsStage != other.guardsStage || loweringStage != other.loweringStage) { return false; } for (int i = 0; i < values.length; i++) { if (values[i] != null && !values[i].equals(other.values[i])) { return false; } } return true; } @Override public int hashCode() { return hash; } } private static final DebugTimer SnippetTemplateCreationTime = Debug.timer("SnippetTemplateCreationTime"); private static final DebugCounter SnippetTemplates = Debug.counter("SnippetTemplateCount"); static class Options { @Option(help = "Use a LRU cache for snippet templates.")// static final OptionValue UseSnippetTemplateCache = new OptionValue<>(true); @Option(help = "")// static final OptionValue MaxTemplatesPerSnippet = new OptionValue<>(50); } /** * Base class for snippet classes. It provides a cache for {@link SnippetTemplate}s. */ public abstract static class AbstractTemplates implements org.graalvm.compiler.api.replacements.SnippetTemplateCache { protected final Providers providers; protected final SnippetReflectionProvider snippetReflection; protected final TargetDescription target; private final Map templates; protected AbstractTemplates(Providers providers, SnippetReflectionProvider snippetReflection, TargetDescription target) { this.providers = providers; this.snippetReflection = snippetReflection; this.target = target; if (Options.UseSnippetTemplateCache.getValue()) { int size = Options.MaxTemplatesPerSnippet.getValue(); this.templates = Collections.synchronizedMap(new LRUCache<>(size, size)); } else { this.templates = null; } } public static Method findMethod(Class declaringClass, String methodName, Method except) { for (Method m : declaringClass.getDeclaredMethods()) { if (m.getName().equals(methodName) && !m.equals(except)) { return m; } } return null; } /** * Finds the unique method in {@code declaringClass} named {@code methodName} annotated by * {@link Snippet} and returns a {@link SnippetInfo} value describing it. There must be * exactly one snippet method in {@code declaringClass}. */ protected SnippetInfo snippet(Class declaringClass, String methodName, LocationIdentity... privateLocations) { assert methodName != null; Method method = findMethod(declaringClass, methodName, null); assert method != null : "did not find @" + Snippet.class.getSimpleName() + " method in " + declaringClass + " named " + methodName; assert method.getAnnotation(Snippet.class) != null : method + " must be annotated with @" + Snippet.class.getSimpleName(); assert findMethod(declaringClass, methodName, method) == null : "found more than one method named " + methodName + " in " + declaringClass; ResolvedJavaMethod javaMethod = providers.getMetaAccess().lookupJavaMethod(method); providers.getReplacements().registerSnippet(javaMethod); if (GraalOptions.EagerSnippets.getValue()) { return new EagerSnippetInfo(javaMethod, privateLocations); } else { return new LazySnippetInfo(javaMethod, privateLocations); } } /** * Gets a template for a given key, creating it first if necessary. */ @SuppressWarnings("try") protected SnippetTemplate template(final Arguments args) { SnippetTemplate template = Options.UseSnippetTemplateCache.getValue() && args.cacheable ? templates.get(args.cacheKey) : null; if (template == null) { SnippetTemplates.increment(); try (DebugCloseable a = SnippetTemplateCreationTime.start(); Scope s = Debug.scope("SnippetSpecialization", args.info.method)) { template = new SnippetTemplate(providers, snippetReflection, args); if (Options.UseSnippetTemplateCache.getValue() && args.cacheable) { templates.put(args.cacheKey, template); } } catch (Throwable e) { throw Debug.handle(e); } } return template; } } private static final class LRUCache extends LinkedHashMap { private static final long serialVersionUID = 1L; private final int maxCacheSize; LRUCache(int initialCapacity, int maxCacheSize) { super(initialCapacity, 0.75F, true); this.maxCacheSize = maxCacheSize; } @Override protected boolean removeEldestEntry(java.util.Map.Entry eldest) { return size() > maxCacheSize; } } // These values must be compared with equals() not '==' to support replay compilation. private static final Object UNUSED_PARAMETER = "UNUSED_PARAMETER"; private static final Object CONSTANT_PARAMETER = "CONSTANT_PARAMETER"; /** * Determines if any parameter of a given method is annotated with {@link ConstantParameter}. */ public static boolean hasConstantParameter(ResolvedJavaMethod method) { for (ConstantParameter p : method.getParameterAnnotations(ConstantParameter.class)) { if (p != null) { return true; } } return false; } private final SnippetReflectionProvider snippetReflection; /** * Creates a snippet template. */ @SuppressWarnings("try") protected SnippetTemplate(final Providers providers, SnippetReflectionProvider snippetReflection, Arguments args) { this.snippetReflection = snippetReflection; this.info = args.info; Object[] constantArgs = getConstantArgs(args); StructuredGraph snippetGraph = providers.getReplacements().getSnippet(args.info.method, args.info.original, constantArgs); instantiationTimer = Debug.timer("SnippetTemplateInstantiationTime[%#s]", args); instantiationCounter = Debug.counter("SnippetTemplateInstantiationCount[%#s]", args); ResolvedJavaMethod method = snippetGraph.method(); Signature signature = method.getSignature(); PhaseContext phaseContext = new PhaseContext(providers); // Copy snippet graph, replacing constant parameters with given arguments final StructuredGraph snippetCopy = new StructuredGraph(snippetGraph.name, snippetGraph.method(), AllowAssumptions.NO, INVALID_COMPILATION_ID); try (Debug.Scope scope = Debug.scope("SpecializeSnippet", snippetCopy)) { if (!snippetGraph.isUnsafeAccessTrackingEnabled()) { snippetCopy.disableUnsafeAccessTracking(); } Map nodeReplacements = Node.newIdentityMap(); nodeReplacements.put(snippetGraph.start(), snippetCopy.start()); MetaAccessProvider metaAccess = providers.getMetaAccess(); assert checkTemplate(metaAccess, args, method, signature); int parameterCount = args.info.getParameterCount(); VarargsPlaceholderNode[] placeholders = new VarargsPlaceholderNode[parameterCount]; for (int i = 0; i < parameterCount; i++) { if (args.info.isConstantParameter(i)) { Object arg = args.values[i]; JavaKind kind = signature.getParameterKind(i); ConstantNode constantNode; if (arg instanceof Constant) { Stamp stamp = args.constStamps[i]; if (stamp == null) { assert arg instanceof JavaConstant : "could not determine type of constant " + arg; constantNode = ConstantNode.forConstant((JavaConstant) arg, metaAccess, snippetCopy); } else { constantNode = ConstantNode.forConstant(stamp, (Constant) arg, metaAccess, snippetCopy); } } else { constantNode = ConstantNode.forConstant(snippetReflection.forBoxed(kind, arg), metaAccess, snippetCopy); } nodeReplacements.put(snippetGraph.getParameter(i), constantNode); } else if (args.info.isVarargsParameter(i)) { Varargs varargs = (Varargs) args.values[i]; VarargsPlaceholderNode placeholder = snippetCopy.unique(new VarargsPlaceholderNode(varargs, providers.getMetaAccess())); nodeReplacements.put(snippetGraph.getParameter(i), placeholder); placeholders[i] = placeholder; } } snippetCopy.addDuplicates(snippetGraph.getNodes(), snippetGraph, snippetGraph.getNodeCount(), nodeReplacements); Debug.dump(Debug.INFO_LOG_LEVEL, snippetCopy, "Before specialization"); // Gather the template parameters parameters = new Object[parameterCount]; for (int i = 0; i < parameterCount; i++) { if (args.info.isConstantParameter(i)) { parameters[i] = CONSTANT_PARAMETER; } else if (args.info.isVarargsParameter(i)) { assert snippetCopy.getParameter(i) == null; Varargs varargs = (Varargs) args.values[i]; int length = varargs.length; ParameterNode[] params = new ParameterNode[length]; Stamp stamp = varargs.stamp; for (int j = 0; j < length; j++) { // Use a decimal friendly numbering make it more obvious how values map assert parameterCount < 10000; int idx = (i + 1) * 10000 + j; assert idx >= parameterCount : "collision in parameter numbering"; ParameterNode local = snippetCopy.addOrUnique(new ParameterNode(idx, StampPair.createSingle(stamp))); params[j] = local; } parameters[i] = params; VarargsPlaceholderNode placeholder = placeholders[i]; assert placeholder != null; for (Node usage : placeholder.usages().snapshot()) { if (usage instanceof LoadIndexedNode) { LoadIndexedNode loadIndexed = (LoadIndexedNode) usage; Debug.dump(Debug.INFO_LOG_LEVEL, snippetCopy, "Before replacing %s", loadIndexed); LoadSnippetVarargParameterNode loadSnippetParameter = snippetCopy.add(new LoadSnippetVarargParameterNode(params, loadIndexed.index(), loadIndexed.stamp())); snippetCopy.replaceFixedWithFixed(loadIndexed, loadSnippetParameter); Debug.dump(Debug.INFO_LOG_LEVEL, snippetCopy, "After replacing %s", loadIndexed); } else if (usage instanceof StoreIndexedNode) { /* * The template lowering doesn't really treat this as an array so you * can't store back into the varargs. Allocate your own array if you * really need this and EA should eliminate it. */ throw new GraalError("Can't store into VarargsParameter array"); } } } else { ParameterNode local = snippetCopy.getParameter(i); if (local == null) { // Parameter value was eliminated parameters[i] = UNUSED_PARAMETER; } else { parameters[i] = local; } } } // Do any required loop explosion boolean exploded = false; do { exploded = false; ExplodeLoopNode explodeLoop = snippetCopy.getNodes().filter(ExplodeLoopNode.class).first(); if (explodeLoop != null) { // Earlier canonicalization may have removed the loop // altogether LoopBeginNode loopBegin = explodeLoop.findLoopBegin(); if (loopBegin != null) { LoopEx loop = new LoopsData(snippetCopy).loop(loopBegin); Mark mark = snippetCopy.getMark(); LoopTransformations.fullUnroll(loop, phaseContext, new CanonicalizerPhase()); new CanonicalizerPhase().applyIncremental(snippetCopy, phaseContext, mark); loop.deleteUnusedNodes(); } GraphUtil.removeFixedWithUnusedInputs(explodeLoop); exploded = true; } } while (exploded); GuardsStage guardsStage = args.cacheKey.guardsStage; // Perform lowering on the snippet if (!guardsStage.allowsFloatingGuards()) { new GuardLoweringPhase().apply(snippetCopy, null); } snippetCopy.setGuardsStage(guardsStage); try (Scope s = Debug.scope("LoweringSnippetTemplate", snippetCopy)) { new LoweringPhase(new CanonicalizerPhase(), args.cacheKey.loweringStage).apply(snippetCopy, phaseContext); } catch (Throwable e) { throw Debug.handle(e); } ArrayList curSideEffectNodes = new ArrayList<>(); ArrayList curDeoptNodes = new ArrayList<>(); ArrayList curStampNodes = new ArrayList<>(); for (Node node : snippetCopy.getNodes()) { if (node instanceof ValueNode && ((ValueNode) node).stamp() == StampFactory.forNodeIntrinsic()) { curStampNodes.add((ValueNode) node); } if (node instanceof StateSplit) { StateSplit stateSplit = (StateSplit) node; FrameState frameState = stateSplit.stateAfter(); if (stateSplit.hasSideEffect()) { curSideEffectNodes.add((StateSplit) node); } if (frameState != null) { stateSplit.setStateAfter(null); } } if (node instanceof DeoptimizingNode) { DeoptimizingNode deoptNode = (DeoptimizingNode) node; if (deoptNode.canDeoptimize()) { curDeoptNodes.add(deoptNode); } } } new DeadCodeEliminationPhase(Required).apply(snippetCopy); assert checkAllVarargPlaceholdersAreDeleted(parameterCount, placeholders); new FloatingReadPhase(true, true).apply(snippetCopy); MemoryAnchorNode anchor = snippetCopy.add(new MemoryAnchorNode()); snippetCopy.start().replaceAtUsages(InputType.Memory, anchor); this.snippet = snippetCopy; StartNode entryPointNode = snippet.start(); if (anchor.hasNoUsages()) { anchor.safeDelete(); this.memoryAnchor = null; } else { // Find out if all the return memory maps point to the anchor (i.e., there's no kill // anywhere) boolean needsMemoryMaps = false; for (ReturnNode retNode : snippet.getNodes(ReturnNode.TYPE)) { MemoryMapNode memoryMap = retNode.getMemoryMap(); if (memoryMap.getLocations().size() > 1 || memoryMap.getLastLocationAccess(ANY_LOCATION) != anchor) { needsMemoryMaps = true; break; } } boolean needsAnchor; if (needsMemoryMaps) { needsAnchor = true; } else { // Check that all those memory maps where the only usages of the anchor needsAnchor = anchor.usages().filter(isNotA(MemoryMapNode.class)).isNotEmpty(); // Remove the useless memory map MemoryMapNode memoryMap = null; for (ReturnNode retNode : snippet.getNodes(ReturnNode.TYPE)) { if (memoryMap == null) { memoryMap = retNode.getMemoryMap(); } else { assert memoryMap == retNode.getMemoryMap(); } retNode.setMemoryMap(null); } memoryMap.safeDelete(); } if (needsAnchor) { snippetCopy.addAfterFixed(snippetCopy.start(), anchor); this.memoryAnchor = anchor; } else { anchor.safeDelete(); this.memoryAnchor = null; } } Debug.dump(Debug.INFO_LOG_LEVEL, snippet, "SnippetTemplate after fixing memory anchoring"); List returnNodes = snippet.getNodes(ReturnNode.TYPE).snapshot(); if (returnNodes.isEmpty()) { this.returnNode = null; } else if (returnNodes.size() == 1) { this.returnNode = returnNodes.get(0); } else { AbstractMergeNode merge = snippet.add(new MergeNode()); List memMaps = returnNodes.stream().map(ReturnNode::getMemoryMap).filter(Objects::nonNull).collect(Collectors.toList()); ValueNode returnValue = InliningUtil.mergeReturns(merge, returnNodes, null); this.returnNode = snippet.add(new ReturnNode(returnValue)); if (!memMaps.isEmpty()) { MemoryMapImpl mmap = FloatingReadPhase.mergeMemoryMaps(merge, memMaps); MemoryMapNode memoryMap = snippet.unique(new MemoryMapNode(mmap.getMap())); this.returnNode.setMemoryMap(memoryMap); for (MemoryMapNode mm : memMaps) { if (mm != memoryMap && mm.isAlive()) { assert mm.hasNoUsages(); GraphUtil.killWithUnusedFloatingInputs(mm); } } } merge.setNext(this.returnNode); } this.sideEffectNodes = curSideEffectNodes; this.deoptNodes = curDeoptNodes; this.stampNodes = curStampNodes; nodes = new ArrayList<>(snippet.getNodeCount()); for (Node node : snippet.getNodes()) { if (node != entryPointNode && node != entryPointNode.stateAfter()) { nodes.add(node); } } Debug.counter("SnippetTemplateNodeCount[%#s]", args).add(nodes.size()); Debug.dump(Debug.INFO_LOG_LEVEL, snippet, "SnippetTemplate final state"); } catch (Throwable ex) { throw Debug.handle(ex); } } protected Object[] getConstantArgs(Arguments args) { Object[] constantArgs = args.values.clone(); for (int i = 0; i < args.info.getParameterCount(); i++) { if (!args.info.isConstantParameter(i)) { constantArgs[i] = null; } else { assert constantArgs[i] != null : "Can't pass raw null through as argument"; } } return constantArgs; } private static boolean checkAllVarargPlaceholdersAreDeleted(int parameterCount, VarargsPlaceholderNode[] placeholders) { for (int i = 0; i < parameterCount; i++) { if (placeholders[i] != null) { assert placeholders[i].isDeleted() : placeholders[i]; } } return true; } private static boolean checkConstantArgument(MetaAccessProvider metaAccess, final ResolvedJavaMethod method, Signature signature, int i, String name, Object arg, JavaKind kind) { ResolvedJavaType type = signature.getParameterType(i, method.getDeclaringClass()).resolve(method.getDeclaringClass()); if (metaAccess.lookupJavaType(WordBase.class).isAssignableFrom(type)) { assert arg instanceof JavaConstant : method + ": word constant parameters must be passed boxed in a Constant value: " + arg; return true; } if (kind != JavaKind.Object) { assert arg != null && kind.toBoxedJavaClass() == arg.getClass() : method + ": wrong value kind for " + name + ": expected " + kind + ", got " + (arg == null ? "null" : arg.getClass().getSimpleName()); } return true; } private static boolean checkVarargs(MetaAccessProvider metaAccess, final ResolvedJavaMethod method, Signature signature, int i, String name, Varargs varargs) { ResolvedJavaType type = (ResolvedJavaType) signature.getParameterType(i, method.getDeclaringClass()); assert type.isArray() : "varargs parameter must be an array type"; assert type.getComponentType().isAssignableFrom(metaAccess.lookupJavaType(varargs.componentType)) : "componentType for " + name + " not matching " + type.toJavaName() + " instance: " + varargs.componentType; return true; } /** * The graph built from the snippet method. */ private final StructuredGraph snippet; private final SnippetInfo info; /** * The named parameters of this template that must be bound to values during instantiation. For * a parameter that is still live after specialization, the value in this map is either a * {@link ParameterNode} instance or a {@link ParameterNode} array. For an eliminated parameter, * the value is identical to the key. */ private final Object[] parameters; /** * The return node (if any) of the snippet. */ private final ReturnNode returnNode; /** * The memory anchor (if any) of the snippet. */ private final MemoryAnchorNode memoryAnchor; /** * Nodes that inherit the {@link StateSplit#stateAfter()} from the replacee during * instantiation. */ private final ArrayList sideEffectNodes; /** * Nodes that inherit a deoptimization {@link FrameState} from the replacee during * instantiation. */ private final ArrayList deoptNodes; /** * The nodes that inherit the {@link ValueNode#stamp()} from the replacee during instantiation. */ private final ArrayList stampNodes; /** * The nodes to be inlined when this specialization is instantiated. */ private final ArrayList nodes; /** * Times instantiations of this template. * * @see SnippetInfo#instantiationTimer */ private final DebugTimer instantiationTimer; /** * Counts instantiations of this template. * * @see SnippetInfo#instantiationCounter */ private final DebugCounter instantiationCounter; /** * Gets the instantiation-time bindings to this template's parameters. * * @return the map that will be used to bind arguments to parameters when inlining this template */ private Map bind(StructuredGraph replaceeGraph, MetaAccessProvider metaAccess, Arguments args) { Map replacements = Node.newIdentityMap(); assert args.info.getParameterCount() == parameters.length : "number of args (" + args.info.getParameterCount() + ") != number of parameters (" + parameters.length + ")"; for (int i = 0; i < parameters.length; i++) { Object parameter = parameters[i]; assert parameter != null : this + " has no parameter named " + args.info.getParameterName(i); Object argument = args.values[i]; if (parameter instanceof ParameterNode) { if (argument instanceof ValueNode) { replacements.put((ParameterNode) parameter, (ValueNode) argument); } else { JavaKind kind = ((ParameterNode) parameter).getStackKind(); assert argument != null || kind == JavaKind.Object : this + " cannot accept null for non-object parameter named " + args.info.getParameterName(i); JavaConstant constant = forBoxed(argument, kind); replacements.put((ParameterNode) parameter, ConstantNode.forConstant(constant, metaAccess, replaceeGraph)); } } else if (parameter instanceof ParameterNode[]) { ParameterNode[] params = (ParameterNode[]) parameter; Varargs varargs = (Varargs) argument; int length = params.length; List list = null; Object array = null; if (varargs.value instanceof List) { list = (List) varargs.value; assert list.size() == length : length + " != " + list.size(); } else { array = varargs.value; assert array != null && array.getClass().isArray(); assert Array.getLength(array) == length : length + " != " + Array.getLength(array); } for (int j = 0; j < length; j++) { ParameterNode param = params[j]; assert param != null; Object value = list != null ? list.get(j) : Array.get(array, j); if (value instanceof ValueNode) { replacements.put(param, (ValueNode) value); } else { JavaConstant constant = forBoxed(value, param.getStackKind()); ConstantNode element = ConstantNode.forConstant(constant, metaAccess, replaceeGraph); replacements.put(param, element); } } } else { assert parameter.equals(CONSTANT_PARAMETER) || parameter.equals(UNUSED_PARAMETER) : "unexpected entry for parameter: " + args.info.getParameterName(i) + " -> " + parameter; } } return replacements; } /** * Converts a Java boxed value to a {@link JavaConstant} of the right kind. This adjusts for the * limitation that a {@link Local}'s kind is a {@linkplain JavaKind#getStackKind() stack kind} * and so cannot be used for re-boxing primitives smaller than an int. * * @param argument a Java boxed value * @param localKind the kind of the {@link Local} to which {@code argument} will be bound */ protected JavaConstant forBoxed(Object argument, JavaKind localKind) { assert localKind == localKind.getStackKind(); if (localKind == JavaKind.Int) { return JavaConstant.forBoxedPrimitive(argument); } return snippetReflection.forBoxed(localKind, argument); } /** * Logic for replacing a snippet-lowered node at its usages with the return value of the * snippet. An alternative to the {@linkplain SnippetTemplate#DEFAULT_REPLACER default} * replacement logic can be used to handle mismatches between the stamp of the node being * lowered and the stamp of the snippet's return value. */ public interface UsageReplacer { /** * Replaces all usages of {@code oldNode} with direct or indirect usages of {@code newNode}. */ void replace(ValueNode oldNode, ValueNode newNode); } /** * Represents the default {@link UsageReplacer usage replacer} logic which simply delegates to * {@link Node#replaceAtUsages(Node)}. */ public static final UsageReplacer DEFAULT_REPLACER = new UsageReplacer() { @Override public void replace(ValueNode oldNode, ValueNode newNode) { if (newNode == null) { assert oldNode.hasNoUsages(); } else { oldNode.replaceAtUsages(newNode); } } }; private boolean assertSnippetKills(ValueNode replacee) { if (!replacee.graph().isAfterFloatingReadPhase()) { // no floating reads yet, ignore locations created while lowering return true; } if (returnNode == null) { // The snippet terminates control flow return true; } MemoryMapNode memoryMap = returnNode.getMemoryMap(); if (memoryMap == null || memoryMap.isEmpty()) { // there are no kills in the snippet graph return true; } Set kills = new HashSet<>(memoryMap.getLocations()); if (replacee instanceof MemoryCheckpoint.Single) { // check if some node in snippet graph also kills the same location LocationIdentity locationIdentity = ((MemoryCheckpoint.Single) replacee).getLocationIdentity(); if (locationIdentity.isAny()) { assert !(memoryMap.getLastLocationAccess(any()) instanceof MemoryAnchorNode) : replacee + " kills ANY_LOCATION, but snippet does not"; // if the replacee kills ANY_LOCATION, the snippet can kill arbitrary locations return true; } assert kills.contains(locationIdentity) : replacee + " kills " + locationIdentity + ", but snippet doesn't contain a kill to this location"; kills.remove(locationIdentity); } assert !(replacee instanceof MemoryCheckpoint.Multi) : replacee + " multi not supported (yet)"; // remove ANY_LOCATION if it's just a kill by the start node if (memoryMap.getLastLocationAccess(any()) instanceof MemoryAnchorNode) { kills.remove(any()); } // node can only lower to a ANY_LOCATION kill if the replacee also kills ANY_LOCATION assert !kills.contains(any()) : "snippet graph contains a kill to ANY_LOCATION, but replacee (" + replacee + ") doesn't kill ANY_LOCATION. kills: " + kills; /* * Kills to private locations are safe, since there can be no floating read to these * locations except reads that are introduced by the snippet itself or related snippets in * the same lowering round. These reads are anchored to a MemoryAnchor at the beginning of * their snippet, so they can not float above a kill in another instance of the same * snippet. */ for (LocationIdentity p : this.info.privateLocations) { kills.remove(p); } assert kills.isEmpty() : "snippet graph kills non-private locations " + Arrays.toString(kills.toArray()) + " that replacee (" + replacee + ") doesn't kill"; return true; } private static class MemoryInputMap implements MemoryMap { private final LocationIdentity locationIdentity; private final MemoryNode lastLocationAccess; MemoryInputMap(ValueNode replacee) { if (replacee instanceof MemoryAccess) { MemoryAccess access = (MemoryAccess) replacee; locationIdentity = access.getLocationIdentity(); lastLocationAccess = access.getLastLocationAccess(); } else { locationIdentity = null; lastLocationAccess = null; } } @Override public MemoryNode getLastLocationAccess(LocationIdentity location) { if (locationIdentity != null && locationIdentity.equals(location)) { return lastLocationAccess; } else { return null; } } @Override public Collection getLocations() { if (locationIdentity == null) { return Collections.emptySet(); } else { return Collections.singleton(locationIdentity); } } } private class MemoryOutputMap extends MemoryInputMap { private final Map duplicates; MemoryOutputMap(ValueNode replacee, Map duplicates) { super(replacee); this.duplicates = duplicates; } @Override public MemoryNode getLastLocationAccess(LocationIdentity locationIdentity) { MemoryMapNode memoryMap = returnNode.getMemoryMap(); assert memoryMap != null : "no memory map stored for this snippet graph (snippet doesn't have a ReturnNode?)"; MemoryNode lastLocationAccess = memoryMap.getLastLocationAccess(locationIdentity); assert lastLocationAccess != null : locationIdentity; if (lastLocationAccess == memoryAnchor) { return super.getLastLocationAccess(locationIdentity); } else { return (MemoryNode) duplicates.get(ValueNodeUtil.asNode(lastLocationAccess)); } } @Override public Collection getLocations() { return returnNode.getMemoryMap().getLocations(); } } private void rewireMemoryGraph(ValueNode replacee, Map duplicates) { if (replacee.graph().isAfterFloatingReadPhase()) { // rewire outgoing memory edges replaceMemoryUsages(replacee, new MemoryOutputMap(replacee, duplicates)); if (returnNode != null) { ReturnNode ret = (ReturnNode) duplicates.get(returnNode); if (ret != null) { MemoryMapNode memoryMap = ret.getMemoryMap(); if (memoryMap != null) { ret.setMemoryMap(null); memoryMap.safeDelete(); } } } if (memoryAnchor != null) { // rewire incoming memory edges MemoryAnchorNode memoryDuplicate = (MemoryAnchorNode) duplicates.get(memoryAnchor); replaceMemoryUsages(memoryDuplicate, new MemoryInputMap(replacee)); if (memoryDuplicate.hasNoUsages()) { if (memoryDuplicate.next() != null) { memoryDuplicate.graph().removeFixed(memoryDuplicate); } else { // this was a dummy memory node used when instantiating pure data-flow // snippets: it was not attached to the control flow. memoryDuplicate.safeDelete(); } } } } } private static LocationIdentity getLocationIdentity(Node node) { if (node instanceof MemoryAccess) { return ((MemoryAccess) node).getLocationIdentity(); } else if (node instanceof MemoryProxy) { return ((MemoryProxy) node).getLocationIdentity(); } else if (node instanceof MemoryPhiNode) { return ((MemoryPhiNode) node).getLocationIdentity(); } else { return null; } } private void replaceMemoryUsages(ValueNode node, MemoryMap map) { for (Node usage : node.usages().snapshot()) { if (usage instanceof MemoryMapNode) { continue; } LocationIdentity location = getLocationIdentity(usage); if (location != null) { for (Position pos : usage.inputPositions()) { if (pos.getInputType() == InputType.Memory && pos.get(usage) == node) { MemoryNode replacement = map.getLastLocationAccess(location); if (replacement == null) { assert mayRemoveLocation || LocationIdentity.any().equals(location) || Arrays.stream(info.privateLocations).anyMatch(Predicate.isEqual(location)) : "Snippet " + info.method.format("%h.%n") + " contains access to the non-private location " + location + ", but replacee doesn't access this location." + map.getLocations(); } else { pos.set(usage, replacement.asNode()); } } } } } } /** * Replaces a given fixed node with this specialized snippet. * * @param metaAccess * @param replacee the node that will be replaced * @param replacer object that replaces the usages of {@code replacee} * @param args the arguments to be bound to the flattened positional parameters of the snippet * @return the map of duplicated nodes (original -> duplicate) */ @SuppressWarnings("try") public Map instantiate(MetaAccessProvider metaAccess, FixedNode replacee, UsageReplacer replacer, Arguments args) { assert assertSnippetKills(replacee); try (DebugCloseable a = args.info.instantiationTimer.start(); DebugCloseable b = instantiationTimer.start()) { args.info.instantiationCounter.increment(); instantiationCounter.increment(); // Inline the snippet nodes, replacing parameters with the given args in the process StartNode entryPointNode = snippet.start(); FixedNode firstCFGNode = entryPointNode.next(); StructuredGraph replaceeGraph = replacee.graph(); Map replacements = bind(replaceeGraph, metaAccess, args); replacements.put(entryPointNode, AbstractBeginNode.prevBegin(replacee)); Map duplicates = replaceeGraph.addDuplicates(nodes, snippet, snippet.getNodeCount(), replacements); Debug.dump(Debug.INFO_LOG_LEVEL, replaceeGraph, "After inlining snippet %s", snippet.method()); // Re-wire the control flow graph around the replacee FixedNode firstCFGNodeDuplicate = (FixedNode) duplicates.get(firstCFGNode); replacee.replaceAtPredecessor(firstCFGNodeDuplicate); rewireFrameStates(replacee, duplicates); if (replacee instanceof DeoptimizingNode) { DeoptimizingNode replaceeDeopt = (DeoptimizingNode) replacee; FrameState stateBefore = null; FrameState stateDuring = null; FrameState stateAfter = null; if (replaceeDeopt.canDeoptimize()) { if (replaceeDeopt instanceof DeoptimizingNode.DeoptBefore) { stateBefore = ((DeoptimizingNode.DeoptBefore) replaceeDeopt).stateBefore(); } if (replaceeDeopt instanceof DeoptimizingNode.DeoptDuring) { stateDuring = ((DeoptimizingNode.DeoptDuring) replaceeDeopt).stateDuring(); } if (replaceeDeopt instanceof DeoptimizingNode.DeoptAfter) { stateAfter = ((DeoptimizingNode.DeoptAfter) replaceeDeopt).stateAfter(); } } for (DeoptimizingNode deoptNode : deoptNodes) { DeoptimizingNode deoptDup = (DeoptimizingNode) duplicates.get(deoptNode); if (deoptDup.canDeoptimize()) { if (deoptDup instanceof DeoptimizingNode.DeoptBefore) { ((DeoptimizingNode.DeoptBefore) deoptDup).setStateBefore(stateBefore); } if (deoptDup instanceof DeoptimizingNode.DeoptDuring) { DeoptimizingNode.DeoptDuring deoptDupDuring = (DeoptimizingNode.DeoptDuring) deoptDup; if (stateDuring != null) { deoptDupDuring.setStateDuring(stateDuring); } else if (stateAfter != null) { deoptDupDuring.computeStateDuring(stateAfter); } else if (stateBefore != null) { assert !deoptDupDuring.hasSideEffect() : "can't use stateBefore as stateDuring for state split " + deoptDupDuring; deoptDupDuring.setStateDuring(stateBefore); } } if (deoptDup instanceof DeoptimizingNode.DeoptAfter) { DeoptimizingNode.DeoptAfter deoptDupAfter = (DeoptimizingNode.DeoptAfter) deoptDup; if (stateAfter != null) { deoptDupAfter.setStateAfter(stateAfter); } else { assert !deoptDupAfter.hasSideEffect() : "can't use stateBefore as stateAfter for state split " + deoptDupAfter; deoptDupAfter.setStateAfter(stateBefore); } } } } } updateStamps(replacee, duplicates); if (UseGraalInstrumentation.getValue()) { for (InstrumentationNode instrumentation : replaceeGraph.getNodes().filter(InstrumentationNode.class)) { if (instrumentation.getTarget() == replacee) { instrumentation.replaceFirstInput(replacee, firstCFGNodeDuplicate); } } } rewireMemoryGraph(replacee, duplicates); // Replace all usages of the replacee with the value returned by the snippet ValueNode returnValue = null; if (returnNode != null && !(replacee instanceof ControlSinkNode)) { ReturnNode returnDuplicate = (ReturnNode) duplicates.get(returnNode); returnValue = returnDuplicate.result(); if (returnValue == null && replacee.usages().isNotEmpty() && replacee instanceof MemoryCheckpoint) { replacer.replace(replacee, null); } else { assert returnValue != null || replacee.hasNoUsages(); replacer.replace(replacee, returnValue); } if (returnDuplicate.isAlive()) { FixedNode next = null; if (replacee instanceof FixedWithNextNode) { FixedWithNextNode fwn = (FixedWithNextNode) replacee; next = fwn.next(); fwn.setNext(null); } returnDuplicate.replaceAndDelete(next); } } // Remove the replacee from its graph GraphUtil.killCFG(replacee); Debug.dump(Debug.INFO_LOG_LEVEL, replaceeGraph, "After lowering %s with %s", replacee, this); return duplicates; } } private void propagateStamp(Node node) { if (node instanceof PhiNode) { PhiNode phi = (PhiNode) node; if (phi.inferStamp()) { for (Node usage : node.usages()) { propagateStamp(usage); } } } } private void updateStamps(ValueNode replacee, Map duplicates) { for (ValueNode stampNode : stampNodes) { Node stampDup = duplicates.get(stampNode); ((ValueNode) stampDup).setStamp(replacee.stamp()); } for (ParameterNode paramNode : snippet.getNodes(ParameterNode.TYPE)) { for (Node usage : paramNode.usages()) { Node usageDup = duplicates.get(usage); propagateStamp(usageDup); } } } /** * Gets a copy of the specialized graph. */ public StructuredGraph copySpecializedGraph() { return (StructuredGraph) snippet.copy(); } /** * Replaces a given floating node with this specialized snippet. * * @param metaAccess * @param replacee the node that will be replaced * @param replacer object that replaces the usages of {@code replacee} * @param tool lowering tool used to insert the snippet into the control-flow * @param args the arguments to be bound to the flattened positional parameters of the snippet */ @SuppressWarnings("try") public void instantiate(MetaAccessProvider metaAccess, FloatingNode replacee, UsageReplacer replacer, LoweringTool tool, Arguments args) { assert assertSnippetKills(replacee); try (DebugCloseable a = args.info.instantiationTimer.start()) { args.info.instantiationCounter.increment(); instantiationCounter.increment(); // Inline the snippet nodes, replacing parameters with the given args in the process StartNode entryPointNode = snippet.start(); FixedNode firstCFGNode = entryPointNode.next(); StructuredGraph replaceeGraph = replacee.graph(); Map replacements = bind(replaceeGraph, metaAccess, args); replacements.put(entryPointNode, tool.getCurrentGuardAnchor().asNode()); Map duplicates = replaceeGraph.addDuplicates(nodes, snippet, snippet.getNodeCount(), replacements); Debug.dump(Debug.INFO_LOG_LEVEL, replaceeGraph, "After inlining snippet %s", snippet.method()); FixedWithNextNode lastFixedNode = tool.lastFixedNode(); assert lastFixedNode != null && lastFixedNode.isAlive() : replaceeGraph + " lastFixed=" + lastFixedNode; FixedNode next = lastFixedNode.next(); lastFixedNode.setNext(null); FixedNode firstCFGNodeDuplicate = (FixedNode) duplicates.get(firstCFGNode); replaceeGraph.addAfterFixed(lastFixedNode, firstCFGNodeDuplicate); rewireFrameStates(replacee, duplicates); updateStamps(replacee, duplicates); rewireMemoryGraph(replacee, duplicates); // Replace all usages of the replacee with the value returned by the snippet ReturnNode returnDuplicate = (ReturnNode) duplicates.get(returnNode); ValueNode returnValue = returnDuplicate.result(); assert returnValue != null || replacee.hasNoUsages(); replacer.replace(replacee, returnValue); if (returnDuplicate.isAlive()) { returnDuplicate.replaceAndDelete(next); } Debug.dump(Debug.INFO_LOG_LEVEL, replaceeGraph, "After lowering %s with %s", replacee, this); } } /** * Replaces a given floating node with this specialized snippet. * * This snippet must be pure data-flow * * @param metaAccess * @param replacee the node that will be replaced * @param replacer object that replaces the usages of {@code replacee} * @param args the arguments to be bound to the flattened positional parameters of the snippet */ @SuppressWarnings("try") public void instantiate(MetaAccessProvider metaAccess, FloatingNode replacee, UsageReplacer replacer, Arguments args) { assert assertSnippetKills(replacee); try (DebugCloseable a = args.info.instantiationTimer.start()) { args.info.instantiationCounter.increment(); instantiationCounter.increment(); // Inline the snippet nodes, replacing parameters with the given args in the process StartNode entryPointNode = snippet.start(); assert entryPointNode.next() == (memoryAnchor == null ? returnNode : memoryAnchor) : entryPointNode.next(); StructuredGraph replaceeGraph = replacee.graph(); Map replacements = bind(replaceeGraph, metaAccess, args); MemoryAnchorNode anchorDuplicate = null; if (memoryAnchor != null) { anchorDuplicate = replaceeGraph.add(new MemoryAnchorNode()); replacements.put(memoryAnchor, anchorDuplicate); } List floatingNodes = new ArrayList<>(nodes.size() - 2); for (Node n : nodes) { if (n != entryPointNode && n != returnNode) { floatingNodes.add(n); } } Map duplicates = replaceeGraph.addDuplicates(floatingNodes, snippet, floatingNodes.size(), replacements); Debug.dump(Debug.INFO_LOG_LEVEL, replaceeGraph, "After inlining snippet %s", snippet.method()); rewireFrameStates(replacee, duplicates); updateStamps(replacee, duplicates); rewireMemoryGraph(replacee, duplicates); assert anchorDuplicate == null || anchorDuplicate.isDeleted(); // Replace all usages of the replacee with the value returned by the snippet ValueNode returnValue = (ValueNode) duplicates.get(returnNode.result()); replacer.replace(replacee, returnValue); Debug.dump(Debug.INFO_LOG_LEVEL, replaceeGraph, "After lowering %s with %s", replacee, this); } } protected void rewireFrameStates(ValueNode replacee, Map duplicates) { if (replacee instanceof StateSplit) { for (StateSplit sideEffectNode : sideEffectNodes) { assert ((StateSplit) replacee).hasSideEffect(); Node sideEffectDup = duplicates.get(sideEffectNode); ((StateSplit) sideEffectDup).setStateAfter(((StateSplit) replacee).stateAfter()); } } } @Override public String toString() { StringBuilder buf = new StringBuilder(snippet.toString()).append('('); String sep = ""; for (int i = 0; i < parameters.length; i++) { String name = "[" + i + "]"; Object value = parameters[i]; buf.append(sep); sep = ", "; if (value == null) { buf.append(" ").append(name); } else if (value.equals(UNUSED_PARAMETER)) { buf.append(" ").append(name); } else if (value.equals(CONSTANT_PARAMETER)) { buf.append(" ").append(name); } else if (value instanceof ParameterNode) { ParameterNode param = (ParameterNode) value; buf.append(param.getStackKind().getJavaName()).append(' ').append(name); } else { ParameterNode[] params = (ParameterNode[]) value; String kind = params.length == 0 ? "?" : params[0].getStackKind().getJavaName(); buf.append(kind).append('[').append(params.length).append("] ").append(name); } } return buf.append(')').toString(); } private static boolean checkTemplate(MetaAccessProvider metaAccess, Arguments args, ResolvedJavaMethod method, Signature signature) { for (int i = 0; i < args.info.getParameterCount(); i++) { if (args.info.isConstantParameter(i)) { JavaKind kind = signature.getParameterKind(i); assert checkConstantArgument(metaAccess, method, signature, i, args.info.getParameterName(i), args.values[i], kind); } else if (args.info.isVarargsParameter(i)) { assert args.values[i] instanceof Varargs; Varargs varargs = (Varargs) args.values[i]; assert checkVarargs(metaAccess, method, signature, i, args.info.getParameterName(i), varargs); } } return true; } public void setMayRemoveLocation(boolean mayRemoveLocation) { this.mayRemoveLocation = mayRemoveLocation; } }