1 /*
2 * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24
25 package org.graalvm.compiler.replacements;
26
27 import static java.util.FormattableFlags.ALTERNATE;
28 import static org.graalvm.compiler.debug.DebugContext.DEFAULT_LOG_STREAM;
29 import static org.graalvm.compiler.debug.DebugContext.applyFormattingFlagsAndWidth;
30 import static org.graalvm.compiler.debug.DebugOptions.DebugStubsAndSnippets;
31 import static org.graalvm.compiler.graph.iterators.NodePredicates.isNotA;
32 import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_IGNORED;
33 import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_IGNORED;
34 import static org.graalvm.compiler.phases.common.DeadCodeEliminationPhase.Optionality.Required;
35 import static jdk.internal.vm.compiler.word.LocationIdentity.any;
36
37 import java.lang.reflect.Array;
38 import java.lang.reflect.Method;
39 import java.util.ArrayList;
40 import java.util.Arrays;
41 import java.util.Collection;
42 import java.util.Collections;
43 import java.util.Formattable;
44 import java.util.Formatter;
45 import java.util.LinkedHashMap;
46 import java.util.List;
47 import java.util.Map;
48 import java.util.concurrent.atomic.AtomicInteger;
49 import java.util.concurrent.atomic.AtomicReference;
50 import java.util.function.Predicate;
51
52 import jdk.internal.vm.compiler.collections.EconomicMap;
53 import jdk.internal.vm.compiler.collections.EconomicSet;
54 import jdk.internal.vm.compiler.collections.Equivalence;
55 import jdk.internal.vm.compiler.collections.UnmodifiableEconomicMap;
56 import org.graalvm.compiler.api.replacements.Fold;
57 import org.graalvm.compiler.api.replacements.Snippet;
58 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
59 import org.graalvm.compiler.api.replacements.Snippet.NonNullParameter;
60 import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter;
61 import org.graalvm.compiler.api.replacements.SnippetReflectionProvider;
62 import org.graalvm.compiler.core.common.GraalOptions;
63 import org.graalvm.compiler.core.common.type.Stamp;
64 import org.graalvm.compiler.core.common.type.StampFactory;
65 import org.graalvm.compiler.core.common.type.StampPair;
66 import org.graalvm.compiler.core.common.type.TypeReference;
67 import org.graalvm.compiler.debug.Assertions;
68 import org.graalvm.compiler.debug.CounterKey;
69 import org.graalvm.compiler.debug.DebugCloseable;
70 import org.graalvm.compiler.debug.DebugContext;
71 import org.graalvm.compiler.debug.DebugContext.Description;
72 import org.graalvm.compiler.debug.DebugHandlersFactory;
73 import org.graalvm.compiler.debug.GraalError;
74 import org.graalvm.compiler.debug.TimerKey;
75 import org.graalvm.compiler.graph.Graph.Mark;
76 import org.graalvm.compiler.graph.Node;
77 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
78 import org.graalvm.compiler.graph.NodeClass;
79 import org.graalvm.compiler.graph.NodeSourcePosition;
80 import org.graalvm.compiler.graph.Position;
81 import org.graalvm.compiler.loop.LoopEx;
82 import org.graalvm.compiler.loop.LoopsData;
83 import org.graalvm.compiler.loop.phases.LoopTransformations;
84 import org.graalvm.compiler.nodeinfo.InputType;
85 import org.graalvm.compiler.nodeinfo.NodeInfo;
86 import org.graalvm.compiler.nodes.AbstractBeginNode;
87 import org.graalvm.compiler.nodes.AbstractMergeNode;
88 import org.graalvm.compiler.nodes.ConstantNode;
89 import org.graalvm.compiler.nodes.ControlSinkNode;
90 import org.graalvm.compiler.nodes.DeoptimizingNode;
91 import org.graalvm.compiler.nodes.FixedNode;
92 import org.graalvm.compiler.nodes.FixedWithNextNode;
93 import org.graalvm.compiler.nodes.FrameState;
94 import org.graalvm.compiler.nodes.InliningLog;
95 import org.graalvm.compiler.nodes.LoopBeginNode;
96 import org.graalvm.compiler.nodes.MergeNode;
97 import org.graalvm.compiler.nodes.NodeView;
98 import org.graalvm.compiler.nodes.ParameterNode;
99 import org.graalvm.compiler.nodes.PhiNode;
100 import org.graalvm.compiler.nodes.PiNode.Placeholder;
101 import org.graalvm.compiler.nodes.PiNode.PlaceholderStamp;
102 import org.graalvm.compiler.nodes.ReturnNode;
103 import org.graalvm.compiler.nodes.StartNode;
104 import org.graalvm.compiler.nodes.StateSplit;
105 import org.graalvm.compiler.nodes.StructuredGraph;
106 import org.graalvm.compiler.nodes.StructuredGraph.GuardsStage;
107 import org.graalvm.compiler.nodes.ValueNode;
108 import org.graalvm.compiler.nodes.ValueNodeUtil;
109 import org.graalvm.compiler.nodes.calc.FloatingNode;
110 import org.graalvm.compiler.nodes.java.LoadIndexedNode;
111 import org.graalvm.compiler.nodes.java.MethodCallTargetNode;
112 import org.graalvm.compiler.nodes.java.StoreIndexedNode;
113 import org.graalvm.compiler.nodes.memory.MemoryAccess;
114 import org.graalvm.compiler.nodes.memory.MemoryAnchorNode;
115 import org.graalvm.compiler.nodes.memory.MemoryCheckpoint;
116 import org.graalvm.compiler.nodes.memory.MemoryMap;
117 import org.graalvm.compiler.nodes.memory.MemoryMapNode;
118 import org.graalvm.compiler.nodes.memory.MemoryNode;
119 import org.graalvm.compiler.nodes.memory.MemoryPhiNode;
120 import org.graalvm.compiler.nodes.spi.ArrayLengthProvider;
121 import org.graalvm.compiler.nodes.spi.LoweringTool;
122 import org.graalvm.compiler.nodes.spi.MemoryProxy;
123 import org.graalvm.compiler.nodes.util.GraphUtil;
124 import org.graalvm.compiler.options.Option;
125 import org.graalvm.compiler.options.OptionKey;
126 import org.graalvm.compiler.options.OptionValues;
127 import org.graalvm.compiler.phases.common.CanonicalizerPhase;
128 import org.graalvm.compiler.phases.common.DeadCodeEliminationPhase;
129 import org.graalvm.compiler.phases.common.FloatingReadPhase;
130 import org.graalvm.compiler.phases.common.FloatingReadPhase.MemoryMapImpl;
131 import org.graalvm.compiler.phases.common.GuardLoweringPhase;
132 import org.graalvm.compiler.phases.common.LoweringPhase;
133 import org.graalvm.compiler.phases.common.RemoveValueProxyPhase;
134 import org.graalvm.compiler.phases.common.inlining.InliningUtil;
135 import org.graalvm.compiler.phases.tiers.PhaseContext;
136 import org.graalvm.compiler.phases.util.Providers;
137 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
138 import org.graalvm.compiler.replacements.nodes.LoadSnippetVarargParameterNode;
139 import org.graalvm.util.CollectionsUtil;
140 import jdk.internal.vm.compiler.word.LocationIdentity;
141 import jdk.internal.vm.compiler.word.WordBase;
142
143 import jdk.vm.ci.code.TargetDescription;
144 import jdk.vm.ci.meta.Constant;
145 import jdk.vm.ci.meta.ConstantReflectionProvider;
146 import jdk.vm.ci.meta.JavaConstant;
147 import jdk.vm.ci.meta.JavaKind;
148 import jdk.vm.ci.meta.Local;
149 import jdk.vm.ci.meta.LocalVariableTable;
150 import jdk.vm.ci.meta.MetaAccessProvider;
151 import jdk.vm.ci.meta.ResolvedJavaMethod;
152 import jdk.vm.ci.meta.ResolvedJavaMethod.Parameter;
153 import jdk.vm.ci.meta.ResolvedJavaType;
154 import jdk.vm.ci.meta.Signature;
155
156 /**
157 * A snippet template is a graph created by parsing a snippet method and then specialized by binding
158 * constants to the snippet's {@link ConstantParameter} parameters.
159 *
160 * Snippet templates can be managed in a cache maintained by {@link AbstractTemplates}.
161 */
162 public class SnippetTemplate {
163
164 private boolean mayRemoveLocation = false;
165
166 /**
167 * Holds the {@link ResolvedJavaMethod} of the snippet, together with some information about the
168 * method that needs to be computed only once. The {@link SnippetInfo} should be created once
169 * per snippet and then cached.
170 */
171 public abstract static class SnippetInfo {
172
173 protected final ResolvedJavaMethod method;
174 protected final ResolvedJavaMethod original;
175 protected final LocationIdentity[] privateLocations;
176 protected final Object receiver;
177
178 public Object getReceiver() {
179 return receiver;
180 }
181
182 boolean hasReceiver() {
183 assert hasReceiver(method) == (receiver != null) : "Snippet with the receiver must have it set as constant. Snippet: " + this;
184 return hasReceiver(method);
185 }
186
187 static boolean hasReceiver(ResolvedJavaMethod method) {
188 return method.hasReceiver();
189 }
190
191 /**
192 * Lazily constructed parts of {@link SnippetInfo}.
193 */
194 static class Lazy {
195 Lazy(ResolvedJavaMethod method) {
196 int count = method.getSignature().getParameterCount(hasReceiver(method));
197 constantParameters = new boolean[count];
198 varargsParameters = new boolean[count];
199 nonNullParameters = new boolean[count];
200 int offset = hasReceiver(method) ? 1 : 0;
201 for (int i = offset; i < count; i++) {
202 constantParameters[i] = method.getParameterAnnotation(ConstantParameter.class, i - offset) != null;
203 varargsParameters[i] = method.getParameterAnnotation(VarargsParameter.class, i - offset) != null;
204 nonNullParameters[i] = method.getParameterAnnotation(NonNullParameter.class, i - offset) != null;
205
206 assert !constantParameters[i - offset] || !varargsParameters[i - offset] : "Parameter cannot be annotated with both @" + ConstantParameter.class.getSimpleName() + " and @" +
207 VarargsParameter.class.getSimpleName();
208 }
209 if (method.hasReceiver()) {
210 // Receiver must be constant.
211 assert !constantParameters[0];
212 constantParameters[0] = true;
213 }
214
215 // Retrieve the names only when assertions are turned on.
216 assert initNames(method, count);
217 }
218
219 final boolean[] constantParameters;
220 final boolean[] varargsParameters;
221 final boolean[] nonNullParameters;
222
223 /**
224 * The parameter names, taken from the local variables table. Only used for assertion
225 * checking, so use only within an assert statement.
226 */
227 String[] names;
228
229 private boolean initNames(ResolvedJavaMethod method, int parameterCount) {
230 names = new String[parameterCount];
231 int offset = 0;
232 if (method.hasReceiver()) {
233 names[0] = "this";
234 offset = 1;
235 }
236 Parameter[] params = method.getParameters();
237 if (params != null) {
238 for (int i = offset; i < names.length; i++) {
239 if (params[i - offset].isNamePresent()) {
240 names[i] = params[i - offset].getName();
241 }
242 }
243 } else {
244 int slotIdx = 0;
245 LocalVariableTable localVariableTable = method.getLocalVariableTable();
246 if (localVariableTable != null) {
247 for (int i = 0; i < names.length; i++) {
248 Local local = localVariableTable.getLocal(slotIdx, 0);
249 if (local != null) {
250 names[i] = local.getName();
251 }
252 JavaKind kind = method.getSignature().getParameterKind(i);
253 slotIdx += kind.getSlotCount();
254 }
255 }
256 }
257 return true;
258 }
259 }
260
261 /**
262 * Times instantiations of all templates derived from this snippet.
263 */
264 private final TimerKey instantiationTimer;
265
266 /**
267 * Counts instantiations of all templates derived from this snippet.
268 */
269 private final CounterKey instantiationCounter;
270
271 protected abstract Lazy lazy();
272
273 protected SnippetInfo(ResolvedJavaMethod method, ResolvedJavaMethod original, LocationIdentity[] privateLocations, Object receiver) {
274 this.method = method;
275 this.original = original;
276 this.privateLocations = privateLocations;
277 instantiationCounter = DebugContext.counter("SnippetInstantiationCount[%s]", method.getName());
278 instantiationTimer = DebugContext.timer("SnippetInstantiationTime[%s]", method.getName());
279 this.receiver = receiver;
280 }
281
282 public ResolvedJavaMethod getMethod() {
283 return method;
284 }
285
286 public int getParameterCount() {
287 return lazy().constantParameters.length;
288 }
289
290 public boolean isConstantParameter(int paramIdx) {
291 return lazy().constantParameters[paramIdx];
292 }
293
294 public boolean isVarargsParameter(int paramIdx) {
295 return lazy().varargsParameters[paramIdx];
296 }
297
298 public boolean isNonNullParameter(int paramIdx) {
299 return lazy().nonNullParameters[paramIdx];
300 }
301
302 public String getParameterName(int paramIdx) {
303 String[] names = lazy().names;
304 if (names != null) {
305 return names[paramIdx];
306 }
307 return null;
308 }
309
310 @Override
311 public String toString() {
312 return getClass().getSimpleName() + ":" + method.format("%h.%n");
313 }
314 }
315
316 protected static class LazySnippetInfo extends SnippetInfo {
317 protected final AtomicReference<Lazy> lazy = new AtomicReference<>(null);
318
319 protected LazySnippetInfo(ResolvedJavaMethod method, ResolvedJavaMethod original, LocationIdentity[] privateLocations, Object receiver) {
320 super(method, original, privateLocations, receiver);
321 }
322
323 @Override
324 protected Lazy lazy() {
325 if (lazy.get() == null) {
326 lazy.compareAndSet(null, new Lazy(method));
327 }
328 return lazy.get();
329 }
330 }
331
332 protected static class EagerSnippetInfo extends SnippetInfo {
333 protected final Lazy lazy;
334
335 protected EagerSnippetInfo(ResolvedJavaMethod method, ResolvedJavaMethod original, LocationIdentity[] privateLocations, Object receiver) {
336 super(method, original, privateLocations, receiver);
337 lazy = new Lazy(method);
338 }
339
340 @Override
341 protected Lazy lazy() {
342 return lazy;
343 }
344 }
345
346 /**
347 * Values that are bound to the snippet method parameters. The methods {@link #add},
348 * {@link #addConst}, and {@link #addVarargs} must be called in the same order as in the
349 * signature of the snippet method. The parameter name is passed to the add methods for
350 * assertion checking, i.e., to enforce that the order matches. Which method needs to be called
351 * depends on the annotation of the snippet method parameter:
352 * <ul>
353 * <li>Use {@link #add} for a parameter without an annotation. The value is bound when the
354 * {@link SnippetTemplate} is {@link SnippetTemplate#instantiate instantiated}.
355 * <li>Use {@link #addConst} for a parameter annotated with {@link ConstantParameter}. The value
356 * is bound when the {@link SnippetTemplate} is {@link SnippetTemplate#SnippetTemplate created}.
357 * <li>Use {@link #addVarargs} for an array parameter annotated with {@link VarargsParameter}. A
358 * separate {@link SnippetTemplate} is {@link SnippetTemplate#SnippetTemplate created} for every
359 * distinct array length. The actual values are bound when the {@link SnippetTemplate} is
360 * {@link SnippetTemplate#instantiate instantiated}
361 * </ul>
362 */
363 public static class Arguments implements Formattable {
364
365 protected final SnippetInfo info;
366 protected final CacheKey cacheKey;
367 protected final Object[] values;
368 protected final Stamp[] constStamps;
369 protected boolean cacheable;
370
371 protected int nextParamIdx;
372
373 public Arguments(SnippetInfo info, GuardsStage guardsStage, LoweringTool.LoweringStage loweringStage) {
374 this.info = info;
375 this.cacheKey = new CacheKey(info, guardsStage, loweringStage);
376 this.values = new Object[info.getParameterCount()];
377 this.constStamps = new Stamp[info.getParameterCount()];
378 this.cacheable = true;
379 if (info.hasReceiver()) {
380 addConst("this", info.getReceiver());
381 }
382 }
383
384 public Arguments add(String name, Object value) {
385 assert check(name, false, false);
386 values[nextParamIdx] = value;
387 nextParamIdx++;
388 return this;
389 }
390
391 public Arguments addConst(String name, Object value) {
392 assert value != null;
393 return addConst(name, value, null);
394 }
395
396 public Arguments addConst(String name, Object value, Stamp stamp) {
397 assert check(name, true, false);
398 values[nextParamIdx] = value;
399 constStamps[nextParamIdx] = stamp;
400 cacheKey.setParam(nextParamIdx, value);
401 nextParamIdx++;
402 return this;
403 }
404
405 public Arguments addVarargs(String name, Class<?> componentType, Stamp argStamp, Object value) {
406 assert check(name, false, true);
407 Varargs varargs = new Varargs(componentType, argStamp, value);
408 values[nextParamIdx] = varargs;
409 // A separate template is necessary for every distinct array length
410 cacheKey.setParam(nextParamIdx, varargs.length);
411 nextParamIdx++;
412 return this;
413 }
414
415 public void setCacheable(boolean cacheable) {
416 this.cacheable = cacheable;
417 }
418
419 private boolean check(String name, boolean constParam, boolean varargsParam) {
420 assert nextParamIdx < info.getParameterCount() : "too many parameters: " + name + " " + this;
421 assert info.getParameterName(nextParamIdx) == null || info.getParameterName(nextParamIdx).equals(name) : "wrong parameter name at " + nextParamIdx + " : " + name + " " + this;
422 assert constParam == info.isConstantParameter(nextParamIdx) : "Parameter " + (constParam ? "not " : "") + "annotated with @" + ConstantParameter.class.getSimpleName() + ": " + name +
423 " " + this;
424 assert varargsParam == info.isVarargsParameter(nextParamIdx) : "Parameter " + (varargsParam ? "not " : "") + "annotated with @" + VarargsParameter.class.getSimpleName() + ": " + name +
425 " " + this;
426 return true;
427 }
428
429 @Override
430 public String toString() {
431 StringBuilder result = new StringBuilder();
432 result.append("Parameters<").append(info.method.format("%h.%n")).append(" [");
433 String sep = "";
434 for (int i = 0; i < info.getParameterCount(); i++) {
435 result.append(sep);
436 if (info.isConstantParameter(i)) {
437 result.append("const ");
438 } else if (info.isVarargsParameter(i)) {
439 result.append("varargs ");
440 }
441 result.append(info.getParameterName(i)).append(" = ").append(values[i]);
442 sep = ", ";
443 }
444 result.append(">");
445 return result.toString();
446 }
447
448 @Override
449 public void formatTo(Formatter formatter, int flags, int width, int precision) {
450 if ((flags & ALTERNATE) == 0) {
451 formatter.format(applyFormattingFlagsAndWidth(toString(), flags, width));
452 } else {
453 StringBuilder sb = new StringBuilder();
454 sb.append(info.method.getName()).append('(');
455 String sep = "";
456 for (int i = 0; i < info.getParameterCount(); i++) {
457 if (info.isConstantParameter(i)) {
458 sb.append(sep);
459 if (info.getParameterName(i) != null) {
460 sb.append(info.getParameterName(i));
461 } else {
462 sb.append(i);
463 }
464 sb.append('=').append(values[i]);
465 sep = ", ";
466 }
467 }
468 sb.append(")");
469 String string = sb.toString();
470 if (string.indexOf('%') != -1) {
471 // Quote any % signs
472 string = string.replace("%", "%%");
473 }
474 formatter.format(applyFormattingFlagsAndWidth(string, flags & ~ALTERNATE, width));
475 }
476 }
477 }
478
479 /**
480 * Wrapper for the prototype value of a {@linkplain VarargsParameter varargs} parameter.
481 */
482 static class Varargs {
483
484 protected final Class<?> componentType;
485 protected final Stamp stamp;
486 protected final Object value;
487 protected final int length;
488
489 protected Varargs(Class<?> componentType, Stamp stamp, Object value) {
490 this.componentType = componentType;
491 this.stamp = stamp;
492 this.value = value;
493 if (value instanceof List) {
494 this.length = ((List<?>) value).size();
495 } else {
496 this.length = Array.getLength(value);
497 }
498 }
499
500 @Override
501 public String toString() {
502 if (value instanceof boolean[]) {
503 return Arrays.toString((boolean[]) value);
504 }
505 if (value instanceof byte[]) {
506 return Arrays.toString((byte[]) value);
507 }
508 if (value instanceof char[]) {
509 return Arrays.toString((char[]) value);
510 }
511 if (value instanceof short[]) {
512 return Arrays.toString((short[]) value);
513 }
514 if (value instanceof int[]) {
515 return Arrays.toString((int[]) value);
516 }
517 if (value instanceof long[]) {
518 return Arrays.toString((long[]) value);
519 }
520 if (value instanceof float[]) {
521 return Arrays.toString((float[]) value);
522 }
523 if (value instanceof double[]) {
524 return Arrays.toString((double[]) value);
525 }
526 if (value instanceof Object[]) {
527 return Arrays.toString((Object[]) value);
528 }
529 return String.valueOf(value);
530 }
531 }
532
533 @NodeInfo(cycles = CYCLES_IGNORED, size = SIZE_IGNORED)
534 static final class VarargsPlaceholderNode extends FloatingNode implements ArrayLengthProvider {
535
536 public static final NodeClass<VarargsPlaceholderNode> TYPE = NodeClass.create(VarargsPlaceholderNode.class);
537 protected final Varargs varargs;
538
539 protected VarargsPlaceholderNode(Varargs varargs, MetaAccessProvider metaAccess) {
540 super(TYPE, StampFactory.objectNonNull(TypeReference.createExactTrusted(metaAccess.lookupJavaType(varargs.componentType).getArrayClass())));
541 this.varargs = varargs;
542 }
543
544 @Override
545 public ValueNode findLength(FindLengthMode mode, ConstantReflectionProvider constantReflection) {
546 return ConstantNode.forInt(varargs.length);
547 }
548 }
549
550 static class CacheKey {
551
552 private final ResolvedJavaMethod method;
553 private final Object[] values;
554 private final GuardsStage guardsStage;
555 private final LoweringTool.LoweringStage loweringStage;
556 private int hash;
557
558 protected CacheKey(SnippetInfo info, GuardsStage guardsStage, LoweringTool.LoweringStage loweringStage) {
559 this.method = info.method;
560 this.guardsStage = guardsStage;
561 this.loweringStage = loweringStage;
562 this.values = new Object[info.getParameterCount()];
563 this.hash = info.method.hashCode() + 31 * guardsStage.ordinal();
564 }
565
566 protected void setParam(int paramIdx, Object value) {
567 values[paramIdx] = value;
568 hash = (hash * 31) ^ (value == null ? 0 : value.hashCode());
569 }
570
571 @Override
572 public boolean equals(Object obj) {
573 if (!(obj instanceof CacheKey)) {
574 return false;
575 }
576 CacheKey other = (CacheKey) obj;
577 if (!method.equals(other.method)) {
578 return false;
579 }
580 if (guardsStage != other.guardsStage || loweringStage != other.loweringStage) {
581 return false;
582 }
583 for (int i = 0; i < values.length; i++) {
584 if (values[i] != null && !values[i].equals(other.values[i])) {
585 return false;
586 }
587 }
588 return true;
589 }
590
591 @Override
592 public int hashCode() {
593 return hash;
594 }
595 }
596
597 private static final TimerKey SnippetTemplateCreationTime = DebugContext.timer("SnippetTemplateCreationTime");
598 private static final CounterKey SnippetTemplates = DebugContext.counter("SnippetTemplateCount");
599
600 static class Options {
601 @Option(help = "Use a LRU cache for snippet templates.")//
602 public static final OptionKey<Boolean> UseSnippetTemplateCache = new OptionKey<>(true);
603
604 @Option(help = "")//
605 static final OptionKey<Integer> MaxTemplatesPerSnippet = new OptionKey<>(50);
606 }
607
608 /**
609 * Base class for snippet classes. It provides a cache for {@link SnippetTemplate}s.
610 */
611 public abstract static class AbstractTemplates implements org.graalvm.compiler.api.replacements.SnippetTemplateCache {
612
613 protected final OptionValues options;
614 protected final Providers providers;
615 protected final SnippetReflectionProvider snippetReflection;
616 protected final Iterable<DebugHandlersFactory> factories;
617 protected final TargetDescription target;
618 private final Map<CacheKey, SnippetTemplate> templates;
619
620 protected AbstractTemplates(OptionValues options, Iterable<DebugHandlersFactory> factories, Providers providers, SnippetReflectionProvider snippetReflection, TargetDescription target) {
621 this.options = options;
622 this.providers = providers;
623 this.snippetReflection = snippetReflection;
624 this.target = target;
625 this.factories = factories;
626 if (Options.UseSnippetTemplateCache.getValue(options)) {
627 int size = Options.MaxTemplatesPerSnippet.getValue(options);
628 this.templates = Collections.synchronizedMap(new LRUCache<>(size, size));
629 } else {
630 this.templates = null;
631 }
632 }
633
634 public static Method findMethod(Class<? extends Snippets> declaringClass, String methodName, Method except) {
635 for (Method m : declaringClass.getDeclaredMethods()) {
636 if (m.getName().equals(methodName) && !m.equals(except)) {
637 return m;
638 }
639 }
640 return null;
641 }
642
643 public static ResolvedJavaMethod findMethod(MetaAccessProvider metaAccess, Class<?> declaringClass, String methodName) {
644 ResolvedJavaType type = metaAccess.lookupJavaType(declaringClass);
645 ResolvedJavaMethod result = null;
646 for (ResolvedJavaMethod m : type.getDeclaredMethods()) {
647 if (m.getName().equals(methodName)) {
648 if (!Assertions.assertionsEnabled()) {
649 return m;
650 } else {
651 assert result == null : "multiple definitions found";
652 result = m;
653 }
654 }
655 }
656 if (result == null) {
657 throw new GraalError("Could not find method in " + declaringClass + " named " + methodName);
658 }
659 return result;
660 }
661
662 protected SnippetInfo snippet(Class<? extends Snippets> declaringClass, String methodName, LocationIdentity... initialPrivateLocations) {
663 return snippet(declaringClass, methodName, null, null, initialPrivateLocations);
664 }
665
666 /**
667 * Finds the unique method in {@code declaringClass} named {@code methodName} annotated by
668 * {@link Snippet} and returns a {@link SnippetInfo} value describing it. There must be
669 * exactly one snippet method in {@code declaringClass} with a given name.
670 */
671 protected SnippetInfo snippet(Class<? extends Snippets> declaringClass, String methodName, ResolvedJavaMethod original, Object receiver, LocationIdentity... initialPrivateLocations) {
672 assert methodName != null;
673 ResolvedJavaMethod javaMethod = findMethod(providers.getMetaAccess(), declaringClass, methodName);
674 assert javaMethod != null : "did not find @" + Snippet.class.getSimpleName() + " method in " + declaringClass + " named " + methodName;
675 assert javaMethod.getAnnotation(Snippet.class) != null : javaMethod + " must be annotated with @" + Snippet.class.getSimpleName();
676 providers.getReplacements().registerSnippet(javaMethod, original, receiver, GraalOptions.TrackNodeSourcePosition.getValue(options));
677 LocationIdentity[] privateLocations = GraalOptions.SnippetCounters.getValue(options) ? SnippetCounterNode.addSnippetCounters(initialPrivateLocations) : initialPrivateLocations;
678 if (GraalOptions.EagerSnippets.getValue(options)) {
679 return new EagerSnippetInfo(javaMethod, original, privateLocations, receiver);
680 } else {
681 return new LazySnippetInfo(javaMethod, original, privateLocations, receiver);
682 }
683 }
684
685 static final AtomicInteger nextSnippetTemplateId = new AtomicInteger();
686
687 private DebugContext openDebugContext(DebugContext outer, Arguments args) {
688 if (DebugStubsAndSnippets.getValue(options)) {
689 Description description = new Description(args.cacheKey.method, "SnippetTemplate_" + nextSnippetTemplateId.incrementAndGet());
690 return DebugContext.create(options, description, outer.getGlobalMetrics(), DEFAULT_LOG_STREAM, factories);
691 }
692 return DebugContext.DISABLED;
693 }
694
695 /**
696 * Gets a template for a given key, creating it first if necessary.
697 */
698 @SuppressWarnings("try")
699 protected SnippetTemplate template(ValueNode replacee, final Arguments args) {
700 StructuredGraph graph = replacee.graph();
701 DebugContext outer = graph.getDebug();
702 SnippetTemplate template = Options.UseSnippetTemplateCache.getValue(options) && args.cacheable ? templates.get(args.cacheKey) : null;
703 if (template == null || (graph.trackNodeSourcePosition() && !template.snippet.trackNodeSourcePosition())) {
704 try (DebugContext debug = openDebugContext(outer, args)) {
705 try (DebugCloseable a = SnippetTemplateCreationTime.start(debug); DebugContext.Scope s = debug.scope("SnippetSpecialization", args.info.method)) {
706 SnippetTemplates.increment(debug);
707 OptionValues snippetOptions = new OptionValues(options, GraalOptions.TraceInlining, GraalOptions.TraceInliningForStubsAndSnippets.getValue(options));
708 template = new SnippetTemplate(snippetOptions, debug, providers, snippetReflection, args, graph.trackNodeSourcePosition(), replacee);
709 if (Options.UseSnippetTemplateCache.getValue(snippetOptions) && args.cacheable) {
710 templates.put(args.cacheKey, template);
711 }
712 } catch (Throwable e) {
713 throw debug.handle(e);
714 }
715 }
716 }
717 return template;
718 }
719 }
720
721 private static final class LRUCache<K, V> extends LinkedHashMap<K, V> {
722 private static final long serialVersionUID = 1L;
723 private final int maxCacheSize;
724
725 LRUCache(int initialCapacity, int maxCacheSize) {
726 super(initialCapacity, 0.75F, true);
727 this.maxCacheSize = maxCacheSize;
728 }
729
730 @Override
731 protected boolean removeEldestEntry(java.util.Map.Entry<K, V> eldest) {
732 return size() > maxCacheSize;
733 }
734 }
735
736 // These values must be compared with equals() not '==' to support replay compilation.
737 private static final Object UNUSED_PARAMETER = "UNUSED_PARAMETER";
738 private static final Object CONSTANT_PARAMETER = "CONSTANT_PARAMETER";
739
740 /**
741 * Determines if any parameter of a given method is annotated with {@link ConstantParameter}.
742 */
743 public static boolean hasConstantParameter(ResolvedJavaMethod method) {
744 for (ConstantParameter p : method.getParameterAnnotations(ConstantParameter.class)) {
745 if (p != null) {
746 return true;
747 }
748 }
749 return false;
750 }
751
752 private final SnippetReflectionProvider snippetReflection;
753
754 /**
755 * Creates a snippet template.
756 */
757 @SuppressWarnings("try")
758 protected SnippetTemplate(OptionValues options, DebugContext debug, final Providers providers, SnippetReflectionProvider snippetReflection, Arguments args, boolean trackNodeSourcePosition,
759 Node replacee) {
760 this.snippetReflection = snippetReflection;
761 this.info = args.info;
762
763 Object[] constantArgs = getConstantArgs(args);
764 boolean shouldTrackNodeSourcePosition1 = trackNodeSourcePosition || (providers.getCodeCache() != null && providers.getCodeCache().shouldDebugNonSafepoints());
765 StructuredGraph snippetGraph = providers.getReplacements().getSnippet(args.info.method, args.info.original, constantArgs, shouldTrackNodeSourcePosition1, replacee.getNodeSourcePosition());
766
767 ResolvedJavaMethod method = snippetGraph.method();
768 Signature signature = method.getSignature();
769
770 PhaseContext phaseContext = new PhaseContext(providers);
771
772 // Copy snippet graph, replacing constant parameters with given arguments
773 final StructuredGraph snippetCopy = new StructuredGraph.Builder(options, debug).name(snippetGraph.name).method(snippetGraph.method()).trackNodeSourcePosition(
774 snippetGraph.trackNodeSourcePosition()).setIsSubstitution(true).build();
775 assert !GraalOptions.TrackNodeSourcePosition.getValue(options) || snippetCopy.trackNodeSourcePosition();
776 try (DebugContext.Scope scope = debug.scope("SpecializeSnippet", snippetCopy)) {
777 if (!snippetGraph.isUnsafeAccessTrackingEnabled()) {
778 snippetCopy.disableUnsafeAccessTracking();
779 }
780
781 EconomicMap<Node, Node> nodeReplacements = EconomicMap.create(Equivalence.IDENTITY);
782 nodeReplacements.put(snippetGraph.start(), snippetCopy.start());
783
784 MetaAccessProvider metaAccess = providers.getMetaAccess();
785 assert checkTemplate(metaAccess, args, method, signature);
786
787 int parameterCount = args.info.getParameterCount();
788 VarargsPlaceholderNode[] placeholders = new VarargsPlaceholderNode[parameterCount];
789
790 for (int i = 0; i < parameterCount; i++) {
791 ParameterNode parameter = snippetGraph.getParameter(i);
792 if (parameter != null) {
793 if (args.info.isConstantParameter(i)) {
794 Object arg = args.values[i];
795 JavaKind kind = signature.getParameterKind(i);
796 ConstantNode constantNode;
797 if (arg instanceof Constant) {
798 Stamp stamp = args.constStamps[i];
799 if (stamp == null) {
800 assert arg instanceof JavaConstant : "could not determine type of constant " + arg;
801 constantNode = ConstantNode.forConstant((JavaConstant) arg, metaAccess, snippetCopy);
802 } else {
803 constantNode = ConstantNode.forConstant(stamp, (Constant) arg, metaAccess, snippetCopy);
804 }
805 } else {
806 constantNode = ConstantNode.forConstant(snippetReflection.forBoxed(kind, arg), metaAccess, snippetCopy);
807 }
808 nodeReplacements.put(parameter, constantNode);
809 } else if (args.info.isVarargsParameter(i)) {
810 Varargs varargs = (Varargs) args.values[i];
811 VarargsPlaceholderNode placeholder = snippetCopy.unique(new VarargsPlaceholderNode(varargs, providers.getMetaAccess()));
812 nodeReplacements.put(parameter, placeholder);
813 placeholders[i] = placeholder;
814 } else if (args.info.isNonNullParameter(i)) {
815 parameter.setStamp(parameter.stamp(NodeView.DEFAULT).join(StampFactory.objectNonNull()));
816 }
817 }
818 }
819 try (InliningLog.UpdateScope updateScope = snippetCopy.getInliningLog().openDefaultUpdateScope()) {
820 UnmodifiableEconomicMap<Node, Node> duplicates = snippetCopy.addDuplicates(snippetGraph.getNodes(), snippetGraph, snippetGraph.getNodeCount(), nodeReplacements);
821 if (updateScope != null) {
822 snippetCopy.getInliningLog().replaceLog(duplicates, snippetGraph.getInliningLog());
823 }
824 }
825
826 debug.dump(DebugContext.INFO_LEVEL, snippetCopy, "Before specialization");
827
828 // Gather the template parameters
829 parameters = new Object[parameterCount];
830 for (int i = 0; i < parameterCount; i++) {
831 if (args.info.isConstantParameter(i)) {
832 parameters[i] = CONSTANT_PARAMETER;
833 } else if (args.info.isVarargsParameter(i)) {
834 assert snippetCopy.getParameter(i) == null;
835 Varargs varargs = (Varargs) args.values[i];
836 int length = varargs.length;
837 ParameterNode[] params = new ParameterNode[length];
838 Stamp stamp = varargs.stamp;
839 for (int j = 0; j < length; j++) {
840 // Use a decimal friendly numbering make it more obvious how values map
841 assert parameterCount < 10000;
842 int idx = (i + 1) * 10000 + j;
843 assert idx >= parameterCount : "collision in parameter numbering";
844 ParameterNode local = snippetCopy.addOrUnique(new ParameterNode(idx, StampPair.createSingle(stamp)));
845 params[j] = local;
846 }
847 parameters[i] = params;
848
849 VarargsPlaceholderNode placeholder = placeholders[i];
850 if (placeholder != null) {
851 for (Node usage : placeholder.usages().snapshot()) {
852 if (usage instanceof LoadIndexedNode) {
853 LoadIndexedNode loadIndexed = (LoadIndexedNode) usage;
854 debug.dump(DebugContext.INFO_LEVEL, snippetCopy, "Before replacing %s", loadIndexed);
855 LoadSnippetVarargParameterNode loadSnippetParameter = snippetCopy.add(
856 new LoadSnippetVarargParameterNode(params, loadIndexed.index(), loadIndexed.stamp(NodeView.DEFAULT)));
857 snippetCopy.replaceFixedWithFixed(loadIndexed, loadSnippetParameter);
858 debug.dump(DebugContext.INFO_LEVEL, snippetCopy, "After replacing %s", loadIndexed);
859 } else if (usage instanceof StoreIndexedNode) {
860 /*
861 * The template lowering doesn't really treat this as an array so
862 * you can't store back into the varargs. Allocate your own array if
863 * you really need this and EA should eliminate it.
864 */
865 throw new GraalError("Can't store into VarargsParameter array");
866 }
867 }
868 }
869 } else {
870 ParameterNode local = snippetCopy.getParameter(i);
871 if (local == null) {
872 // Parameter value was eliminated
873 parameters[i] = UNUSED_PARAMETER;
874 } else {
875 parameters[i] = local;
876 }
877 }
878 }
879
880 explodeLoops(snippetCopy, phaseContext);
881
882 GuardsStage guardsStage = args.cacheKey.guardsStage;
883 // Perform lowering on the snippet
884 if (!guardsStage.allowsFloatingGuards()) {
885 new GuardLoweringPhase().apply(snippetCopy, null);
886 }
887 snippetCopy.setGuardsStage(guardsStage);
888 try (DebugContext.Scope s = debug.scope("LoweringSnippetTemplate", snippetCopy)) {
889 new LoweringPhase(new CanonicalizerPhase(), args.cacheKey.loweringStage).apply(snippetCopy, phaseContext);
890 } catch (Throwable e) {
891 throw debug.handle(e);
892 }
893
894 ArrayList<StateSplit> curSideEffectNodes = new ArrayList<>();
895 ArrayList<DeoptimizingNode> curDeoptNodes = new ArrayList<>();
896 ArrayList<ValueNode> curPlaceholderStampedNodes = new ArrayList<>();
897 for (Node node : snippetCopy.getNodes()) {
898 if (node instanceof ValueNode) {
899 ValueNode valueNode = (ValueNode) node;
900 if (valueNode.stamp(NodeView.DEFAULT) == PlaceholderStamp.singleton()) {
901 curPlaceholderStampedNodes.add(valueNode);
902 }
903 }
904
905 if (node instanceof StateSplit) {
906 StateSplit stateSplit = (StateSplit) node;
907 FrameState frameState = stateSplit.stateAfter();
908 if (stateSplit.hasSideEffect()) {
909 curSideEffectNodes.add((StateSplit) node);
910 }
911 if (frameState != null) {
912 stateSplit.setStateAfter(null);
913 }
914 }
915 if (node instanceof DeoptimizingNode) {
916 DeoptimizingNode deoptNode = (DeoptimizingNode) node;
917 if (deoptNode.canDeoptimize()) {
918 curDeoptNodes.add(deoptNode);
919 }
920 }
921 }
922
923 new DeadCodeEliminationPhase(Required).apply(snippetCopy);
924
925 assert checkAllVarargPlaceholdersAreDeleted(parameterCount, placeholders);
926
927 new FloatingReadPhase(true, true).apply(snippetCopy);
928 new RemoveValueProxyPhase().apply(snippetCopy);
929
930 MemoryAnchorNode anchor = snippetCopy.add(new MemoryAnchorNode());
931 snippetCopy.start().replaceAtUsages(InputType.Memory, anchor);
932
933 this.snippet = snippetCopy;
934
935 StartNode entryPointNode = snippet.start();
936 if (anchor.hasNoUsages()) {
937 anchor.safeDelete();
938 this.memoryAnchor = null;
939 } else {
940 // Find out if all the return memory maps point to the anchor (i.e., there's no kill
941 // anywhere)
942 boolean needsMemoryMaps = false;
943 for (ReturnNode retNode : snippet.getNodes(ReturnNode.TYPE)) {
944 MemoryMapNode memoryMap = retNode.getMemoryMap();
945 if (memoryMap.getLocations().size() > 1 || memoryMap.getLastLocationAccess(LocationIdentity.any()) != anchor) {
946 needsMemoryMaps = true;
947 break;
948 }
949 }
950 boolean needsAnchor;
951 if (needsMemoryMaps) {
952 needsAnchor = true;
953 } else {
954 // Check that all those memory maps where the only usages of the anchor
955 needsAnchor = anchor.usages().filter(isNotA(MemoryMapNode.class)).isNotEmpty();
956 // Remove the useless memory map
957 MemoryMapNode memoryMap = null;
958 for (ReturnNode retNode : snippet.getNodes(ReturnNode.TYPE)) {
959 if (memoryMap == null) {
960 memoryMap = retNode.getMemoryMap();
961 } else {
962 assert memoryMap == retNode.getMemoryMap();
963 }
964 retNode.setMemoryMap(null);
965 }
966 memoryMap.safeDelete();
967 }
968 if (needsAnchor) {
969 snippetCopy.addAfterFixed(snippetCopy.start(), anchor);
970 this.memoryAnchor = anchor;
971 } else {
972 anchor.safeDelete();
973 this.memoryAnchor = null;
974 }
975 }
976 debug.dump(DebugContext.INFO_LEVEL, snippet, "SnippetTemplate after fixing memory anchoring");
977
978 List<ReturnNode> returnNodes = snippet.getNodes(ReturnNode.TYPE).snapshot();
979 if (returnNodes.isEmpty()) {
980 this.returnNode = null;
981 } else if (returnNodes.size() == 1) {
982 this.returnNode = returnNodes.get(0);
983 } else {
984 AbstractMergeNode merge = snippet.add(new MergeNode());
985 List<MemoryMapNode> memMaps = new ArrayList<>();
986 for (ReturnNode retNode : returnNodes) {
987 MemoryMapNode memoryMapNode = retNode.getMemoryMap();
988 if (memoryMapNode != null) {
989 memMaps.add(memoryMapNode);
990 }
991 }
992
993 ValueNode returnValue = InliningUtil.mergeReturns(merge, returnNodes);
994 this.returnNode = snippet.add(new ReturnNode(returnValue));
995 if (!memMaps.isEmpty()) {
996 MemoryMapImpl mmap = FloatingReadPhase.mergeMemoryMaps(merge, memMaps);
997 MemoryMapNode memoryMap = snippet.unique(new MemoryMapNode(mmap.getMap()));
998 this.returnNode.setMemoryMap(memoryMap);
999 for (MemoryMapNode mm : memMaps) {
1000 if (mm != memoryMap && mm.isAlive()) {
1001 assert mm.hasNoUsages();
1002 GraphUtil.killWithUnusedFloatingInputs(mm);
1003 }
1004 }
1005 }
1006 merge.setNext(this.returnNode);
1007 }
1008
1009 assert verifyIntrinsicsProcessed(snippetCopy);
1010
1011 this.sideEffectNodes = curSideEffectNodes;
1012 this.deoptNodes = curDeoptNodes;
1013 this.placeholderStampedNodes = curPlaceholderStampedNodes;
1014
1015 nodes = new ArrayList<>(snippet.getNodeCount());
1016 for (Node node : snippet.getNodes()) {
1017 if (node != entryPointNode && node != entryPointNode.stateAfter()) {
1018 nodes.add(node);
1019 }
1020 }
1021
1022 if (debug.areMetricsEnabled()) {
1023 DebugContext.counter("SnippetTemplateNodeCount[%#s]", args).add(debug, nodes.size());
1024 }
1025 debug.dump(DebugContext.INFO_LEVEL, snippet, "SnippetTemplate final state");
1026 this.snippet.freeze();
1027
1028 } catch (Throwable ex) {
1029 throw debug.handle(ex);
1030 }
1031 }
1032
1033 private static boolean verifyIntrinsicsProcessed(StructuredGraph snippetCopy) {
1034 for (MethodCallTargetNode target : snippetCopy.getNodes(MethodCallTargetNode.TYPE)) {
1035 ResolvedJavaMethod targetMethod = target.targetMethod();
1036 if (targetMethod != null) {
1037 assert targetMethod.getAnnotation(Fold.class) == null && targetMethod.getAnnotation(NodeIntrinsic.class) == null : "plugin should have been processed";
1038 }
1039 }
1040 return true;
1041 }
1042
1043 public static void explodeLoops(final StructuredGraph snippetCopy, PhaseContext phaseContext) {
1044 // Do any required loop explosion
1045 boolean exploded = false;
1046 do {
1047 exploded = false;
1048 ExplodeLoopNode explodeLoop = snippetCopy.getNodes().filter(ExplodeLoopNode.class).first();
1049 if (explodeLoop != null) { // Earlier canonicalization may have removed the loop
1050 // altogether
1051 LoopBeginNode loopBegin = explodeLoop.findLoopBegin();
1052 if (loopBegin != null) {
1053 LoopEx loop = new LoopsData(snippetCopy).loop(loopBegin);
1054 Mark mark = snippetCopy.getMark();
1055 LoopTransformations.fullUnroll(loop, phaseContext, new CanonicalizerPhase());
1056 new CanonicalizerPhase().applyIncremental(snippetCopy, phaseContext, mark, false);
1057 loop.deleteUnusedNodes();
1058 }
1059 GraphUtil.removeFixedWithUnusedInputs(explodeLoop);
1060 exploded = true;
1061 }
1062 } while (exploded);
1063 }
1064
1065 protected Object[] getConstantArgs(Arguments args) {
1066 Object[] constantArgs = args.values.clone();
1067 for (int i = 0; i < args.info.getParameterCount(); i++) {
1068 if (!args.info.isConstantParameter(i)) {
1069 constantArgs[i] = null;
1070 } else {
1071 assert constantArgs[i] != null : "Can't pass raw null through as argument";
1072 }
1073 }
1074 return constantArgs;
1075 }
1076
1077 private static boolean checkAllVarargPlaceholdersAreDeleted(int parameterCount, VarargsPlaceholderNode[] placeholders) {
1078 for (int i = 0; i < parameterCount; i++) {
1079 if (placeholders[i] != null) {
1080 assert placeholders[i].isDeleted() : placeholders[i];
1081 }
1082 }
1083 return true;
1084 }
1085
1086 private static boolean checkConstantArgument(MetaAccessProvider metaAccess, final ResolvedJavaMethod method, Signature signature, int paramIndex, String name, Object arg, JavaKind kind) {
1087 ResolvedJavaType type = signature.getParameterType(paramIndex, method.getDeclaringClass()).resolve(method.getDeclaringClass());
1088 if (metaAccess.lookupJavaType(WordBase.class).isAssignableFrom(type)) {
1089 assert arg instanceof JavaConstant : method + ": word constant parameters must be passed boxed in a Constant value: " + arg;
1090 return true;
1091 }
1092 if (kind != JavaKind.Object) {
1093 assert arg != null && kind.toBoxedJavaClass() == arg.getClass() : method + ": wrong value kind for " + name + ": expected " + kind + ", got " +
1094 (arg == null ? "null" : arg.getClass().getSimpleName());
1095 }
1096 return true;
1097 }
1098
1099 private static boolean checkVarargs(MetaAccessProvider metaAccess, final ResolvedJavaMethod method, Signature signature, int i, String name, Varargs varargs) {
1100 ResolvedJavaType type = (ResolvedJavaType) signature.getParameterType(i, method.getDeclaringClass());
1101 assert type.isArray() : "varargs parameter must be an array type";
1102 assert type.getComponentType().isAssignableFrom(metaAccess.lookupJavaType(varargs.componentType)) : "componentType for " + name + " not matching " + type.toJavaName() + " instance: " +
1103 varargs.componentType;
1104 return true;
1105 }
1106
1107 /**
1108 * The graph built from the snippet method.
1109 */
1110 private final StructuredGraph snippet;
1111
1112 private final SnippetInfo info;
1113
1114 /**
1115 * The named parameters of this template that must be bound to values during instantiation. For
1116 * a parameter that is still live after specialization, the value in this map is either a
1117 * {@link ParameterNode} instance or a {@link ParameterNode} array. For an eliminated parameter,
1118 * the value is identical to the key.
1119 */
1120 private final Object[] parameters;
1121
1122 /**
1123 * The return node (if any) of the snippet.
1124 */
1125 private final ReturnNode returnNode;
1126
1127 /**
1128 * The memory anchor (if any) of the snippet.
1129 */
1130 private final MemoryAnchorNode memoryAnchor;
1131
1132 /**
1133 * Nodes that inherit the {@link StateSplit#stateAfter()} from the replacee during
1134 * instantiation.
1135 */
1136 private final ArrayList<StateSplit> sideEffectNodes;
1137
1138 /**
1139 * Nodes that inherit a deoptimization {@link FrameState} from the replacee during
1140 * instantiation.
1141 */
1142 private final ArrayList<DeoptimizingNode> deoptNodes;
1143
1144 /**
1145 * Nodes that have a stamp originating from a {@link Placeholder}.
1146 */
1147 private final ArrayList<ValueNode> placeholderStampedNodes;
1148
1149 /**
1150 * The nodes to be inlined when this specialization is instantiated.
1151 */
1152 private final ArrayList<Node> nodes;
1153
1154 /**
1155 * Gets the instantiation-time bindings to this template's parameters.
1156 *
1157 * @return the map that will be used to bind arguments to parameters when inlining this template
1158 */
1159 private EconomicMap<Node, Node> bind(StructuredGraph replaceeGraph, MetaAccessProvider metaAccess, Arguments args) {
1160 EconomicMap<Node, Node> replacements = EconomicMap.create(Equivalence.IDENTITY);
1161 assert args.info.getParameterCount() == parameters.length : "number of args (" + args.info.getParameterCount() + ") != number of parameters (" + parameters.length + ")";
1162 for (int i = 0; i < parameters.length; i++) {
1163 Object parameter = parameters[i];
1164 assert parameter != null : this + " has no parameter named " + args.info.getParameterName(i);
1165 Object argument = args.values[i];
1166 if (parameter instanceof ParameterNode) {
1167 if (argument instanceof ValueNode) {
1168 replacements.put((ParameterNode) parameter, (ValueNode) argument);
1169 } else {
1170 JavaKind kind = ((ParameterNode) parameter).getStackKind();
1171 assert argument != null || kind == JavaKind.Object : this + " cannot accept null for non-object parameter named " + args.info.getParameterName(i);
1172 JavaConstant constant = forBoxed(argument, kind);
1173 replacements.put((ParameterNode) parameter, ConstantNode.forConstant(constant, metaAccess, replaceeGraph));
1174 }
1175 } else if (parameter instanceof ParameterNode[]) {
1176 ParameterNode[] params = (ParameterNode[]) parameter;
1177 Varargs varargs = (Varargs) argument;
1178 int length = params.length;
1179 List<?> list = null;
1180 Object array = null;
1181 if (varargs.value instanceof List) {
1182 list = (List<?>) varargs.value;
1183 assert list.size() == length : length + " != " + list.size();
1184 } else {
1185 array = varargs.value;
1186 assert array != null && array.getClass().isArray();
1187 assert Array.getLength(array) == length : length + " != " + Array.getLength(array);
1188 }
1189
1190 for (int j = 0; j < length; j++) {
1191 ParameterNode param = params[j];
1192 assert param != null;
1193 Object value = list != null ? list.get(j) : Array.get(array, j);
1194 if (value instanceof ValueNode) {
1195 replacements.put(param, (ValueNode) value);
1196 } else {
1197 JavaConstant constant = forBoxed(value, param.getStackKind());
1198 ConstantNode element = ConstantNode.forConstant(constant, metaAccess, replaceeGraph);
1199 replacements.put(param, element);
1200 }
1201 }
1202 } else {
1203 assert parameter.equals(CONSTANT_PARAMETER) || parameter.equals(UNUSED_PARAMETER) : "unexpected entry for parameter: " + args.info.getParameterName(i) + " -> " + parameter;
1204 }
1205 }
1206 return replacements;
1207 }
1208
1209 /**
1210 * Converts a Java boxed value to a {@link JavaConstant} of the right kind. This adjusts for the
1211 * limitation that a {@link Local}'s kind is a {@linkplain JavaKind#getStackKind() stack kind}
1212 * and so cannot be used for re-boxing primitives smaller than an int.
1213 *
1214 * @param argument a Java boxed value
1215 * @param localKind the kind of the {@link Local} to which {@code argument} will be bound
1216 */
1217 protected JavaConstant forBoxed(Object argument, JavaKind localKind) {
1218 assert localKind == localKind.getStackKind();
1219 if (localKind == JavaKind.Int) {
1220 return JavaConstant.forBoxedPrimitive(argument);
1221 }
1222 return snippetReflection.forBoxed(localKind, argument);
1223 }
1224
1225 /**
1226 * Logic for replacing a snippet-lowered node at its usages with the return value of the
1227 * snippet. An alternative to the {@linkplain SnippetTemplate#DEFAULT_REPLACER default}
1228 * replacement logic can be used to handle mismatches between the stamp of the node being
1229 * lowered and the stamp of the snippet's return value.
1230 */
1231 public interface UsageReplacer {
1232 /**
1233 * Replaces all usages of {@code oldNode} with direct or indirect usages of {@code newNode}.
1234 */
1235 void replace(ValueNode oldNode, ValueNode newNode);
1236 }
1237
1238 /**
1239 * Represents the default {@link UsageReplacer usage replacer} logic which simply delegates to
1240 * {@link Node#replaceAtUsages(Node)}.
1241 */
1242 public static final UsageReplacer DEFAULT_REPLACER = new UsageReplacer() {
1243
1244 @Override
1245 public void replace(ValueNode oldNode, ValueNode newNode) {
1246 if (newNode == null) {
1247 assert oldNode.hasNoUsages();
1248 } else {
1249 oldNode.replaceAtUsages(newNode);
1250 }
1251 }
1252 };
1253
1254 private boolean assertSnippetKills(ValueNode replacee) {
1255 if (!replacee.graph().isAfterFloatingReadPhase()) {
1256 // no floating reads yet, ignore locations created while lowering
1257 return true;
1258 }
1259 if (returnNode == null) {
1260 // The snippet terminates control flow
1261 return true;
1262 }
1263 MemoryMapNode memoryMap = returnNode.getMemoryMap();
1264 if (memoryMap == null || memoryMap.isEmpty()) {
1265 // there are no kills in the snippet graph
1266 return true;
1267 }
1268
1269 EconomicSet<LocationIdentity> kills = EconomicSet.create(Equivalence.DEFAULT);
1270 kills.addAll(memoryMap.getLocations());
1271
1272 if (replacee instanceof MemoryCheckpoint.Single) {
1273 // check if some node in snippet graph also kills the same location
1274 LocationIdentity locationIdentity = ((MemoryCheckpoint.Single) replacee).getLocationIdentity();
1275 if (locationIdentity.isAny()) {
1276 assert !(memoryMap.getLastLocationAccess(any()) instanceof MemoryAnchorNode) : replacee + " kills ANY_LOCATION, but snippet does not";
1277 // if the replacee kills ANY_LOCATION, the snippet can kill arbitrary locations
1278 return true;
1279 }
1280 assert kills.contains(locationIdentity) : replacee + " kills " + locationIdentity + ", but snippet doesn't contain a kill to this location";
1281 kills.remove(locationIdentity);
1282 }
1283 assert !(replacee instanceof MemoryCheckpoint.Multi) : replacee + " multi not supported (yet)";
1284
1285 // remove ANY_LOCATION if it's just a kill by the start node
1286 if (memoryMap.getLastLocationAccess(any()) instanceof MemoryAnchorNode) {
1287 kills.remove(any());
1288 }
1289
1290 // node can only lower to a ANY_LOCATION kill if the replacee also kills ANY_LOCATION
1291 assert !kills.contains(any()) : "snippet graph contains a kill to ANY_LOCATION, but replacee (" + replacee + ") doesn't kill ANY_LOCATION. kills: " + kills;
1292
1293 /*
1294 * Kills to private locations are safe, since there can be no floating read to these
1295 * locations except reads that are introduced by the snippet itself or related snippets in
1296 * the same lowering round. These reads are anchored to a MemoryAnchor at the beginning of
1297 * their snippet, so they can not float above a kill in another instance of the same
1298 * snippet.
1299 */
1300 for (LocationIdentity p : this.info.privateLocations) {
1301 kills.remove(p);
1302 }
1303
1304 assert kills.isEmpty() : "snippet graph kills non-private locations " + kills + " that replacee (" + replacee + ") doesn't kill";
1305 return true;
1306 }
1307
1308 private static class MemoryInputMap implements MemoryMap {
1309
1310 private final LocationIdentity locationIdentity;
1311 private final MemoryNode lastLocationAccess;
1312
1313 MemoryInputMap(ValueNode replacee) {
1314 if (replacee instanceof MemoryAccess) {
1315 MemoryAccess access = (MemoryAccess) replacee;
1316 locationIdentity = access.getLocationIdentity();
1317 lastLocationAccess = access.getLastLocationAccess();
1318 } else {
1319 locationIdentity = null;
1320 lastLocationAccess = null;
1321 }
1322 }
1323
1324 @Override
1325 public MemoryNode getLastLocationAccess(LocationIdentity location) {
1326 if (locationIdentity != null && locationIdentity.equals(location)) {
1327 return lastLocationAccess;
1328 } else {
1329 return null;
1330 }
1331 }
1332
1333 @Override
1334 public Collection<LocationIdentity> getLocations() {
1335 if (locationIdentity == null) {
1336 return Collections.emptySet();
1337 } else {
1338 return Collections.singleton(locationIdentity);
1339 }
1340 }
1341 }
1342
1343 private class MemoryOutputMap extends MemoryInputMap {
1344
1345 private final UnmodifiableEconomicMap<Node, Node> duplicates;
1346
1347 MemoryOutputMap(ValueNode replacee, UnmodifiableEconomicMap<Node, Node> duplicates) {
1348 super(replacee);
1349 this.duplicates = duplicates;
1350 }
1351
1352 @Override
1353 public MemoryNode getLastLocationAccess(LocationIdentity locationIdentity) {
1354 MemoryMapNode memoryMap = returnNode.getMemoryMap();
1355 assert memoryMap != null : "no memory map stored for this snippet graph (snippet doesn't have a ReturnNode?)";
1356 MemoryNode lastLocationAccess = memoryMap.getLastLocationAccess(locationIdentity);
1357 assert lastLocationAccess != null : locationIdentity;
1358 if (lastLocationAccess == memoryAnchor) {
1359 return super.getLastLocationAccess(locationIdentity);
1360 } else {
1361 return (MemoryNode) duplicates.get(ValueNodeUtil.asNode(lastLocationAccess));
1362 }
1363 }
1364
1365 @Override
1366 public Collection<LocationIdentity> getLocations() {
1367 return returnNode.getMemoryMap().getLocations();
1368 }
1369 }
1370
1371 private void rewireMemoryGraph(ValueNode replacee, UnmodifiableEconomicMap<Node, Node> duplicates) {
1372 if (replacee.graph().isAfterFloatingReadPhase()) {
1373 // rewire outgoing memory edges
1374 replaceMemoryUsages(replacee, new MemoryOutputMap(replacee, duplicates));
1375
1376 if (returnNode != null) {
1377 ReturnNode ret = (ReturnNode) duplicates.get(returnNode);
1378 if (ret != null) {
1379 MemoryMapNode memoryMap = ret.getMemoryMap();
1380 if (memoryMap != null) {
1381 ret.setMemoryMap(null);
1382 memoryMap.safeDelete();
1383 }
1384 }
1385 }
1386 if (memoryAnchor != null) {
1387 // rewire incoming memory edges
1388 MemoryAnchorNode memoryDuplicate = (MemoryAnchorNode) duplicates.get(memoryAnchor);
1389 replaceMemoryUsages(memoryDuplicate, new MemoryInputMap(replacee));
1390
1391 if (memoryDuplicate.hasNoUsages()) {
1392 if (memoryDuplicate.next() != null) {
1393 memoryDuplicate.graph().removeFixed(memoryDuplicate);
1394 } else {
1395 // this was a dummy memory node used when instantiating pure data-flow
1396 // snippets: it was not attached to the control flow.
1397 memoryDuplicate.safeDelete();
1398 }
1399 }
1400 }
1401 }
1402 }
1403
1404 private static LocationIdentity getLocationIdentity(Node node) {
1405 if (node instanceof MemoryAccess) {
1406 return ((MemoryAccess) node).getLocationIdentity();
1407 } else if (node instanceof MemoryProxy) {
1408 return ((MemoryProxy) node).getLocationIdentity();
1409 } else if (node instanceof MemoryPhiNode) {
1410 return ((MemoryPhiNode) node).getLocationIdentity();
1411 } else {
1412 return null;
1413 }
1414 }
1415
1416 private void replaceMemoryUsages(ValueNode node, MemoryMap map) {
1417 for (Node usage : node.usages().snapshot()) {
1418 if (usage instanceof MemoryMapNode) {
1419 continue;
1420 }
1421
1422 LocationIdentity location = getLocationIdentity(usage);
1423 if (location != null) {
1424 for (Position pos : usage.inputPositions()) {
1425 if (pos.getInputType() == InputType.Memory && pos.get(usage) == node) {
1426 MemoryNode replacement = map.getLastLocationAccess(location);
1427 if (replacement == null) {
1428 assert mayRemoveLocation || LocationIdentity.any().equals(location) ||
1429 CollectionsUtil.anyMatch(info.privateLocations, Predicate.isEqual(location)) : "Snippet " +
1430 info.method.format("%h.%n") + " contains access to the non-private location " +
1431 location + ", but replacee doesn't access this location." + map.getLocations();
1432 } else {
1433 pos.set(usage, replacement.asNode());
1434 }
1435 }
1436 }
1437 }
1438 }
1439 }
1440
1441 /**
1442 * Replaces a given fixed node with this specialized snippet.
1443 *
1444 * @param metaAccess
1445 * @param replacee the node that will be replaced
1446 * @param replacer object that replaces the usages of {@code replacee}
1447 * @param args the arguments to be bound to the flattened positional parameters of the snippet
1448 * @return the map of duplicated nodes (original -> duplicate)
1449 */
1450 @SuppressWarnings("try")
1451 public UnmodifiableEconomicMap<Node, Node> instantiate(MetaAccessProvider metaAccess, FixedNode replacee, UsageReplacer replacer, Arguments args) {
1452 return instantiate(metaAccess, replacee, replacer, args, true);
1453 }
1454
1455 /**
1456 * Replaces a given fixed node with this specialized snippet.
1457 *
1458 * @param metaAccess
1459 * @param replacee the node that will be replaced
1460 * @param replacer object that replaces the usages of {@code replacee}
1461 * @param args the arguments to be bound to the flattened positional parameters of the snippet
1462 * @param killReplacee is true, the replacee node is deleted
1463 * @return the map of duplicated nodes (original -> duplicate)
1464 */
1465 @SuppressWarnings("try")
1466 public UnmodifiableEconomicMap<Node, Node> instantiate(MetaAccessProvider metaAccess, FixedNode replacee, UsageReplacer replacer, Arguments args, boolean killReplacee) {
1467 DebugContext debug = replacee.getDebug();
1468 assert assertSnippetKills(replacee);
1469 try (DebugCloseable a = args.info.instantiationTimer.start(debug)) {
1470 args.info.instantiationCounter.increment(debug);
1471 // Inline the snippet nodes, replacing parameters with the given args in the process
1472 StartNode entryPointNode = snippet.start();
1473 FixedNode firstCFGNode = entryPointNode.next();
1474 StructuredGraph replaceeGraph = replacee.graph();
1475 EconomicMap<Node, Node> replacements = bind(replaceeGraph, metaAccess, args);
1476 replacements.put(entryPointNode, AbstractBeginNode.prevBegin(replacee));
1477 UnmodifiableEconomicMap<Node, Node> duplicates = inlineSnippet(replacee, debug, replaceeGraph, replacements);
1478
1479 // Re-wire the control flow graph around the replacee
1480 FixedNode firstCFGNodeDuplicate = (FixedNode) duplicates.get(firstCFGNode);
1481 replacee.replaceAtPredecessor(firstCFGNodeDuplicate);
1482
1483 rewireFrameStates(replacee, duplicates);
1484
1485 if (replacee instanceof DeoptimizingNode) {
1486 DeoptimizingNode replaceeDeopt = (DeoptimizingNode) replacee;
1487
1488 FrameState stateBefore = null;
1489 FrameState stateDuring = null;
1490 FrameState stateAfter = null;
1491 if (replaceeDeopt.canDeoptimize()) {
1492 if (replaceeDeopt instanceof DeoptimizingNode.DeoptBefore) {
1493 stateBefore = ((DeoptimizingNode.DeoptBefore) replaceeDeopt).stateBefore();
1494 }
1495 if (replaceeDeopt instanceof DeoptimizingNode.DeoptDuring) {
1496 stateDuring = ((DeoptimizingNode.DeoptDuring) replaceeDeopt).stateDuring();
1497 }
1498 if (replaceeDeopt instanceof DeoptimizingNode.DeoptAfter) {
1499 stateAfter = ((DeoptimizingNode.DeoptAfter) replaceeDeopt).stateAfter();
1500 }
1501 }
1502
1503 for (DeoptimizingNode deoptNode : deoptNodes) {
1504 DeoptimizingNode deoptDup = (DeoptimizingNode) duplicates.get(deoptNode.asNode());
1505 if (deoptDup.canDeoptimize()) {
1506 if (deoptDup instanceof DeoptimizingNode.DeoptBefore) {
1507 ((DeoptimizingNode.DeoptBefore) deoptDup).setStateBefore(stateBefore);
1508 }
1509 if (deoptDup instanceof DeoptimizingNode.DeoptDuring) {
1510 DeoptimizingNode.DeoptDuring deoptDupDuring = (DeoptimizingNode.DeoptDuring) deoptDup;
1511 if (stateDuring != null) {
1512 deoptDupDuring.setStateDuring(stateDuring);
1513 } else if (stateAfter != null) {
1514 deoptDupDuring.computeStateDuring(stateAfter);
1515 } else if (stateBefore != null) {
1516 assert !deoptDupDuring.hasSideEffect() : "can't use stateBefore as stateDuring for state split " + deoptDupDuring;
1517 deoptDupDuring.setStateDuring(stateBefore);
1518 }
1519 }
1520 if (deoptDup instanceof DeoptimizingNode.DeoptAfter) {
1521 DeoptimizingNode.DeoptAfter deoptDupAfter = (DeoptimizingNode.DeoptAfter) deoptDup;
1522 if (stateAfter != null) {
1523 deoptDupAfter.setStateAfter(stateAfter);
1524 } else {
1525 assert !deoptDupAfter.hasSideEffect() : "can't use stateBefore as stateAfter for state split " + deoptDupAfter;
1526 deoptDupAfter.setStateAfter(stateBefore);
1527 }
1528
1529 }
1530 }
1531 }
1532 }
1533
1534 updateStamps(replacee, duplicates);
1535
1536 rewireMemoryGraph(replacee, duplicates);
1537
1538 // Replace all usages of the replacee with the value returned by the snippet
1539 ValueNode returnValue = null;
1540 if (returnNode != null && !(replacee instanceof ControlSinkNode)) {
1541 ReturnNode returnDuplicate = (ReturnNode) duplicates.get(returnNode);
1542 returnValue = returnDuplicate.result();
1543 if (returnValue == null && replacee.usages().isNotEmpty() && replacee instanceof MemoryCheckpoint) {
1544 replacer.replace(replacee, null);
1545 } else {
1546 assert returnValue != null || replacee.hasNoUsages();
1547 replacer.replace(replacee, returnValue);
1548 }
1549 if (returnDuplicate.isAlive()) {
1550 FixedNode next = null;
1551 if (replacee instanceof FixedWithNextNode) {
1552 FixedWithNextNode fwn = (FixedWithNextNode) replacee;
1553 next = fwn.next();
1554 fwn.setNext(null);
1555 }
1556 returnDuplicate.replaceAndDelete(next);
1557 }
1558 }
1559
1560 if (killReplacee) {
1561 // Remove the replacee from its graph
1562 GraphUtil.killCFG(replacee);
1563 }
1564
1565 debug.dump(DebugContext.DETAILED_LEVEL, replaceeGraph, "After lowering %s with %s", replacee, this);
1566 return duplicates;
1567 }
1568 }
1569
1570 private UnmodifiableEconomicMap<Node, Node> inlineSnippet(Node replacee, DebugContext debug, StructuredGraph replaceeGraph, EconomicMap<Node, Node> replacements) {
1571 Mark mark = replaceeGraph.getMark();
1572 try (InliningLog.UpdateScope scope = replaceeGraph.getInliningLog().openUpdateScope((oldNode, newNode) -> {
1573 InliningLog log = replaceeGraph.getInliningLog();
1574 if (oldNode == null) {
1575 log.trackNewCallsite(newNode);
1576 }
1577 })) {
1578 UnmodifiableEconomicMap<Node, Node> duplicates = replaceeGraph.addDuplicates(nodes, snippet, snippet.getNodeCount(), replacements);
1579 if (scope != null) {
1580 replaceeGraph.getInliningLog().addLog(duplicates, snippet.getInliningLog());
1581 }
1582 NodeSourcePosition position = replacee.getNodeSourcePosition();
1583 InliningUtil.updateSourcePosition(replaceeGraph, duplicates, mark, position, true);
1584 debug.dump(DebugContext.DETAILED_LEVEL, replaceeGraph, "After inlining snippet %s", snippet.method());
1585 return duplicates;
1586 }
1587 }
1588
1589 private void propagateStamp(Node node) {
1590 if (node instanceof PhiNode) {
1591 PhiNode phi = (PhiNode) node;
1592 if (phi.inferStamp()) {
1593 for (Node usage : node.usages()) {
1594 propagateStamp(usage);
1595 }
1596 }
1597 }
1598 }
1599
1600 private void updateStamps(ValueNode replacee, UnmodifiableEconomicMap<Node, Node> duplicates) {
1601 for (ValueNode node : placeholderStampedNodes) {
1602 ValueNode dup = (ValueNode) duplicates.get(node);
1603 Stamp replaceeStamp = replacee.stamp(NodeView.DEFAULT);
1604 if (node instanceof Placeholder) {
1605 Placeholder placeholderDup = (Placeholder) dup;
1606 placeholderDup.makeReplacement(replaceeStamp);
1607 } else {
1608 dup.setStamp(replaceeStamp);
1609 }
1610 }
1611 for (ParameterNode paramNode : snippet.getNodes(ParameterNode.TYPE)) {
1612 for (Node usage : paramNode.usages()) {
1613 Node usageDup = duplicates.get(usage);
1614 propagateStamp(usageDup);
1615 }
1616 }
1617 }
1618
1619 /**
1620 * Gets a copy of the specialized graph.
1621 */
1622 public StructuredGraph copySpecializedGraph(DebugContext debugForCopy) {
1623 return (StructuredGraph) snippet.copy(debugForCopy);
1624 }
1625
1626 /**
1627 * Replaces a given floating node with this specialized snippet.
1628 *
1629 * @param metaAccess
1630 * @param replacee the node that will be replaced
1631 * @param replacer object that replaces the usages of {@code replacee}
1632 * @param tool lowering tool used to insert the snippet into the control-flow
1633 * @param args the arguments to be bound to the flattened positional parameters of the snippet
1634 */
1635 @SuppressWarnings("try")
1636 public void instantiate(MetaAccessProvider metaAccess, FloatingNode replacee, UsageReplacer replacer, LoweringTool tool, Arguments args) {
1637 DebugContext debug = replacee.getDebug();
1638 assert assertSnippetKills(replacee);
1639 try (DebugCloseable a = args.info.instantiationTimer.start(debug)) {
1640 args.info.instantiationCounter.increment(debug);
1641
1642 // Inline the snippet nodes, replacing parameters with the given args in the process
1643 StartNode entryPointNode = snippet.start();
1644 FixedNode firstCFGNode = entryPointNode.next();
1645 StructuredGraph replaceeGraph = replacee.graph();
1646 EconomicMap<Node, Node> replacements = bind(replaceeGraph, metaAccess, args);
1647 replacements.put(entryPointNode, tool.getCurrentGuardAnchor().asNode());
1648 UnmodifiableEconomicMap<Node, Node> duplicates = inlineSnippet(replacee, debug, replaceeGraph, replacements);
1649
1650 FixedWithNextNode lastFixedNode = tool.lastFixedNode();
1651 assert lastFixedNode != null && lastFixedNode.isAlive() : replaceeGraph + " lastFixed=" + lastFixedNode;
1652 FixedNode next = lastFixedNode.next();
1653 lastFixedNode.setNext(null);
1654 FixedNode firstCFGNodeDuplicate = (FixedNode) duplicates.get(firstCFGNode);
1655 replaceeGraph.addAfterFixed(lastFixedNode, firstCFGNodeDuplicate);
1656
1657 rewireFrameStates(replacee, duplicates);
1658 updateStamps(replacee, duplicates);
1659
1660 rewireMemoryGraph(replacee, duplicates);
1661
1662 // Replace all usages of the replacee with the value returned by the snippet
1663 ReturnNode returnDuplicate = (ReturnNode) duplicates.get(returnNode);
1664 ValueNode returnValue = returnDuplicate.result();
1665 assert returnValue != null || replacee.hasNoUsages();
1666 replacer.replace(replacee, returnValue);
1667
1668 if (returnDuplicate.isAlive()) {
1669 returnDuplicate.replaceAndDelete(next);
1670 }
1671
1672 debug.dump(DebugContext.DETAILED_LEVEL, replaceeGraph, "After lowering %s with %s", replacee, this);
1673 }
1674 }
1675
1676 /**
1677 * Replaces a given floating node with this specialized snippet.
1678 *
1679 * This snippet must be pure data-flow
1680 *
1681 * @param metaAccess
1682 * @param replacee the node that will be replaced
1683 * @param replacer object that replaces the usages of {@code replacee}
1684 * @param args the arguments to be bound to the flattened positional parameters of the snippet
1685 */
1686 @SuppressWarnings("try")
1687 public void instantiate(MetaAccessProvider metaAccess, FloatingNode replacee, UsageReplacer replacer, Arguments args) {
1688 DebugContext debug = replacee.getDebug();
1689 assert assertSnippetKills(replacee);
1690 try (DebugCloseable a = args.info.instantiationTimer.start(debug)) {
1691 args.info.instantiationCounter.increment(debug);
1692
1693 // Inline the snippet nodes, replacing parameters with the given args in the process
1694 StartNode entryPointNode = snippet.start();
1695 assert entryPointNode.next() == (memoryAnchor == null ? returnNode : memoryAnchor) : entryPointNode.next();
1696 StructuredGraph replaceeGraph = replacee.graph();
1697 EconomicMap<Node, Node> replacements = bind(replaceeGraph, metaAccess, args);
1698 MemoryAnchorNode anchorDuplicate = null;
1699 if (memoryAnchor != null) {
1700 anchorDuplicate = replaceeGraph.add(new MemoryAnchorNode());
1701 replacements.put(memoryAnchor, anchorDuplicate);
1702 }
1703 List<Node> floatingNodes = new ArrayList<>(nodes.size() - 2);
1704 for (Node n : nodes) {
1705 if (n != entryPointNode && n != returnNode) {
1706 floatingNodes.add(n);
1707 }
1708 }
1709 UnmodifiableEconomicMap<Node, Node> duplicates = inlineSnippet(replacee, debug, replaceeGraph, replacements);
1710
1711 rewireFrameStates(replacee, duplicates);
1712 updateStamps(replacee, duplicates);
1713
1714 rewireMemoryGraph(replacee, duplicates);
1715 assert anchorDuplicate == null || anchorDuplicate.isDeleted();
1716
1717 // Replace all usages of the replacee with the value returned by the snippet
1718 ValueNode returnValue = (ValueNode) duplicates.get(returnNode.result());
1719 replacer.replace(replacee, returnValue);
1720
1721 debug.dump(DebugContext.DETAILED_LEVEL, replaceeGraph, "After lowering %s with %s", replacee, this);
1722 }
1723 }
1724
1725 protected void rewireFrameStates(ValueNode replacee, UnmodifiableEconomicMap<Node, Node> duplicates) {
1726 if (replacee instanceof StateSplit) {
1727 for (StateSplit sideEffectNode : sideEffectNodes) {
1728 assert ((StateSplit) replacee).hasSideEffect();
1729 Node sideEffectDup = duplicates.get(sideEffectNode.asNode());
1730 ((StateSplit) sideEffectDup).setStateAfter(((StateSplit) replacee).stateAfter());
1731 }
1732 }
1733 }
1734
1735 @Override
1736 public String toString() {
1737 StringBuilder buf = new StringBuilder(snippet.toString()).append('(');
1738 String sep = "";
1739 for (int i = 0; i < parameters.length; i++) {
1740 String name = "[" + i + "]";
1741 Object value = parameters[i];
1742 buf.append(sep);
1743 sep = ", ";
1744 if (value == null) {
1745 buf.append("<null> ").append(name);
1746 } else if (value.equals(UNUSED_PARAMETER)) {
1747 buf.append("<unused> ").append(name);
1748 } else if (value.equals(CONSTANT_PARAMETER)) {
1749 buf.append("<constant> ").append(name);
1750 } else if (value instanceof ParameterNode) {
1751 ParameterNode param = (ParameterNode) value;
1752 buf.append(param.getStackKind().getJavaName()).append(' ').append(name);
1753 } else {
1754 ParameterNode[] params = (ParameterNode[]) value;
1755 String kind = params.length == 0 ? "?" : params[0].getStackKind().getJavaName();
1756 buf.append(kind).append('[').append(params.length).append("] ").append(name);
1757 }
1758 }
1759 return buf.append(')').toString();
1760 }
1761
1762 private static boolean checkTemplate(MetaAccessProvider metaAccess, Arguments args, ResolvedJavaMethod method, Signature signature) {
1763 int offset = args.info.hasReceiver() ? 1 : 0;
1764 for (int i = offset; i < args.info.getParameterCount(); i++) {
1765 if (args.info.isConstantParameter(i)) {
1766 JavaKind kind = signature.getParameterKind(i - offset);
1767 assert checkConstantArgument(metaAccess, method, signature, i - offset, args.info.getParameterName(i), args.values[i], kind);
1768
1769 } else if (args.info.isVarargsParameter(i)) {
1770 assert args.values[i] instanceof Varargs;
1771 Varargs varargs = (Varargs) args.values[i];
1772 assert checkVarargs(metaAccess, method, signature, i, args.info.getParameterName(i), varargs);
1773 }
1774 }
1775 return true;
1776 }
1777
1778 public void setMayRemoveLocation(boolean mayRemoveLocation) {
1779 this.mayRemoveLocation = mayRemoveLocation;
1780 }
1781 }