1 /*
2 * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24
25 package org.graalvm.compiler.replacements;
26
27 import static java.util.FormattableFlags.ALTERNATE;
28 import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE;
29 import static org.graalvm.compiler.debug.DebugContext.DEFAULT_LOG_STREAM;
30 import static org.graalvm.compiler.debug.DebugContext.applyFormattingFlagsAndWidth;
31 import static org.graalvm.compiler.debug.DebugOptions.DebugStubsAndSnippets;
32 import static org.graalvm.compiler.graph.iterators.NodePredicates.isNotA;
33 import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_IGNORED;
34 import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_IGNORED;
35 import static org.graalvm.compiler.phases.common.DeadCodeEliminationPhase.Optionality.Required;
36 import static jdk.internal.vm.compiler.word.LocationIdentity.any;
37
38 import java.lang.reflect.Array;
39 import java.lang.reflect.Method;
40 import java.util.ArrayList;
41 import java.util.Arrays;
42 import java.util.Collection;
43 import java.util.Collections;
44 import java.util.Formattable;
45 import java.util.Formatter;
46 import java.util.LinkedHashMap;
47 import java.util.List;
48 import java.util.Map;
49 import java.util.concurrent.atomic.AtomicInteger;
50 import java.util.concurrent.atomic.AtomicReference;
51 import java.util.function.Predicate;
52
53 import jdk.internal.vm.compiler.collections.EconomicMap;
54 import jdk.internal.vm.compiler.collections.EconomicSet;
55 import jdk.internal.vm.compiler.collections.Equivalence;
56 import jdk.internal.vm.compiler.collections.UnmodifiableEconomicMap;
57 import org.graalvm.compiler.api.replacements.Fold;
58 import org.graalvm.compiler.api.replacements.Snippet;
59 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
60 import org.graalvm.compiler.api.replacements.Snippet.NonNullParameter;
61 import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter;
62 import org.graalvm.compiler.api.replacements.SnippetReflectionProvider;
63 import org.graalvm.compiler.core.common.GraalOptions;
64 import org.graalvm.compiler.core.common.type.Stamp;
65 import org.graalvm.compiler.core.common.type.StampFactory;
66 import org.graalvm.compiler.core.common.type.StampPair;
67 import org.graalvm.compiler.core.common.type.TypeReference;
68 import org.graalvm.compiler.debug.Assertions;
69 import org.graalvm.compiler.debug.CounterKey;
70 import org.graalvm.compiler.debug.DebugCloseable;
71 import org.graalvm.compiler.debug.DebugContext;
72 import org.graalvm.compiler.debug.DebugContext.Description;
73 import org.graalvm.compiler.debug.DebugHandlersFactory;
74 import org.graalvm.compiler.debug.GraalError;
75 import org.graalvm.compiler.debug.TimerKey;
76 import org.graalvm.compiler.graph.Graph.Mark;
77 import org.graalvm.compiler.graph.Node;
78 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
79 import org.graalvm.compiler.graph.NodeClass;
80 import org.graalvm.compiler.graph.NodeSourcePosition;
81 import org.graalvm.compiler.graph.Position;
82 import org.graalvm.compiler.loop.LoopEx;
83 import org.graalvm.compiler.loop.LoopsData;
84 import org.graalvm.compiler.loop.phases.LoopTransformations;
85 import org.graalvm.compiler.nodeinfo.InputType;
86 import org.graalvm.compiler.nodeinfo.NodeInfo;
87 import org.graalvm.compiler.nodes.AbstractBeginNode;
88 import org.graalvm.compiler.nodes.AbstractMergeNode;
89 import org.graalvm.compiler.nodes.ConstantNode;
90 import org.graalvm.compiler.nodes.ControlSinkNode;
91 import org.graalvm.compiler.nodes.DeoptimizingNode;
92 import org.graalvm.compiler.nodes.FixedNode;
93 import org.graalvm.compiler.nodes.FixedWithNextNode;
94 import org.graalvm.compiler.nodes.FrameState;
95 import org.graalvm.compiler.nodes.InliningLog;
96 import org.graalvm.compiler.nodes.LoopBeginNode;
97 import org.graalvm.compiler.nodes.MergeNode;
98 import org.graalvm.compiler.nodes.NodeView;
99 import org.graalvm.compiler.nodes.ParameterNode;
100 import org.graalvm.compiler.nodes.PhiNode;
101 import org.graalvm.compiler.nodes.PiNode.Placeholder;
102 import org.graalvm.compiler.nodes.PiNode.PlaceholderStamp;
103 import org.graalvm.compiler.nodes.ReturnNode;
104 import org.graalvm.compiler.nodes.StartNode;
105 import org.graalvm.compiler.nodes.StateSplit;
106 import org.graalvm.compiler.nodes.StructuredGraph;
107 import org.graalvm.compiler.nodes.StructuredGraph.GuardsStage;
108 import org.graalvm.compiler.nodes.ValueNode;
109 import org.graalvm.compiler.nodes.ValueNodeUtil;
110 import org.graalvm.compiler.nodes.calc.FloatingNode;
111 import org.graalvm.compiler.nodes.java.LoadIndexedNode;
112 import org.graalvm.compiler.nodes.java.MethodCallTargetNode;
113 import org.graalvm.compiler.nodes.java.StoreIndexedNode;
114 import org.graalvm.compiler.nodes.memory.MemoryAccess;
115 import org.graalvm.compiler.nodes.memory.MemoryAnchorNode;
116 import org.graalvm.compiler.nodes.memory.MemoryCheckpoint;
117 import org.graalvm.compiler.nodes.memory.MemoryMap;
118 import org.graalvm.compiler.nodes.memory.MemoryMapNode;
119 import org.graalvm.compiler.nodes.memory.MemoryNode;
120 import org.graalvm.compiler.nodes.memory.MemoryPhiNode;
121 import org.graalvm.compiler.nodes.spi.ArrayLengthProvider;
122 import org.graalvm.compiler.nodes.spi.CoreProviders;
123 import org.graalvm.compiler.nodes.spi.LoweringTool;
124 import org.graalvm.compiler.nodes.spi.MemoryProxy;
125 import org.graalvm.compiler.nodes.util.GraphUtil;
126 import org.graalvm.compiler.options.Option;
127 import org.graalvm.compiler.options.OptionKey;
128 import org.graalvm.compiler.options.OptionValues;
129 import org.graalvm.compiler.phases.common.CanonicalizerPhase;
130 import org.graalvm.compiler.phases.common.DeadCodeEliminationPhase;
131 import org.graalvm.compiler.phases.common.FloatingReadPhase;
132 import org.graalvm.compiler.phases.common.FloatingReadPhase.MemoryMapImpl;
133 import org.graalvm.compiler.phases.common.GuardLoweringPhase;
134 import org.graalvm.compiler.phases.common.LoweringPhase;
135 import org.graalvm.compiler.phases.common.RemoveValueProxyPhase;
136 import org.graalvm.compiler.phases.common.inlining.InliningUtil;
137 import org.graalvm.compiler.phases.util.Providers;
138 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode;
139 import org.graalvm.compiler.replacements.nodes.LoadSnippetVarargParameterNode;
140 import org.graalvm.util.CollectionsUtil;
141 import jdk.internal.vm.compiler.word.LocationIdentity;
142 import jdk.internal.vm.compiler.word.WordBase;
143
144 import jdk.vm.ci.code.TargetDescription;
145 import jdk.vm.ci.meta.Constant;
146 import jdk.vm.ci.meta.ConstantReflectionProvider;
147 import jdk.vm.ci.meta.JavaConstant;
148 import jdk.vm.ci.meta.JavaKind;
149 import jdk.vm.ci.meta.Local;
150 import jdk.vm.ci.meta.LocalVariableTable;
151 import jdk.vm.ci.meta.MetaAccessProvider;
152 import jdk.vm.ci.meta.ResolvedJavaMethod;
153 import jdk.vm.ci.meta.ResolvedJavaMethod.Parameter;
154 import jdk.vm.ci.meta.ResolvedJavaType;
155 import jdk.vm.ci.meta.Signature;
156
157 /**
158 * A snippet template is a graph created by parsing a snippet method and then specialized by binding
159 * constants to the snippet's {@link ConstantParameter} parameters.
160 *
161 * Snippet templates can be managed in a cache maintained by {@link AbstractTemplates}.
162 */
163 public class SnippetTemplate {
164
165 private boolean mayRemoveLocation = false;
166
167 /**
168 * Holds the {@link ResolvedJavaMethod} of the snippet, together with some information about the
169 * method that needs to be computed only once. The {@link SnippetInfo} should be created once
170 * per snippet and then cached.
171 */
172 public abstract static class SnippetInfo {
173
174 protected final ResolvedJavaMethod method;
175 protected final ResolvedJavaMethod original;
176 protected final LocationIdentity[] privateLocations;
177 protected final Object receiver;
178
179 public Object getReceiver() {
180 return receiver;
181 }
182
183 boolean hasReceiver() {
184 assert hasReceiver(method) == (receiver != null) : "Snippet with the receiver must have it set as constant. Snippet: " + this;
185 return hasReceiver(method);
186 }
187
188 static boolean hasReceiver(ResolvedJavaMethod method) {
189 return method.hasReceiver();
190 }
191
192 /**
193 * Lazily constructed parts of {@link SnippetInfo}.
194 */
195 static class Lazy {
196 Lazy(ResolvedJavaMethod method) {
197 int count = method.getSignature().getParameterCount(hasReceiver(method));
198 constantParameters = new boolean[count];
199 varargsParameters = new boolean[count];
200 nonNullParameters = new boolean[count];
201 int offset = hasReceiver(method) ? 1 : 0;
202 for (int i = offset; i < count; i++) {
203 constantParameters[i] = method.getParameterAnnotation(ConstantParameter.class, i - offset) != null;
204 varargsParameters[i] = method.getParameterAnnotation(VarargsParameter.class, i - offset) != null;
205 nonNullParameters[i] = method.getParameterAnnotation(NonNullParameter.class, i - offset) != null;
206
207 assert !constantParameters[i - offset] || !varargsParameters[i - offset] : "Parameter cannot be annotated with both @" + ConstantParameter.class.getSimpleName() + " and @" +
208 VarargsParameter.class.getSimpleName();
209 }
210 if (method.hasReceiver()) {
211 // Receiver must be constant.
212 assert !constantParameters[0];
213 constantParameters[0] = true;
214 }
215
216 // Retrieve the names only when assertions are turned on. Parameter annotations are
217 // unsupported in the native image.
218 assert IS_IN_NATIVE_IMAGE || initNames(method, count);
219 }
220
221 final boolean[] constantParameters;
222 final boolean[] varargsParameters;
223 final boolean[] nonNullParameters;
224
225 /**
226 * The parameter names, taken from the local variables table. Only used for assertion
227 * checking, so use only within an assert statement.
228 */
229 String[] names;
230
231 private boolean initNames(ResolvedJavaMethod method, int parameterCount) {
232 names = new String[parameterCount];
233 int offset = 0;
234 if (method.hasReceiver()) {
235 names[0] = "this";
236 offset = 1;
237 }
238 Parameter[] params = method.getParameters();
239 if (params != null) {
240 for (int i = offset; i < names.length; i++) {
241 if (params[i - offset].isNamePresent()) {
242 names[i] = params[i - offset].getName();
243 }
244 }
245 } else {
246 int slotIdx = 0;
247 LocalVariableTable localVariableTable = method.getLocalVariableTable();
248 if (localVariableTable != null) {
249 for (int i = 0; i < names.length; i++) {
250 Local local = localVariableTable.getLocal(slotIdx, 0);
251 if (local != null) {
252 names[i] = local.getName();
253 }
254 JavaKind kind = method.getSignature().getParameterKind(i);
255 slotIdx += kind.getSlotCount();
256 }
257 }
258 }
259 return true;
260 }
261 }
262
263 /**
264 * Times instantiations of all templates derived from this snippet.
265 */
266 private final TimerKey instantiationTimer;
267
268 /**
269 * Counts instantiations of all templates derived from this snippet.
270 */
271 private final CounterKey instantiationCounter;
272
273 protected abstract Lazy lazy();
274
275 protected SnippetInfo(ResolvedJavaMethod method, ResolvedJavaMethod original, LocationIdentity[] privateLocations, Object receiver) {
276 this.method = method;
277 this.original = original;
278 this.privateLocations = privateLocations;
279 instantiationCounter = DebugContext.counter("SnippetInstantiationCount[%s]", method.getName());
280 instantiationTimer = DebugContext.timer("SnippetInstantiationTime[%s]", method.getName());
281 this.receiver = receiver;
282 }
283
284 public ResolvedJavaMethod getMethod() {
285 return method;
286 }
287
288 public int getParameterCount() {
289 return lazy().constantParameters.length;
290 }
291
292 public boolean isConstantParameter(int paramIdx) {
293 return lazy().constantParameters[paramIdx];
294 }
295
296 public boolean isVarargsParameter(int paramIdx) {
297 return lazy().varargsParameters[paramIdx];
298 }
299
300 public boolean isNonNullParameter(int paramIdx) {
301 return lazy().nonNullParameters[paramIdx];
302 }
303
304 public String getParameterName(int paramIdx) {
305 String[] names = lazy().names;
306 if (names != null) {
307 return names[paramIdx];
308 }
309 return null;
310 }
311
312 @Override
313 public String toString() {
314 return getClass().getSimpleName() + ":" + method.format("%h.%n");
315 }
316 }
317
318 protected static class LazySnippetInfo extends SnippetInfo {
319 protected final AtomicReference<Lazy> lazy = new AtomicReference<>(null);
320
321 protected LazySnippetInfo(ResolvedJavaMethod method, ResolvedJavaMethod original, LocationIdentity[] privateLocations, Object receiver) {
322 super(method, original, privateLocations, receiver);
323 }
324
325 @Override
326 protected Lazy lazy() {
327 if (lazy.get() == null) {
328 lazy.compareAndSet(null, new Lazy(method));
329 }
330 return lazy.get();
331 }
332 }
333
334 protected static class EagerSnippetInfo extends SnippetInfo {
335 protected final Lazy lazy;
336
337 protected EagerSnippetInfo(ResolvedJavaMethod method, ResolvedJavaMethod original, LocationIdentity[] privateLocations, Object receiver) {
338 super(method, original, privateLocations, receiver);
339 lazy = new Lazy(method);
340 }
341
342 @Override
343 protected Lazy lazy() {
344 return lazy;
345 }
346 }
347
348 /**
349 * Values that are bound to the snippet method parameters. The methods {@link #add},
350 * {@link #addConst}, and {@link #addVarargs} must be called in the same order as in the
351 * signature of the snippet method. The parameter name is passed to the add methods for
352 * assertion checking, i.e., to enforce that the order matches. Which method needs to be called
353 * depends on the annotation of the snippet method parameter:
354 * <ul>
355 * <li>Use {@link #add} for a parameter without an annotation. The value is bound when the
356 * {@link SnippetTemplate} is {@link SnippetTemplate#instantiate instantiated}.
357 * <li>Use {@link #addConst} for a parameter annotated with {@link ConstantParameter}. The value
358 * is bound when the {@link SnippetTemplate} is {@link SnippetTemplate#SnippetTemplate created}.
359 * <li>Use {@link #addVarargs} for an array parameter annotated with {@link VarargsParameter}. A
360 * separate {@link SnippetTemplate} is {@link SnippetTemplate#SnippetTemplate created} for every
361 * distinct array length. The actual values are bound when the {@link SnippetTemplate} is
362 * {@link SnippetTemplate#instantiate instantiated}
363 * </ul>
364 */
365 public static class Arguments implements Formattable {
366
367 protected final SnippetInfo info;
368 protected final CacheKey cacheKey;
369 protected final Object[] values;
370 protected final Stamp[] constStamps;
371 protected boolean cacheable;
372
373 protected int nextParamIdx;
374
375 public Arguments(SnippetInfo info, GuardsStage guardsStage, LoweringTool.LoweringStage loweringStage) {
376 this.info = info;
377 this.cacheKey = new CacheKey(info, guardsStage, loweringStage);
378 this.values = new Object[info.getParameterCount()];
379 this.constStamps = new Stamp[info.getParameterCount()];
380 this.cacheable = true;
381 if (info.hasReceiver()) {
382 addConst("this", info.getReceiver());
383 }
384 }
385
386 public Arguments add(String name, Object value) {
387 assert check(name, false, false);
388 values[nextParamIdx] = value;
389 nextParamIdx++;
390 return this;
391 }
392
393 public Arguments addConst(String name, Object value) {
394 assert value != null;
395 return addConst(name, value, null);
396 }
397
398 public Arguments addConst(String name, Object value, Stamp stamp) {
399 assert check(name, true, false);
400 values[nextParamIdx] = value;
401 constStamps[nextParamIdx] = stamp;
402 cacheKey.setParam(nextParamIdx, value);
403 nextParamIdx++;
404 return this;
405 }
406
407 public Arguments addVarargs(String name, Class<?> componentType, Stamp argStamp, Object value) {
408 assert check(name, false, true);
409 Varargs varargs = new Varargs(componentType, argStamp, value);
410 values[nextParamIdx] = varargs;
411 // A separate template is necessary for every distinct array length
412 cacheKey.setParam(nextParamIdx, varargs.length);
413 nextParamIdx++;
414 return this;
415 }
416
417 public void setCacheable(boolean cacheable) {
418 this.cacheable = cacheable;
419 }
420
421 private boolean check(String name, boolean constParam, boolean varargsParam) {
422 assert nextParamIdx < info.getParameterCount() : "too many parameters: " + name + " " + this;
423 assert info.getParameterName(nextParamIdx) == null || info.getParameterName(nextParamIdx).equals(name) : "wrong parameter name at " + nextParamIdx + " : " + name + " " + this;
424 assert constParam == info.isConstantParameter(nextParamIdx) : "Parameter " + (constParam ? "not " : "") + "annotated with @" + ConstantParameter.class.getSimpleName() + ": " + name +
425 " " + this;
426 assert varargsParam == info.isVarargsParameter(nextParamIdx) : "Parameter " + (varargsParam ? "not " : "") + "annotated with @" + VarargsParameter.class.getSimpleName() + ": " + name +
427 " " + this;
428 return true;
429 }
430
431 @Override
432 public String toString() {
433 StringBuilder result = new StringBuilder();
434 result.append("Parameters<").append(info.method.format("%h.%n")).append(" [");
435 String sep = "";
436 for (int i = 0; i < info.getParameterCount(); i++) {
437 result.append(sep);
438 if (info.isConstantParameter(i)) {
439 result.append("const ");
440 } else if (info.isVarargsParameter(i)) {
441 result.append("varargs ");
442 }
443 result.append(info.getParameterName(i)).append(" = ").append(values[i]);
444 sep = ", ";
445 }
446 result.append(">");
447 return result.toString();
448 }
449
450 @Override
451 public void formatTo(Formatter formatter, int flags, int width, int precision) {
452 if ((flags & ALTERNATE) == 0) {
453 formatter.format(applyFormattingFlagsAndWidth(toString(), flags, width));
454 } else {
455 StringBuilder sb = new StringBuilder();
456 sb.append(info.method.getName()).append('(');
457 String sep = "";
458 for (int i = 0; i < info.getParameterCount(); i++) {
459 if (info.isConstantParameter(i)) {
460 sb.append(sep);
461 if (info.getParameterName(i) != null) {
462 sb.append(info.getParameterName(i));
463 } else {
464 sb.append(i);
465 }
466 sb.append('=').append(values[i]);
467 sep = ", ";
468 }
469 }
470 sb.append(")");
471 String string = sb.toString();
472 if (string.indexOf('%') != -1) {
473 // Quote any % signs
474 string = string.replace("%", "%%");
475 }
476 formatter.format(applyFormattingFlagsAndWidth(string, flags & ~ALTERNATE, width));
477 }
478 }
479 }
480
481 /**
482 * Wrapper for the prototype value of a {@linkplain VarargsParameter varargs} parameter.
483 */
484 static class Varargs {
485
486 protected final Class<?> componentType;
487 protected final Stamp stamp;
488 protected final Object value;
489 protected final int length;
490
491 protected Varargs(Class<?> componentType, Stamp stamp, Object value) {
492 this.componentType = componentType;
493 this.stamp = stamp;
494 this.value = value;
495 if (value instanceof List) {
496 this.length = ((List<?>) value).size();
497 } else {
498 this.length = Array.getLength(value);
499 }
500 }
501
502 @Override
503 public String toString() {
504 if (value instanceof boolean[]) {
505 return Arrays.toString((boolean[]) value);
506 }
507 if (value instanceof byte[]) {
508 return Arrays.toString((byte[]) value);
509 }
510 if (value instanceof char[]) {
511 return Arrays.toString((char[]) value);
512 }
513 if (value instanceof short[]) {
514 return Arrays.toString((short[]) value);
515 }
516 if (value instanceof int[]) {
517 return Arrays.toString((int[]) value);
518 }
519 if (value instanceof long[]) {
520 return Arrays.toString((long[]) value);
521 }
522 if (value instanceof float[]) {
523 return Arrays.toString((float[]) value);
524 }
525 if (value instanceof double[]) {
526 return Arrays.toString((double[]) value);
527 }
528 if (value instanceof Object[]) {
529 return Arrays.toString((Object[]) value);
530 }
531 return String.valueOf(value);
532 }
533 }
534
535 @NodeInfo(cycles = CYCLES_IGNORED, size = SIZE_IGNORED)
536 static final class VarargsPlaceholderNode extends FloatingNode implements ArrayLengthProvider {
537
538 public static final NodeClass<VarargsPlaceholderNode> TYPE = NodeClass.create(VarargsPlaceholderNode.class);
539 protected final Varargs varargs;
540
541 protected VarargsPlaceholderNode(Varargs varargs, MetaAccessProvider metaAccess) {
542 super(TYPE, StampFactory.objectNonNull(TypeReference.createExactTrusted(metaAccess.lookupJavaType(varargs.componentType).getArrayClass())));
543 this.varargs = varargs;
544 }
545
546 @Override
547 public ValueNode findLength(FindLengthMode mode, ConstantReflectionProvider constantReflection) {
548 return ConstantNode.forInt(varargs.length);
549 }
550 }
551
552 static class CacheKey {
553
554 private final ResolvedJavaMethod method;
555 private final Object[] values;
556 private final GuardsStage guardsStage;
557 private final LoweringTool.LoweringStage loweringStage;
558 private int hash;
559
560 protected CacheKey(SnippetInfo info, GuardsStage guardsStage, LoweringTool.LoweringStage loweringStage) {
561 this.method = info.method;
562 this.guardsStage = guardsStage;
563 this.loweringStage = loweringStage;
564 this.values = new Object[info.getParameterCount()];
565 this.hash = info.method.hashCode() + 31 * guardsStage.ordinal();
566 }
567
568 protected void setParam(int paramIdx, Object value) {
569 values[paramIdx] = value;
570 hash = (hash * 31) ^ (value == null ? 0 : value.hashCode());
571 }
572
573 @Override
574 public boolean equals(Object obj) {
575 if (!(obj instanceof CacheKey)) {
576 return false;
577 }
578 CacheKey other = (CacheKey) obj;
579 if (!method.equals(other.method)) {
580 return false;
581 }
582 if (guardsStage != other.guardsStage || loweringStage != other.loweringStage) {
583 return false;
584 }
585 for (int i = 0; i < values.length; i++) {
586 if (values[i] != null && !values[i].equals(other.values[i])) {
587 return false;
588 }
589 }
590 return true;
591 }
592
593 @Override
594 public int hashCode() {
595 return hash;
596 }
597 }
598
599 private static final TimerKey SnippetTemplateCreationTime = DebugContext.timer("SnippetTemplateCreationTime");
600 private static final CounterKey SnippetTemplates = DebugContext.counter("SnippetTemplateCount");
601
602 static class Options {
603 @Option(help = "Use a LRU cache for snippet templates.")//
604 public static final OptionKey<Boolean> UseSnippetTemplateCache = new OptionKey<>(true);
605
606 @Option(help = "")//
607 static final OptionKey<Integer> MaxTemplatesPerSnippet = new OptionKey<>(50);
608 }
609
610 /**
611 * Base class for snippet classes. It provides a cache for {@link SnippetTemplate}s.
612 */
613 public abstract static class AbstractTemplates implements org.graalvm.compiler.api.replacements.SnippetTemplateCache {
614
615 protected final OptionValues options;
616 protected final Providers providers;
617 protected final SnippetReflectionProvider snippetReflection;
618 protected final Iterable<DebugHandlersFactory> factories;
619 protected final TargetDescription target;
620 private final Map<CacheKey, SnippetTemplate> templates;
621
622 protected AbstractTemplates(OptionValues options, Iterable<DebugHandlersFactory> factories, Providers providers, SnippetReflectionProvider snippetReflection, TargetDescription target) {
623 this.options = options;
624 this.providers = providers;
625 this.snippetReflection = snippetReflection;
626 this.target = target;
627 this.factories = factories;
628 if (Options.UseSnippetTemplateCache.getValue(options)) {
629 int size = Options.MaxTemplatesPerSnippet.getValue(options);
630 this.templates = Collections.synchronizedMap(new LRUCache<>(size, size));
631 } else {
632 this.templates = null;
633 }
634 }
635
636 public Providers getProviders() {
637 return providers;
638 }
639
640 public static Method findMethod(Class<? extends Snippets> declaringClass, String methodName, Method except) {
641 for (Method m : declaringClass.getDeclaredMethods()) {
642 if (m.getName().equals(methodName) && !m.equals(except)) {
643 return m;
644 }
645 }
646 return null;
647 }
648
649 public static ResolvedJavaMethod findMethod(MetaAccessProvider metaAccess, Class<?> declaringClass, String methodName) {
650 ResolvedJavaType type = metaAccess.lookupJavaType(declaringClass);
651 ResolvedJavaMethod result = null;
652 for (ResolvedJavaMethod m : type.getDeclaredMethods()) {
653 if (m.getName().equals(methodName)) {
654 if (!Assertions.assertionsEnabled()) {
655 return m;
656 } else {
657 assert result == null : "multiple definitions found";
658 result = m;
659 }
660 }
661 }
662 if (result == null) {
663 throw new GraalError("Could not find method in " + declaringClass + " named " + methodName);
664 }
665 return result;
666 }
667
668 protected SnippetInfo snippet(Class<? extends Snippets> declaringClass, String methodName, LocationIdentity... initialPrivateLocations) {
669 return snippet(declaringClass, methodName, null, null, initialPrivateLocations);
670 }
671
672 /**
673 * Finds the unique method in {@code declaringClass} named {@code methodName} annotated by
674 * {@link Snippet} and returns a {@link SnippetInfo} value describing it. There must be
675 * exactly one snippet method in {@code declaringClass} with a given name.
676 */
677 protected SnippetInfo snippet(Class<? extends Snippets> declaringClass, String methodName, ResolvedJavaMethod original, Object receiver, LocationIdentity... initialPrivateLocations) {
678 assert methodName != null;
679 ResolvedJavaMethod javaMethod = findMethod(providers.getMetaAccess(), declaringClass, methodName);
680 assert javaMethod != null : "did not find @" + Snippet.class.getSimpleName() + " method in " + declaringClass + " named " + methodName;
681 assert javaMethod.getAnnotation(Snippet.class) != null : javaMethod + " must be annotated with @" + Snippet.class.getSimpleName();
682 providers.getReplacements().registerSnippet(javaMethod, original, receiver, GraalOptions.TrackNodeSourcePosition.getValue(options), options);
683 LocationIdentity[] privateLocations = GraalOptions.SnippetCounters.getValue(options) ? SnippetCounterNode.addSnippetCounters(initialPrivateLocations) : initialPrivateLocations;
684 if (GraalOptions.EagerSnippets.getValue(options)) {
685 return new EagerSnippetInfo(javaMethod, original, privateLocations, receiver);
686 } else {
687 return new LazySnippetInfo(javaMethod, original, privateLocations, receiver);
688 }
689 }
690
691 static final AtomicInteger nextSnippetTemplateId = new AtomicInteger();
692
693 private DebugContext openDebugContext(DebugContext outer, Arguments args) {
694 if (DebugStubsAndSnippets.getValue(options)) {
695 Description description = new Description(args.cacheKey.method, "SnippetTemplate_" + nextSnippetTemplateId.incrementAndGet());
696 return DebugContext.create(options, description, outer.getGlobalMetrics(), DEFAULT_LOG_STREAM, factories);
697 }
698 return DebugContext.disabled(options);
699 }
700
701 /**
702 * Gets a template for a given key, creating it first if necessary.
703 */
704 @SuppressWarnings("try")
705 public SnippetTemplate template(ValueNode replacee, final Arguments args) {
706 StructuredGraph graph = replacee.graph();
707 DebugContext outer = graph.getDebug();
708 SnippetTemplate template = Options.UseSnippetTemplateCache.getValue(options) && args.cacheable ? templates.get(args.cacheKey) : null;
709 if (template == null || (graph.trackNodeSourcePosition() && !template.snippet.trackNodeSourcePosition())) {
710 try (DebugContext debug = openDebugContext(outer, args)) {
711 try (DebugCloseable a = SnippetTemplateCreationTime.start(debug); DebugContext.Scope s = debug.scope("SnippetSpecialization", args.info.method)) {
712 SnippetTemplates.increment(debug);
713 OptionValues snippetOptions = new OptionValues(options, GraalOptions.TraceInlining, GraalOptions.TraceInliningForStubsAndSnippets.getValue(options));
714 template = new SnippetTemplate(snippetOptions, debug, providers, snippetReflection, args, graph.trackNodeSourcePosition(), replacee);
715 if (Options.UseSnippetTemplateCache.getValue(snippetOptions) && args.cacheable) {
716 templates.put(args.cacheKey, template);
717 }
718 } catch (Throwable e) {
719 throw debug.handle(e);
720 }
721 }
722 }
723 return template;
724 }
725 }
726
727 private static final class LRUCache<K, V> extends LinkedHashMap<K, V> {
728 private static final long serialVersionUID = 1L;
729 private final int maxCacheSize;
730
731 LRUCache(int initialCapacity, int maxCacheSize) {
732 super(initialCapacity, 0.75F, true);
733 this.maxCacheSize = maxCacheSize;
734 }
735
736 @Override
737 protected boolean removeEldestEntry(java.util.Map.Entry<K, V> eldest) {
738 return size() > maxCacheSize;
739 }
740 }
741
742 // These values must be compared with equals() not '==' to support replay compilation.
743 private static final Object UNUSED_PARAMETER = "UNUSED_PARAMETER";
744 private static final Object CONSTANT_PARAMETER = "CONSTANT_PARAMETER";
745
746 /**
747 * Determines if any parameter of a given method is annotated with {@link ConstantParameter}.
748 */
749 public static boolean hasConstantParameter(ResolvedJavaMethod method) {
750 for (ConstantParameter p : method.getParameterAnnotations(ConstantParameter.class)) {
751 if (p != null) {
752 return true;
753 }
754 }
755 return false;
756 }
757
758 private final SnippetReflectionProvider snippetReflection;
759
760 /**
761 * Creates a snippet template.
762 */
763 @SuppressWarnings("try")
764 protected SnippetTemplate(OptionValues options, DebugContext debug, final Providers providers, SnippetReflectionProvider snippetReflection, Arguments args, boolean trackNodeSourcePosition,
765 Node replacee) {
766 this.snippetReflection = snippetReflection;
767 this.info = args.info;
768
769 Object[] constantArgs = getConstantArgs(args);
770 boolean shouldTrackNodeSourcePosition1 = trackNodeSourcePosition || (providers.getCodeCache() != null && providers.getCodeCache().shouldDebugNonSafepoints());
771 StructuredGraph snippetGraph = providers.getReplacements().getSnippet(args.info.method, args.info.original, constantArgs, shouldTrackNodeSourcePosition1, replacee.getNodeSourcePosition(),
772 options);
773
774 ResolvedJavaMethod method = snippetGraph.method();
775 Signature signature = method.getSignature();
776
777 // Copy snippet graph, replacing constant parameters with given arguments
778 final StructuredGraph snippetCopy = new StructuredGraph.Builder(options, debug).name(snippetGraph.name).method(snippetGraph.method()).trackNodeSourcePosition(
779 snippetGraph.trackNodeSourcePosition()).setIsSubstitution(true).build();
780 assert !GraalOptions.TrackNodeSourcePosition.getValue(options) || snippetCopy.trackNodeSourcePosition();
781 try (DebugContext.Scope scope = debug.scope("SpecializeSnippet", snippetCopy)) {
782 if (!snippetGraph.isUnsafeAccessTrackingEnabled()) {
783 snippetCopy.disableUnsafeAccessTracking();
784 }
785
786 EconomicMap<Node, Node> nodeReplacements = EconomicMap.create(Equivalence.IDENTITY);
787 nodeReplacements.put(snippetGraph.start(), snippetCopy.start());
788
789 MetaAccessProvider metaAccess = providers.getMetaAccess();
790 assert checkTemplate(metaAccess, args, method, signature);
791
792 int parameterCount = args.info.getParameterCount();
793 VarargsPlaceholderNode[] placeholders = new VarargsPlaceholderNode[parameterCount];
794
795 for (int i = 0; i < parameterCount; i++) {
796 ParameterNode parameter = snippetGraph.getParameter(i);
797 if (parameter != null) {
798 if (args.info.isConstantParameter(i)) {
799 Object arg = args.values[i];
800 JavaKind kind = signature.getParameterKind(i);
801 ConstantNode constantNode;
802 if (arg instanceof Constant) {
803 Stamp stamp = args.constStamps[i];
804 if (stamp == null) {
805 assert arg instanceof JavaConstant : "could not determine type of constant " + arg;
806 constantNode = ConstantNode.forConstant((JavaConstant) arg, metaAccess, snippetCopy);
807 } else {
808 constantNode = ConstantNode.forConstant(stamp, (Constant) arg, metaAccess, snippetCopy);
809 }
810 } else {
811 constantNode = ConstantNode.forConstant(snippetReflection.forBoxed(kind, arg), metaAccess, snippetCopy);
812 }
813 nodeReplacements.put(parameter, constantNode);
814 } else if (args.info.isVarargsParameter(i)) {
815 Varargs varargs = (Varargs) args.values[i];
816 VarargsPlaceholderNode placeholder = snippetCopy.unique(new VarargsPlaceholderNode(varargs, providers.getMetaAccess()));
817 nodeReplacements.put(parameter, placeholder);
818 placeholders[i] = placeholder;
819 } else if (args.info.isNonNullParameter(i)) {
820 parameter.setStamp(parameter.stamp(NodeView.DEFAULT).join(StampFactory.objectNonNull()));
821 }
822 }
823 }
824 try (InliningLog.UpdateScope updateScope = snippetCopy.getInliningLog().openDefaultUpdateScope()) {
825 UnmodifiableEconomicMap<Node, Node> duplicates = snippetCopy.addDuplicates(snippetGraph.getNodes(), snippetGraph, snippetGraph.getNodeCount(), nodeReplacements);
826 if (updateScope != null) {
827 snippetCopy.getInliningLog().replaceLog(duplicates, snippetGraph.getInliningLog());
828 }
829 }
830
831 debug.dump(DebugContext.INFO_LEVEL, snippetCopy, "Before specialization");
832
833 // Gather the template parameters
834 parameters = new Object[parameterCount];
835 for (int i = 0; i < parameterCount; i++) {
836 if (args.info.isConstantParameter(i)) {
837 parameters[i] = CONSTANT_PARAMETER;
838 } else if (args.info.isVarargsParameter(i)) {
839 assert snippetCopy.getParameter(i) == null;
840 Varargs varargs = (Varargs) args.values[i];
841 int length = varargs.length;
842 ParameterNode[] params = new ParameterNode[length];
843 Stamp stamp = varargs.stamp;
844 for (int j = 0; j < length; j++) {
845 // Use a decimal friendly numbering make it more obvious how values map
846 assert parameterCount < 10000;
847 int idx = (i + 1) * 10000 + j;
848 assert idx >= parameterCount : "collision in parameter numbering";
849 ParameterNode local = snippetCopy.addOrUnique(new ParameterNode(idx, StampPair.createSingle(stamp)));
850 params[j] = local;
851 }
852 parameters[i] = params;
853
854 VarargsPlaceholderNode placeholder = placeholders[i];
855 if (placeholder != null) {
856 for (Node usage : placeholder.usages().snapshot()) {
857 if (usage instanceof LoadIndexedNode) {
858 LoadIndexedNode loadIndexed = (LoadIndexedNode) usage;
859 debug.dump(DebugContext.INFO_LEVEL, snippetCopy, "Before replacing %s", loadIndexed);
860 LoadSnippetVarargParameterNode loadSnippetParameter = snippetCopy.add(
861 new LoadSnippetVarargParameterNode(params, loadIndexed.index(), loadIndexed.stamp(NodeView.DEFAULT)));
862 snippetCopy.replaceFixedWithFixed(loadIndexed, loadSnippetParameter);
863 debug.dump(DebugContext.INFO_LEVEL, snippetCopy, "After replacing %s", loadIndexed);
864 } else if (usage instanceof StoreIndexedNode) {
865 /*
866 * The template lowering doesn't really treat this as an array so
867 * you can't store back into the varargs. Allocate your own array if
868 * you really need this and EA should eliminate it.
869 */
870 throw new GraalError("Can't store into VarargsParameter array");
871 }
872 }
873 }
874 } else {
875 ParameterNode local = snippetCopy.getParameter(i);
876 if (local == null) {
877 // Parameter value was eliminated
878 parameters[i] = UNUSED_PARAMETER;
879 } else {
880 parameters[i] = local;
881 }
882 }
883 }
884
885 explodeLoops(snippetCopy, providers);
886
887 GuardsStage guardsStage = args.cacheKey.guardsStage;
888 // Perform lowering on the snippet
889 if (!guardsStage.allowsFloatingGuards()) {
890 new GuardLoweringPhase().apply(snippetCopy, null);
891 }
892 snippetCopy.setGuardsStage(guardsStage);
893 try (DebugContext.Scope s = debug.scope("LoweringSnippetTemplate", snippetCopy)) {
894 new LoweringPhase(new CanonicalizerPhase(), args.cacheKey.loweringStage).apply(snippetCopy, providers);
895 } catch (Throwable e) {
896 throw debug.handle(e);
897 }
898
899 ArrayList<StateSplit> curSideEffectNodes = new ArrayList<>();
900 ArrayList<DeoptimizingNode> curDeoptNodes = new ArrayList<>();
901 ArrayList<ValueNode> curPlaceholderStampedNodes = new ArrayList<>();
902 for (Node node : snippetCopy.getNodes()) {
903 if (node instanceof ValueNode) {
904 ValueNode valueNode = (ValueNode) node;
905 if (valueNode.stamp(NodeView.DEFAULT) == PlaceholderStamp.singleton()) {
906 curPlaceholderStampedNodes.add(valueNode);
907 }
908 }
909
910 if (node instanceof StateSplit) {
911 StateSplit stateSplit = (StateSplit) node;
912 FrameState frameState = stateSplit.stateAfter();
913 if (stateSplit.hasSideEffect()) {
914 curSideEffectNodes.add((StateSplit) node);
915 }
916 if (frameState != null) {
917 stateSplit.setStateAfter(null);
918 }
919 }
920 if (node instanceof DeoptimizingNode) {
921 DeoptimizingNode deoptNode = (DeoptimizingNode) node;
922 if (deoptNode.canDeoptimize()) {
923 curDeoptNodes.add(deoptNode);
924 }
925 }
926 }
927
928 new DeadCodeEliminationPhase(Required).apply(snippetCopy);
929
930 assert checkAllVarargPlaceholdersAreDeleted(parameterCount, placeholders);
931
932 new FloatingReadPhase(true, true).apply(snippetCopy);
933 new RemoveValueProxyPhase().apply(snippetCopy);
934
935 MemoryAnchorNode anchor = snippetCopy.add(new MemoryAnchorNode());
936 snippetCopy.start().replaceAtUsages(InputType.Memory, anchor);
937
938 this.snippet = snippetCopy;
939
940 StartNode entryPointNode = snippet.start();
941 if (anchor.hasNoUsages()) {
942 anchor.safeDelete();
943 this.memoryAnchor = null;
944 } else {
945 // Find out if all the return memory maps point to the anchor (i.e., there's no kill
946 // anywhere)
947 boolean needsMemoryMaps = false;
948 for (ReturnNode retNode : snippet.getNodes(ReturnNode.TYPE)) {
949 MemoryMapNode memoryMap = retNode.getMemoryMap();
950 if (memoryMap.getLocations().size() > 1 || memoryMap.getLastLocationAccess(LocationIdentity.any()) != anchor) {
951 needsMemoryMaps = true;
952 break;
953 }
954 }
955 boolean needsAnchor;
956 if (needsMemoryMaps) {
957 needsAnchor = true;
958 } else {
959 // Check that all those memory maps where the only usages of the anchor
960 needsAnchor = anchor.usages().filter(isNotA(MemoryMapNode.class)).isNotEmpty();
961 // Remove the useless memory map
962 MemoryMapNode memoryMap = null;
963 for (ReturnNode retNode : snippet.getNodes(ReturnNode.TYPE)) {
964 if (memoryMap == null) {
965 memoryMap = retNode.getMemoryMap();
966 } else {
967 assert memoryMap == retNode.getMemoryMap();
968 }
969 retNode.setMemoryMap(null);
970 }
971 if (memoryMap != null) {
972 memoryMap.safeDelete();
973 }
974 }
975 if (needsAnchor) {
976 snippetCopy.addAfterFixed(snippetCopy.start(), anchor);
977 this.memoryAnchor = anchor;
978 } else {
979 anchor.safeDelete();
980 this.memoryAnchor = null;
981 }
982 }
983 debug.dump(DebugContext.INFO_LEVEL, snippet, "SnippetTemplate after fixing memory anchoring");
984
985 List<ReturnNode> returnNodes = snippet.getNodes(ReturnNode.TYPE).snapshot();
986 if (returnNodes.isEmpty()) {
987 this.returnNode = null;
988 } else if (returnNodes.size() == 1) {
989 this.returnNode = returnNodes.get(0);
990 } else {
991 AbstractMergeNode merge = snippet.add(new MergeNode());
992 List<MemoryMapNode> memMaps = new ArrayList<>();
993 for (ReturnNode retNode : returnNodes) {
994 MemoryMapNode memoryMapNode = retNode.getMemoryMap();
995 if (memoryMapNode != null) {
996 memMaps.add(memoryMapNode);
997 }
998 }
999
1000 ValueNode returnValue = InliningUtil.mergeReturns(merge, returnNodes);
1001 this.returnNode = snippet.add(new ReturnNode(returnValue));
1002 if (!memMaps.isEmpty()) {
1003 MemoryMapImpl mmap = FloatingReadPhase.mergeMemoryMaps(merge, memMaps);
1004 MemoryMapNode memoryMap = snippet.unique(new MemoryMapNode(mmap.getMap()));
1005 this.returnNode.setMemoryMap(memoryMap);
1006 for (MemoryMapNode mm : memMaps) {
1007 if (mm != memoryMap && mm.isAlive()) {
1008 assert mm.hasNoUsages();
1009 GraphUtil.killWithUnusedFloatingInputs(mm);
1010 }
1011 }
1012 }
1013 merge.setNext(this.returnNode);
1014 }
1015
1016 assert verifyIntrinsicsProcessed(snippetCopy);
1017
1018 this.sideEffectNodes = curSideEffectNodes;
1019 this.deoptNodes = curDeoptNodes;
1020 this.placeholderStampedNodes = curPlaceholderStampedNodes;
1021
1022 nodes = new ArrayList<>(snippet.getNodeCount());
1023 for (Node node : snippet.getNodes()) {
1024 if (node != entryPointNode && node != entryPointNode.stateAfter()) {
1025 nodes.add(node);
1026 }
1027 }
1028
1029 if (debug.areMetricsEnabled()) {
1030 DebugContext.counter("SnippetTemplateNodeCount[%#s]", args).add(debug, nodes.size());
1031 }
1032 debug.dump(DebugContext.INFO_LEVEL, snippet, "SnippetTemplate final state");
1033 this.snippet.freeze();
1034
1035 } catch (Throwable ex) {
1036 throw debug.handle(ex);
1037 }
1038 }
1039
1040 private static boolean verifyIntrinsicsProcessed(StructuredGraph snippetCopy) {
1041 for (MethodCallTargetNode target : snippetCopy.getNodes(MethodCallTargetNode.TYPE)) {
1042 ResolvedJavaMethod targetMethod = target.targetMethod();
1043 if (targetMethod != null) {
1044 assert targetMethod.getAnnotation(Fold.class) == null && targetMethod.getAnnotation(NodeIntrinsic.class) == null : "plugin should have been processed";
1045 }
1046 }
1047 return true;
1048 }
1049
1050 public static void explodeLoops(final StructuredGraph snippetCopy, CoreProviders providers) {
1051 // Do any required loop explosion
1052 boolean exploded = false;
1053 do {
1054 exploded = false;
1055 ExplodeLoopNode explodeLoop = snippetCopy.getNodes().filter(ExplodeLoopNode.class).first();
1056 if (explodeLoop != null) { // Earlier canonicalization may have removed the loop
1057 // altogether
1058 LoopBeginNode loopBegin = explodeLoop.findLoopBegin();
1059 if (loopBegin != null) {
1060 LoopEx loop = new LoopsData(snippetCopy).loop(loopBegin);
1061 Mark mark = snippetCopy.getMark();
1062 LoopTransformations.fullUnroll(loop, providers, new CanonicalizerPhase());
1063 new CanonicalizerPhase().applyIncremental(snippetCopy, providers, mark, false);
1064 loop.deleteUnusedNodes();
1065 }
1066 GraphUtil.removeFixedWithUnusedInputs(explodeLoop);
1067 exploded = true;
1068 }
1069 } while (exploded);
1070 }
1071
1072 protected Object[] getConstantArgs(Arguments args) {
1073 Object[] constantArgs = args.values.clone();
1074 for (int i = 0; i < args.info.getParameterCount(); i++) {
1075 if (!args.info.isConstantParameter(i)) {
1076 constantArgs[i] = null;
1077 } else {
1078 assert constantArgs[i] != null : "Can't pass raw null through as argument";
1079 }
1080 }
1081 return constantArgs;
1082 }
1083
1084 private static boolean checkAllVarargPlaceholdersAreDeleted(int parameterCount, VarargsPlaceholderNode[] placeholders) {
1085 for (int i = 0; i < parameterCount; i++) {
1086 if (placeholders[i] != null) {
1087 assert placeholders[i].isDeleted() : placeholders[i];
1088 }
1089 }
1090 return true;
1091 }
1092
1093 private static boolean checkConstantArgument(MetaAccessProvider metaAccess, final ResolvedJavaMethod method, Signature signature, int paramIndex, String name, Object arg, JavaKind kind) {
1094 ResolvedJavaType type = signature.getParameterType(paramIndex, method.getDeclaringClass()).resolve(method.getDeclaringClass());
1095 if (metaAccess.lookupJavaType(WordBase.class).isAssignableFrom(type)) {
1096 assert arg instanceof JavaConstant : method + ": word constant parameters must be passed boxed in a Constant value: " + arg;
1097 return true;
1098 }
1099 if (kind != JavaKind.Object) {
1100 assert arg != null && kind.toBoxedJavaClass() == arg.getClass() : method + ": wrong value kind for " + name + ": expected " + kind + ", got " +
1101 (arg == null ? "null" : arg.getClass().getSimpleName());
1102 }
1103 return true;
1104 }
1105
1106 private static boolean checkVarargs(MetaAccessProvider metaAccess, final ResolvedJavaMethod method, Signature signature, int i, String name, Varargs varargs) {
1107 ResolvedJavaType type = (ResolvedJavaType) signature.getParameterType(i, method.getDeclaringClass());
1108 assert type.isArray() : "varargs parameter must be an array type";
1109 assert type.getComponentType().isAssignableFrom(metaAccess.lookupJavaType(varargs.componentType)) : "componentType for " + name + " not matching " + type.toJavaName() + " instance: " +
1110 varargs.componentType;
1111 return true;
1112 }
1113
1114 /**
1115 * The graph built from the snippet method.
1116 */
1117 private final StructuredGraph snippet;
1118
1119 private final SnippetInfo info;
1120
1121 /**
1122 * The named parameters of this template that must be bound to values during instantiation. For
1123 * a parameter that is still live after specialization, the value in this map is either a
1124 * {@link ParameterNode} instance or a {@link ParameterNode} array. For an eliminated parameter,
1125 * the value is identical to the key.
1126 */
1127 private final Object[] parameters;
1128
1129 /**
1130 * The return node (if any) of the snippet.
1131 */
1132 private final ReturnNode returnNode;
1133
1134 /**
1135 * The memory anchor (if any) of the snippet.
1136 */
1137 private final MemoryAnchorNode memoryAnchor;
1138
1139 /**
1140 * Nodes that inherit the {@link StateSplit#stateAfter()} from the replacee during
1141 * instantiation.
1142 */
1143 private final ArrayList<StateSplit> sideEffectNodes;
1144
1145 /**
1146 * Nodes that inherit a deoptimization {@link FrameState} from the replacee during
1147 * instantiation.
1148 */
1149 private final ArrayList<DeoptimizingNode> deoptNodes;
1150
1151 /**
1152 * Nodes that have a stamp originating from a {@link Placeholder}.
1153 */
1154 private final ArrayList<ValueNode> placeholderStampedNodes;
1155
1156 /**
1157 * The nodes to be inlined when this specialization is instantiated.
1158 */
1159 private final ArrayList<Node> nodes;
1160
1161 /**
1162 * Gets the instantiation-time bindings to this template's parameters.
1163 *
1164 * @return the map that will be used to bind arguments to parameters when inlining this template
1165 */
1166 private EconomicMap<Node, Node> bind(StructuredGraph replaceeGraph, MetaAccessProvider metaAccess, Arguments args) {
1167 EconomicMap<Node, Node> replacements = EconomicMap.create(Equivalence.IDENTITY);
1168 assert args.info.getParameterCount() == parameters.length : "number of args (" + args.info.getParameterCount() + ") != number of parameters (" + parameters.length + ")";
1169 for (int i = 0; i < parameters.length; i++) {
1170 Object parameter = parameters[i];
1171 assert parameter != null : this + " has no parameter named " + args.info.getParameterName(i);
1172 Object argument = args.values[i];
1173 if (parameter instanceof ParameterNode) {
1174 if (argument instanceof ValueNode) {
1175 replacements.put((ParameterNode) parameter, (ValueNode) argument);
1176 } else {
1177 JavaKind kind = ((ParameterNode) parameter).getStackKind();
1178 assert argument != null || kind == JavaKind.Object : this + " cannot accept null for non-object parameter named " + args.info.getParameterName(i);
1179 JavaConstant constant = forBoxed(argument, kind);
1180 replacements.put((ParameterNode) parameter, ConstantNode.forConstant(constant, metaAccess, replaceeGraph));
1181 }
1182 } else if (parameter instanceof ParameterNode[]) {
1183 ParameterNode[] params = (ParameterNode[]) parameter;
1184 Varargs varargs = (Varargs) argument;
1185 int length = params.length;
1186 List<?> list = null;
1187 Object array = null;
1188 if (varargs.value instanceof List) {
1189 list = (List<?>) varargs.value;
1190 assert list.size() == length : length + " != " + list.size();
1191 } else {
1192 array = varargs.value;
1193 assert array != null && array.getClass().isArray();
1194 assert Array.getLength(array) == length : length + " != " + Array.getLength(array);
1195 }
1196
1197 for (int j = 0; j < length; j++) {
1198 ParameterNode param = params[j];
1199 assert param != null;
1200 Object value = list != null ? list.get(j) : Array.get(array, j);
1201 if (value instanceof ValueNode) {
1202 replacements.put(param, (ValueNode) value);
1203 } else {
1204 JavaConstant constant = forBoxed(value, param.getStackKind());
1205 ConstantNode element = ConstantNode.forConstant(constant, metaAccess, replaceeGraph);
1206 replacements.put(param, element);
1207 }
1208 }
1209 } else {
1210 assert parameter.equals(CONSTANT_PARAMETER) || parameter.equals(UNUSED_PARAMETER) : "unexpected entry for parameter: " + args.info.getParameterName(i) + " -> " + parameter;
1211 }
1212 }
1213 return replacements;
1214 }
1215
1216 /**
1217 * Converts a Java boxed value to a {@link JavaConstant} of the right kind. This adjusts for the
1218 * limitation that a {@link Local}'s kind is a {@linkplain JavaKind#getStackKind() stack kind}
1219 * and so cannot be used for re-boxing primitives smaller than an int.
1220 *
1221 * @param argument a Java boxed value
1222 * @param localKind the kind of the {@link Local} to which {@code argument} will be bound
1223 */
1224 protected JavaConstant forBoxed(Object argument, JavaKind localKind) {
1225 assert localKind == localKind.getStackKind();
1226 if (localKind == JavaKind.Int) {
1227 return JavaConstant.forBoxedPrimitive(argument);
1228 }
1229 return snippetReflection.forBoxed(localKind, argument);
1230 }
1231
1232 /**
1233 * Logic for replacing a snippet-lowered node at its usages with the return value of the
1234 * snippet. An alternative to the {@linkplain SnippetTemplate#DEFAULT_REPLACER default}
1235 * replacement logic can be used to handle mismatches between the stamp of the node being
1236 * lowered and the stamp of the snippet's return value.
1237 */
1238 public interface UsageReplacer {
1239 /**
1240 * Replaces all usages of {@code oldNode} with direct or indirect usages of {@code newNode}.
1241 */
1242 void replace(ValueNode oldNode, ValueNode newNode);
1243 }
1244
1245 /**
1246 * Represents the default {@link UsageReplacer usage replacer} logic which simply delegates to
1247 * {@link Node#replaceAtUsages(Node)}.
1248 */
1249 public static final UsageReplacer DEFAULT_REPLACER = new UsageReplacer() {
1250
1251 @Override
1252 public void replace(ValueNode oldNode, ValueNode newNode) {
1253 if (newNode == null) {
1254 assert oldNode.hasNoUsages();
1255 } else {
1256 oldNode.replaceAtUsages(newNode);
1257 }
1258 }
1259 };
1260
1261 private boolean assertSnippetKills(ValueNode replacee) {
1262 if (!replacee.graph().isAfterFloatingReadPhase()) {
1263 // no floating reads yet, ignore locations created while lowering
1264 return true;
1265 }
1266 if (returnNode == null) {
1267 // The snippet terminates control flow
1268 return true;
1269 }
1270 MemoryMapNode memoryMap = returnNode.getMemoryMap();
1271 if (memoryMap == null || memoryMap.isEmpty()) {
1272 // there are no kills in the snippet graph
1273 return true;
1274 }
1275
1276 EconomicSet<LocationIdentity> kills = EconomicSet.create(Equivalence.DEFAULT);
1277 kills.addAll(memoryMap.getLocations());
1278
1279 if (replacee instanceof MemoryCheckpoint.Single) {
1280 // check if some node in snippet graph also kills the same location
1281 LocationIdentity locationIdentity = ((MemoryCheckpoint.Single) replacee).getLocationIdentity();
1282 if (locationIdentity.isAny()) {
1283 assert !(memoryMap.getLastLocationAccess(any()) instanceof MemoryAnchorNode) : replacee + " kills ANY_LOCATION, but snippet does not";
1284 // if the replacee kills ANY_LOCATION, the snippet can kill arbitrary locations
1285 return true;
1286 }
1287 assert kills.contains(locationIdentity) : replacee + " kills " + locationIdentity + ", but snippet doesn't contain a kill to this location";
1288 kills.remove(locationIdentity);
1289 }
1290 assert !(replacee instanceof MemoryCheckpoint.Multi) : replacee + " multi not supported (yet)";
1291
1292 // remove ANY_LOCATION if it's just a kill by the start node
1293 if (memoryMap.getLastLocationAccess(any()) instanceof MemoryAnchorNode) {
1294 kills.remove(any());
1295 }
1296
1297 // node can only lower to a ANY_LOCATION kill if the replacee also kills ANY_LOCATION
1298 assert !kills.contains(any()) : "snippet graph contains a kill to ANY_LOCATION, but replacee (" + replacee + ") doesn't kill ANY_LOCATION. kills: " + kills;
1299
1300 /*
1301 * Kills to private locations are safe, since there can be no floating read to these
1302 * locations except reads that are introduced by the snippet itself or related snippets in
1303 * the same lowering round. These reads are anchored to a MemoryAnchor at the beginning of
1304 * their snippet, so they can not float above a kill in another instance of the same
1305 * snippet.
1306 */
1307 for (LocationIdentity p : this.info.privateLocations) {
1308 kills.remove(p);
1309 }
1310
1311 assert kills.isEmpty() : "snippet graph kills non-private locations " + kills + " that replacee (" + replacee + ") doesn't kill";
1312 return true;
1313 }
1314
1315 private static class MemoryInputMap implements MemoryMap {
1316
1317 private final LocationIdentity locationIdentity;
1318 private final MemoryNode lastLocationAccess;
1319
1320 MemoryInputMap(ValueNode replacee) {
1321 if (replacee instanceof MemoryAccess) {
1322 MemoryAccess access = (MemoryAccess) replacee;
1323 locationIdentity = access.getLocationIdentity();
1324 lastLocationAccess = access.getLastLocationAccess();
1325 } else {
1326 locationIdentity = null;
1327 lastLocationAccess = null;
1328 }
1329 }
1330
1331 @Override
1332 public MemoryNode getLastLocationAccess(LocationIdentity location) {
1333 if (locationIdentity != null && locationIdentity.equals(location)) {
1334 return lastLocationAccess;
1335 } else {
1336 return null;
1337 }
1338 }
1339
1340 @Override
1341 public Collection<LocationIdentity> getLocations() {
1342 if (locationIdentity == null) {
1343 return Collections.emptySet();
1344 } else {
1345 return Collections.singleton(locationIdentity);
1346 }
1347 }
1348 }
1349
1350 private class MemoryOutputMap extends MemoryInputMap {
1351
1352 private final UnmodifiableEconomicMap<Node, Node> duplicates;
1353
1354 MemoryOutputMap(ValueNode replacee, UnmodifiableEconomicMap<Node, Node> duplicates) {
1355 super(replacee);
1356 this.duplicates = duplicates;
1357 }
1358
1359 @Override
1360 public MemoryNode getLastLocationAccess(LocationIdentity locationIdentity) {
1361 MemoryMapNode memoryMap = returnNode.getMemoryMap();
1362 assert memoryMap != null : "no memory map stored for this snippet graph (snippet doesn't have a ReturnNode?)";
1363 MemoryNode lastLocationAccess = memoryMap.getLastLocationAccess(locationIdentity);
1364 assert lastLocationAccess != null : locationIdentity;
1365 if (lastLocationAccess == memoryAnchor) {
1366 return super.getLastLocationAccess(locationIdentity);
1367 } else {
1368 return (MemoryNode) duplicates.get(ValueNodeUtil.asNode(lastLocationAccess));
1369 }
1370 }
1371
1372 @Override
1373 public Collection<LocationIdentity> getLocations() {
1374 return returnNode.getMemoryMap().getLocations();
1375 }
1376 }
1377
1378 private void rewireMemoryGraph(ValueNode replacee, UnmodifiableEconomicMap<Node, Node> duplicates) {
1379 if (replacee.graph().isAfterFloatingReadPhase()) {
1380 // rewire outgoing memory edges
1381 replaceMemoryUsages(replacee, new MemoryOutputMap(replacee, duplicates));
1382
1383 if (returnNode != null) {
1384 ReturnNode ret = (ReturnNode) duplicates.get(returnNode);
1385 if (ret != null) {
1386 MemoryMapNode memoryMap = ret.getMemoryMap();
1387 if (memoryMap != null) {
1388 ret.setMemoryMap(null);
1389 memoryMap.safeDelete();
1390 }
1391 }
1392 }
1393 if (memoryAnchor != null) {
1394 // rewire incoming memory edges
1395 MemoryAnchorNode memoryDuplicate = (MemoryAnchorNode) duplicates.get(memoryAnchor);
1396 replaceMemoryUsages(memoryDuplicate, new MemoryInputMap(replacee));
1397
1398 if (memoryDuplicate.hasNoUsages()) {
1399 if (memoryDuplicate.next() != null) {
1400 memoryDuplicate.graph().removeFixed(memoryDuplicate);
1401 } else {
1402 // this was a dummy memory node used when instantiating pure data-flow
1403 // snippets: it was not attached to the control flow.
1404 memoryDuplicate.safeDelete();
1405 }
1406 }
1407 }
1408 }
1409 }
1410
1411 private static LocationIdentity getLocationIdentity(Node node) {
1412 if (node instanceof MemoryAccess) {
1413 return ((MemoryAccess) node).getLocationIdentity();
1414 } else if (node instanceof MemoryProxy) {
1415 return ((MemoryProxy) node).getLocationIdentity();
1416 } else if (node instanceof MemoryPhiNode) {
1417 return ((MemoryPhiNode) node).getLocationIdentity();
1418 } else {
1419 return null;
1420 }
1421 }
1422
1423 private void replaceMemoryUsages(ValueNode node, MemoryMap map) {
1424 for (Node usage : node.usages().snapshot()) {
1425 if (usage instanceof MemoryMapNode) {
1426 continue;
1427 }
1428
1429 LocationIdentity location = getLocationIdentity(usage);
1430 if (location != null) {
1431 for (Position pos : usage.inputPositions()) {
1432 if (pos.getInputType() == InputType.Memory && pos.get(usage) == node) {
1433 MemoryNode replacement = map.getLastLocationAccess(location);
1434 if (replacement == null) {
1435 assert mayRemoveLocation || LocationIdentity.any().equals(location) ||
1436 CollectionsUtil.anyMatch(info.privateLocations, Predicate.isEqual(location)) : "Snippet " +
1437 info.method.format("%h.%n") + " contains access to the non-private location " +
1438 location + ", but replacee doesn't access this location." + map.getLocations();
1439 } else {
1440 pos.set(usage, replacement.asNode());
1441 }
1442 }
1443 }
1444 }
1445 }
1446 }
1447
1448 /**
1449 * Replaces a given fixed node with this specialized snippet.
1450 *
1451 * @param metaAccess
1452 * @param replacee the node that will be replaced
1453 * @param replacer object that replaces the usages of {@code replacee}
1454 * @param args the arguments to be bound to the flattened positional parameters of the snippet
1455 * @return the map of duplicated nodes (original -> duplicate)
1456 */
1457 @SuppressWarnings("try")
1458 public UnmodifiableEconomicMap<Node, Node> instantiate(MetaAccessProvider metaAccess, FixedNode replacee, UsageReplacer replacer, Arguments args) {
1459 return instantiate(metaAccess, replacee, replacer, args, true);
1460 }
1461
1462 /**
1463 * Replaces a given fixed node with this specialized snippet.
1464 *
1465 * @param metaAccess
1466 * @param replacee the node that will be replaced
1467 * @param replacer object that replaces the usages of {@code replacee}
1468 * @param args the arguments to be bound to the flattened positional parameters of the snippet
1469 * @param killReplacee is true, the replacee node is deleted
1470 * @return the map of duplicated nodes (original -> duplicate)
1471 */
1472 @SuppressWarnings("try")
1473 public UnmodifiableEconomicMap<Node, Node> instantiate(MetaAccessProvider metaAccess, FixedNode replacee, UsageReplacer replacer, Arguments args, boolean killReplacee) {
1474 DebugContext debug = replacee.getDebug();
1475 assert assertSnippetKills(replacee);
1476 try (DebugCloseable a = args.info.instantiationTimer.start(debug)) {
1477 args.info.instantiationCounter.increment(debug);
1478 // Inline the snippet nodes, replacing parameters with the given args in the process
1479 StartNode entryPointNode = snippet.start();
1480 FixedNode firstCFGNode = entryPointNode.next();
1481 StructuredGraph replaceeGraph = replacee.graph();
1482 EconomicMap<Node, Node> replacements = bind(replaceeGraph, metaAccess, args);
1483 replacements.put(entryPointNode, AbstractBeginNode.prevBegin(replacee));
1484 UnmodifiableEconomicMap<Node, Node> duplicates = inlineSnippet(replacee, debug, replaceeGraph, replacements);
1485
1486 // Re-wire the control flow graph around the replacee
1487 FixedNode firstCFGNodeDuplicate = (FixedNode) duplicates.get(firstCFGNode);
1488 replacee.replaceAtPredecessor(firstCFGNodeDuplicate);
1489
1490 rewireFrameStates(replacee, duplicates);
1491
1492 updateStamps(replacee, duplicates);
1493
1494 rewireMemoryGraph(replacee, duplicates);
1495
1496 // Replace all usages of the replacee with the value returned by the snippet
1497 ValueNode returnValue = null;
1498 if (returnNode != null && !(replacee instanceof ControlSinkNode)) {
1499 ReturnNode returnDuplicate = (ReturnNode) duplicates.get(returnNode);
1500 returnValue = returnDuplicate.result();
1501 if (returnValue == null && replacee.usages().isNotEmpty() && replacee instanceof MemoryCheckpoint) {
1502 replacer.replace(replacee, null);
1503 } else {
1504 assert returnValue != null || replacee.hasNoUsages();
1505 replacer.replace(replacee, returnValue);
1506 }
1507 if (returnDuplicate.isAlive()) {
1508 FixedNode next = null;
1509 if (replacee instanceof FixedWithNextNode) {
1510 FixedWithNextNode fwn = (FixedWithNextNode) replacee;
1511 next = fwn.next();
1512 fwn.setNext(null);
1513 }
1514 returnDuplicate.replaceAndDelete(next);
1515 }
1516 }
1517
1518 if (killReplacee) {
1519 // Remove the replacee from its graph
1520 GraphUtil.killCFG(replacee);
1521 }
1522
1523 debug.dump(DebugContext.DETAILED_LEVEL, replaceeGraph, "After lowering %s with %s", replacee, this);
1524 return duplicates;
1525 }
1526 }
1527
1528 private UnmodifiableEconomicMap<Node, Node> inlineSnippet(Node replacee, DebugContext debug, StructuredGraph replaceeGraph, EconomicMap<Node, Node> replacements) {
1529 Mark mark = replaceeGraph.getMark();
1530 try (InliningLog.UpdateScope scope = replaceeGraph.getInliningLog().openUpdateScope((oldNode, newNode) -> {
1531 InliningLog log = replaceeGraph.getInliningLog();
1532 if (oldNode == null) {
1533 log.trackNewCallsite(newNode);
1534 }
1535 })) {
1536 UnmodifiableEconomicMap<Node, Node> duplicates = replaceeGraph.addDuplicates(nodes, snippet, snippet.getNodeCount(), replacements);
1537 if (scope != null) {
1538 replaceeGraph.getInliningLog().addLog(duplicates, snippet.getInliningLog());
1539 }
1540 NodeSourcePosition position = replacee.getNodeSourcePosition();
1541 InliningUtil.updateSourcePosition(replaceeGraph, duplicates, mark, position, true);
1542 debug.dump(DebugContext.DETAILED_LEVEL, replaceeGraph, "After inlining snippet %s", snippet.method());
1543 return duplicates;
1544 }
1545 }
1546
1547 private void propagateStamp(Node node) {
1548 if (node instanceof PhiNode) {
1549 PhiNode phi = (PhiNode) node;
1550 if (phi.inferStamp()) {
1551 for (Node usage : node.usages()) {
1552 propagateStamp(usage);
1553 }
1554 }
1555 }
1556 }
1557
1558 private void updateStamps(ValueNode replacee, UnmodifiableEconomicMap<Node, Node> duplicates) {
1559 for (ValueNode node : placeholderStampedNodes) {
1560 ValueNode dup = (ValueNode) duplicates.get(node);
1561 Stamp replaceeStamp = replacee.stamp(NodeView.DEFAULT);
1562 if (node instanceof Placeholder) {
1563 Placeholder placeholderDup = (Placeholder) dup;
1564 placeholderDup.makeReplacement(replaceeStamp);
1565 } else {
1566 dup.setStamp(replaceeStamp);
1567 }
1568 }
1569 for (ParameterNode paramNode : snippet.getNodes(ParameterNode.TYPE)) {
1570 for (Node usage : paramNode.usages()) {
1571 Node usageDup = duplicates.get(usage);
1572 propagateStamp(usageDup);
1573 }
1574 }
1575 }
1576
1577 /**
1578 * Gets a copy of the specialized graph.
1579 */
1580 public StructuredGraph copySpecializedGraph(DebugContext debugForCopy) {
1581 return (StructuredGraph) snippet.copy(debugForCopy);
1582 }
1583
1584 /**
1585 * Replaces a given floating node with this specialized snippet.
1586 *
1587 * @param metaAccess
1588 * @param replacee the node that will be replaced
1589 * @param replacer object that replaces the usages of {@code replacee}
1590 * @param tool lowering tool used to insert the snippet into the control-flow
1591 * @param args the arguments to be bound to the flattened positional parameters of the snippet
1592 */
1593 @SuppressWarnings("try")
1594 public void instantiate(MetaAccessProvider metaAccess, FloatingNode replacee, UsageReplacer replacer, LoweringTool tool, Arguments args) {
1595 DebugContext debug = replacee.getDebug();
1596 assert assertSnippetKills(replacee);
1597 try (DebugCloseable a = args.info.instantiationTimer.start(debug)) {
1598 args.info.instantiationCounter.increment(debug);
1599
1600 // Inline the snippet nodes, replacing parameters with the given args in the process
1601 StartNode entryPointNode = snippet.start();
1602 FixedNode firstCFGNode = entryPointNode.next();
1603 StructuredGraph replaceeGraph = replacee.graph();
1604 EconomicMap<Node, Node> replacements = bind(replaceeGraph, metaAccess, args);
1605 replacements.put(entryPointNode, tool.getCurrentGuardAnchor().asNode());
1606 UnmodifiableEconomicMap<Node, Node> duplicates = inlineSnippet(replacee, debug, replaceeGraph, replacements);
1607
1608 FixedWithNextNode lastFixedNode = tool.lastFixedNode();
1609 assert lastFixedNode != null && lastFixedNode.isAlive() : replaceeGraph + " lastFixed=" + lastFixedNode;
1610 FixedNode next = lastFixedNode.next();
1611 lastFixedNode.setNext(null);
1612 FixedNode firstCFGNodeDuplicate = (FixedNode) duplicates.get(firstCFGNode);
1613 replaceeGraph.addAfterFixed(lastFixedNode, firstCFGNodeDuplicate);
1614
1615 // floating nodes are not state-splits not need to re-wire frame states
1616 assert !(replacee instanceof StateSplit);
1617 updateStamps(replacee, duplicates);
1618
1619 rewireMemoryGraph(replacee, duplicates);
1620
1621 // Replace all usages of the replacee with the value returned by the snippet
1622 ReturnNode returnDuplicate = (ReturnNode) duplicates.get(returnNode);
1623 ValueNode returnValue = returnDuplicate.result();
1624 assert returnValue != null || replacee.hasNoUsages();
1625 replacer.replace(replacee, returnValue);
1626
1627 if (returnDuplicate.isAlive()) {
1628 returnDuplicate.replaceAndDelete(next);
1629 }
1630
1631 debug.dump(DebugContext.DETAILED_LEVEL, replaceeGraph, "After lowering %s with %s", replacee, this);
1632 }
1633 }
1634
1635 /**
1636 * Replaces a given floating node with this specialized snippet.
1637 *
1638 * This snippet must be pure data-flow
1639 *
1640 * @param metaAccess
1641 * @param replacee the node that will be replaced
1642 * @param replacer object that replaces the usages of {@code replacee}
1643 * @param args the arguments to be bound to the flattened positional parameters of the snippet
1644 */
1645 @SuppressWarnings("try")
1646 public void instantiate(MetaAccessProvider metaAccess, FloatingNode replacee, UsageReplacer replacer, Arguments args) {
1647 DebugContext debug = replacee.getDebug();
1648 assert assertSnippetKills(replacee);
1649 try (DebugCloseable a = args.info.instantiationTimer.start(debug)) {
1650 args.info.instantiationCounter.increment(debug);
1651
1652 // Inline the snippet nodes, replacing parameters with the given args in the process
1653 StartNode entryPointNode = snippet.start();
1654 assert entryPointNode.next() == (memoryAnchor == null ? returnNode : memoryAnchor) : entryPointNode.next();
1655 StructuredGraph replaceeGraph = replacee.graph();
1656 EconomicMap<Node, Node> replacements = bind(replaceeGraph, metaAccess, args);
1657 MemoryAnchorNode anchorDuplicate = null;
1658 if (memoryAnchor != null) {
1659 anchorDuplicate = replaceeGraph.add(new MemoryAnchorNode());
1660 replacements.put(memoryAnchor, anchorDuplicate);
1661 }
1662 List<Node> floatingNodes = new ArrayList<>(nodes.size() - 2);
1663 for (Node n : nodes) {
1664 if (n != entryPointNode && n != returnNode) {
1665 floatingNodes.add(n);
1666 }
1667 }
1668 UnmodifiableEconomicMap<Node, Node> duplicates = inlineSnippet(replacee, debug, replaceeGraph, replacements);
1669
1670 // floating nodes are not state-splits not need to re-wire frame states
1671 assert !(replacee instanceof StateSplit);
1672 updateStamps(replacee, duplicates);
1673
1674 rewireMemoryGraph(replacee, duplicates);
1675 assert anchorDuplicate == null || anchorDuplicate.isDeleted();
1676
1677 // Replace all usages of the replacee with the value returned by the snippet
1678 ValueNode returnValue = (ValueNode) duplicates.get(returnNode.result());
1679 replacer.replace(replacee, returnValue);
1680
1681 debug.dump(DebugContext.DETAILED_LEVEL, replaceeGraph, "After lowering %s with %s", replacee, this);
1682 }
1683 }
1684
1685 protected void rewireFrameStates(ValueNode replacee, UnmodifiableEconomicMap<Node, Node> duplicates) {
1686 if (replacee.graph().getGuardsStage().areFrameStatesAtSideEffects() && replacee instanceof StateSplit) {
1687 for (StateSplit sideEffectNode : sideEffectNodes) {
1688 assert ((StateSplit) replacee).hasSideEffect();
1689 Node sideEffectDup = duplicates.get(sideEffectNode.asNode());
1690 ((StateSplit) sideEffectDup).setStateAfter(((StateSplit) replacee).stateAfter());
1691 }
1692 } else if (replacee.graph().getGuardsStage().areFrameStatesAtDeopts() && replacee instanceof DeoptimizingNode) {
1693 DeoptimizingNode replaceeDeopt = (DeoptimizingNode) replacee;
1694
1695 FrameState stateBefore = null;
1696 FrameState stateDuring = null;
1697 FrameState stateAfter = null;
1698 if (replaceeDeopt.canDeoptimize()) {
1699 if (replaceeDeopt instanceof DeoptimizingNode.DeoptBefore) {
1700 stateBefore = ((DeoptimizingNode.DeoptBefore) replaceeDeopt).stateBefore();
1701 }
1702 if (replaceeDeopt instanceof DeoptimizingNode.DeoptDuring) {
1703 stateDuring = ((DeoptimizingNode.DeoptDuring) replaceeDeopt).stateDuring();
1704 }
1705 if (replaceeDeopt instanceof DeoptimizingNode.DeoptAfter) {
1706 stateAfter = ((DeoptimizingNode.DeoptAfter) replaceeDeopt).stateAfter();
1707 }
1708 }
1709
1710 for (DeoptimizingNode deoptNode : deoptNodes) {
1711 DeoptimizingNode deoptDup = (DeoptimizingNode) duplicates.get(deoptNode.asNode());
1712 if (deoptDup.canDeoptimize()) {
1713 if (deoptDup instanceof DeoptimizingNode.DeoptBefore) {
1714 ((DeoptimizingNode.DeoptBefore) deoptDup).setStateBefore(stateBefore);
1715 }
1716 if (deoptDup instanceof DeoptimizingNode.DeoptDuring) {
1717 // compute a state "during" for a DeoptDuring inside the snippet depending
1718 // on what kind of states we had on the node we are replacing.
1719 // If the original node had a state "during" already, we just use that,
1720 // otherwise we need to find a strategy to compute a state during based on
1721 // some other state (before or after).
1722 DeoptimizingNode.DeoptDuring deoptDupDuring = (DeoptimizingNode.DeoptDuring) deoptDup;
1723 if (stateDuring != null) {
1724 deoptDupDuring.setStateDuring(stateDuring);
1725 } else if (stateAfter != null) {
1726 deoptDupDuring.computeStateDuring(stateAfter);
1727 } else if (stateBefore != null) {
1728 assert !deoptDupDuring.hasSideEffect() : "can't use stateBefore as stateDuring for state split " + deoptDupDuring;
1729 deoptDupDuring.setStateDuring(stateBefore);
1730 }
1731 }
1732 if (deoptDup instanceof DeoptimizingNode.DeoptAfter) {
1733 DeoptimizingNode.DeoptAfter deoptDupAfter = (DeoptimizingNode.DeoptAfter) deoptDup;
1734 if (stateAfter != null) {
1735 deoptDupAfter.setStateAfter(stateAfter);
1736 } else {
1737 assert !deoptDupAfter.hasSideEffect() : "can't use stateBefore as stateAfter for state split " + deoptDupAfter;
1738 deoptDupAfter.setStateAfter(stateBefore);
1739 }
1740
1741 }
1742 }
1743 }
1744 }
1745 }
1746
1747 @Override
1748 public String toString() {
1749 StringBuilder buf = new StringBuilder(snippet.toString()).append('(');
1750 String sep = "";
1751 for (int i = 0; i < parameters.length; i++) {
1752 String name = "[" + i + "]";
1753 Object value = parameters[i];
1754 buf.append(sep);
1755 sep = ", ";
1756 if (value == null) {
1757 buf.append("<null> ").append(name);
1758 } else if (value.equals(UNUSED_PARAMETER)) {
1759 buf.append("<unused> ").append(name);
1760 } else if (value.equals(CONSTANT_PARAMETER)) {
1761 buf.append("<constant> ").append(name);
1762 } else if (value instanceof ParameterNode) {
1763 ParameterNode param = (ParameterNode) value;
1764 buf.append(param.getStackKind().getJavaName()).append(' ').append(name);
1765 } else {
1766 ParameterNode[] params = (ParameterNode[]) value;
1767 String kind = params.length == 0 ? "?" : params[0].getStackKind().getJavaName();
1768 buf.append(kind).append('[').append(params.length).append("] ").append(name);
1769 }
1770 }
1771 return buf.append(')').toString();
1772 }
1773
1774 private static boolean checkTemplate(MetaAccessProvider metaAccess, Arguments args, ResolvedJavaMethod method, Signature signature) {
1775 int offset = args.info.hasReceiver() ? 1 : 0;
1776 for (int i = offset; i < args.info.getParameterCount(); i++) {
1777 if (args.info.isConstantParameter(i)) {
1778 JavaKind kind = signature.getParameterKind(i - offset);
1779 assert checkConstantArgument(metaAccess, method, signature, i - offset, args.info.getParameterName(i), args.values[i], kind);
1780
1781 } else if (args.info.isVarargsParameter(i)) {
1782 assert args.values[i] instanceof Varargs;
1783 Varargs varargs = (Varargs) args.values[i];
1784 assert checkVarargs(metaAccess, method, signature, i, args.info.getParameterName(i), varargs);
1785 }
1786 }
1787 return true;
1788 }
1789
1790 public void setMayRemoveLocation(boolean mayRemoveLocation) {
1791 this.mayRemoveLocation = mayRemoveLocation;
1792 }
1793 }