1 /* 2 * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 25 package org.graalvm.compiler.replacements; 26 27 import static java.util.FormattableFlags.ALTERNATE; 28 import static org.graalvm.compiler.debug.DebugContext.DEFAULT_LOG_STREAM; 29 import static org.graalvm.compiler.debug.DebugContext.applyFormattingFlagsAndWidth; 30 import static org.graalvm.compiler.debug.DebugOptions.DebugStubsAndSnippets; 31 import static org.graalvm.compiler.graph.iterators.NodePredicates.isNotA; 32 import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_IGNORED; 33 import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_IGNORED; 34 import static org.graalvm.compiler.phases.common.DeadCodeEliminationPhase.Optionality.Required; 35 import static jdk.internal.vm.compiler.word.LocationIdentity.any; 36 37 import java.lang.reflect.Array; 38 import java.lang.reflect.Method; 39 import java.util.ArrayList; 40 import java.util.Arrays; 41 import java.util.Collection; 42 import java.util.Collections; 43 import java.util.Formattable; 44 import java.util.Formatter; 45 import java.util.LinkedHashMap; 46 import java.util.List; 47 import java.util.Map; 48 import java.util.concurrent.atomic.AtomicInteger; 49 import java.util.concurrent.atomic.AtomicReference; 50 import java.util.function.Predicate; 51 52 import jdk.internal.vm.compiler.collections.EconomicMap; 53 import jdk.internal.vm.compiler.collections.EconomicSet; 54 import jdk.internal.vm.compiler.collections.Equivalence; 55 import jdk.internal.vm.compiler.collections.UnmodifiableEconomicMap; 56 import org.graalvm.compiler.api.replacements.Fold; 57 import org.graalvm.compiler.api.replacements.Snippet; 58 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter; 59 import org.graalvm.compiler.api.replacements.Snippet.NonNullParameter; 60 import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter; 61 import org.graalvm.compiler.api.replacements.SnippetReflectionProvider; 62 import org.graalvm.compiler.core.common.GraalOptions; 63 import org.graalvm.compiler.core.common.type.Stamp; 64 import org.graalvm.compiler.core.common.type.StampFactory; 65 import org.graalvm.compiler.core.common.type.StampPair; 66 import org.graalvm.compiler.core.common.type.TypeReference; 67 import org.graalvm.compiler.debug.CounterKey; 68 import org.graalvm.compiler.debug.DebugCloseable; 69 import org.graalvm.compiler.debug.DebugContext; 70 import org.graalvm.compiler.debug.DebugContext.Description; 71 import org.graalvm.compiler.debug.DebugHandlersFactory; 72 import org.graalvm.compiler.debug.GraalError; 73 import org.graalvm.compiler.debug.TimerKey; 74 import org.graalvm.compiler.graph.Graph.Mark; 75 import org.graalvm.compiler.graph.Node; 76 import org.graalvm.compiler.graph.Node.NodeIntrinsic; 77 import org.graalvm.compiler.graph.NodeClass; 78 import org.graalvm.compiler.graph.NodeSourcePosition; 79 import org.graalvm.compiler.graph.Position; 80 import org.graalvm.compiler.loop.LoopEx; 81 import org.graalvm.compiler.loop.LoopsData; 82 import org.graalvm.compiler.loop.phases.LoopTransformations; 83 import org.graalvm.compiler.nodeinfo.InputType; 84 import org.graalvm.compiler.nodeinfo.NodeInfo; 85 import org.graalvm.compiler.nodes.AbstractBeginNode; 86 import org.graalvm.compiler.nodes.AbstractMergeNode; 87 import org.graalvm.compiler.nodes.ConstantNode; 88 import org.graalvm.compiler.nodes.ControlSinkNode; 89 import org.graalvm.compiler.nodes.DeoptimizingNode; 90 import org.graalvm.compiler.nodes.FixedNode; 91 import org.graalvm.compiler.nodes.FixedWithNextNode; 92 import org.graalvm.compiler.nodes.FrameState; 93 import org.graalvm.compiler.nodes.InliningLog; 94 import org.graalvm.compiler.nodes.LoopBeginNode; 95 import org.graalvm.compiler.nodes.MergeNode; 96 import org.graalvm.compiler.nodes.NodeView; 97 import org.graalvm.compiler.nodes.ParameterNode; 98 import org.graalvm.compiler.nodes.PhiNode; 99 import org.graalvm.compiler.nodes.PiNode.Placeholder; 100 import org.graalvm.compiler.nodes.PiNode.PlaceholderStamp; 101 import org.graalvm.compiler.nodes.ReturnNode; 102 import org.graalvm.compiler.nodes.StartNode; 103 import org.graalvm.compiler.nodes.StateSplit; 104 import org.graalvm.compiler.nodes.StructuredGraph; 105 import org.graalvm.compiler.nodes.StructuredGraph.GuardsStage; 106 import org.graalvm.compiler.nodes.ValueNode; 107 import org.graalvm.compiler.nodes.ValueNodeUtil; 108 import org.graalvm.compiler.nodes.calc.FloatingNode; 109 import org.graalvm.compiler.nodes.java.LoadIndexedNode; 110 import org.graalvm.compiler.nodes.java.MethodCallTargetNode; 111 import org.graalvm.compiler.nodes.java.StoreIndexedNode; 112 import org.graalvm.compiler.nodes.memory.MemoryAccess; 113 import org.graalvm.compiler.nodes.memory.MemoryAnchorNode; 114 import org.graalvm.compiler.nodes.memory.MemoryCheckpoint; 115 import org.graalvm.compiler.nodes.memory.MemoryMap; 116 import org.graalvm.compiler.nodes.memory.MemoryMapNode; 117 import org.graalvm.compiler.nodes.memory.MemoryNode; 118 import org.graalvm.compiler.nodes.memory.MemoryPhiNode; 119 import org.graalvm.compiler.nodes.spi.ArrayLengthProvider; 120 import org.graalvm.compiler.nodes.spi.LoweringTool; 121 import org.graalvm.compiler.nodes.spi.MemoryProxy; 122 import org.graalvm.compiler.nodes.util.GraphUtil; 123 import org.graalvm.compiler.options.Option; 124 import org.graalvm.compiler.options.OptionKey; 125 import org.graalvm.compiler.options.OptionValues; 126 import org.graalvm.compiler.phases.common.CanonicalizerPhase; 127 import org.graalvm.compiler.phases.common.DeadCodeEliminationPhase; 128 import org.graalvm.compiler.phases.common.FloatingReadPhase; 129 import org.graalvm.compiler.phases.common.FloatingReadPhase.MemoryMapImpl; 130 import org.graalvm.compiler.phases.common.GuardLoweringPhase; 131 import org.graalvm.compiler.phases.common.LoweringPhase; 132 import org.graalvm.compiler.phases.common.RemoveValueProxyPhase; 133 import org.graalvm.compiler.phases.common.inlining.InliningUtil; 134 import org.graalvm.compiler.phases.tiers.PhaseContext; 135 import org.graalvm.compiler.phases.util.Providers; 136 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode; 137 import org.graalvm.compiler.replacements.nodes.LoadSnippetVarargParameterNode; 138 import org.graalvm.util.CollectionsUtil; 139 import jdk.internal.vm.compiler.word.LocationIdentity; 140 import jdk.internal.vm.compiler.word.WordBase; 141 142 import jdk.vm.ci.code.TargetDescription; 143 import jdk.vm.ci.meta.Constant; 144 import jdk.vm.ci.meta.ConstantReflectionProvider; 145 import jdk.vm.ci.meta.JavaConstant; 146 import jdk.vm.ci.meta.JavaKind; 147 import jdk.vm.ci.meta.Local; 148 import jdk.vm.ci.meta.LocalVariableTable; 149 import jdk.vm.ci.meta.MetaAccessProvider; 150 import jdk.vm.ci.meta.ResolvedJavaMethod; 151 import jdk.vm.ci.meta.ResolvedJavaMethod.Parameter; 152 import jdk.vm.ci.meta.ResolvedJavaType; 153 import jdk.vm.ci.meta.Signature; 154 155 /** 156 * A snippet template is a graph created by parsing a snippet method and then specialized by binding 157 * constants to the snippet's {@link ConstantParameter} parameters. 158 * 159 * Snippet templates can be managed in a cache maintained by {@link AbstractTemplates}. 160 */ 161 public class SnippetTemplate { 162 163 private boolean mayRemoveLocation = false; 164 165 /** 166 * Holds the {@link ResolvedJavaMethod} of the snippet, together with some information about the 167 * method that needs to be computed only once. The {@link SnippetInfo} should be created once 168 * per snippet and then cached. 169 */ 170 public abstract static class SnippetInfo { 171 172 protected final ResolvedJavaMethod method; 173 protected ResolvedJavaMethod original; 174 protected final LocationIdentity[] privateLocations; 175 protected final Object receiver; 176 177 public Object getReceiver() { 178 return receiver; 179 } 180 181 boolean hasReceiver() { 182 assert hasReceiver(method) == (receiver != null) : "Snippet with the receiver must have it set as constant. Snippet: " + this; 183 return hasReceiver(method); 184 } 185 186 static boolean hasReceiver(ResolvedJavaMethod method) { 187 return method.hasReceiver(); 188 } 189 190 /** 191 * Lazily constructed parts of {@link SnippetInfo}. 192 */ 193 static class Lazy { 194 Lazy(ResolvedJavaMethod method) { 195 int count = method.getSignature().getParameterCount(hasReceiver(method)); 196 constantParameters = new boolean[count]; 197 varargsParameters = new boolean[count]; 198 nonNullParameters = new boolean[count]; 199 int offset = hasReceiver(method) ? 1 : 0; 200 for (int i = offset; i < count; i++) { 201 constantParameters[i] = method.getParameterAnnotation(ConstantParameter.class, i - offset) != null; 202 varargsParameters[i] = method.getParameterAnnotation(VarargsParameter.class, i - offset) != null; 203 nonNullParameters[i] = method.getParameterAnnotation(NonNullParameter.class, i - offset) != null; 204 205 assert !constantParameters[i - offset] || !varargsParameters[i - offset] : "Parameter cannot be annotated with both @" + ConstantParameter.class.getSimpleName() + " and @" + 206 VarargsParameter.class.getSimpleName(); 207 } 208 if (method.hasReceiver()) { 209 // Receiver must be constant. 210 assert !constantParameters[0]; 211 constantParameters[0] = true; 212 } 213 214 // Retrieve the names only when assertions are turned on. 215 assert initNames(method, count); 216 } 217 218 final boolean[] constantParameters; 219 final boolean[] varargsParameters; 220 final boolean[] nonNullParameters; 221 222 /** 223 * The parameter names, taken from the local variables table. Only used for assertion 224 * checking, so use only within an assert statement. 225 */ 226 String[] names; 227 228 private boolean initNames(ResolvedJavaMethod method, int parameterCount) { 229 names = new String[parameterCount]; 230 int offset = 0; 231 if (method.hasReceiver()) { 232 names[0] = "this"; 233 offset = 1; 234 } 235 Parameter[] params = method.getParameters(); 236 if (params != null) { 237 for (int i = offset; i < names.length; i++) { 238 if (params[i - offset].isNamePresent()) { 239 names[i] = params[i - offset].getName(); 240 } 241 } 242 } else { 243 int slotIdx = 0; 244 LocalVariableTable localVariableTable = method.getLocalVariableTable(); 245 if (localVariableTable != null) { 246 for (int i = 0; i < names.length; i++) { 247 Local local = localVariableTable.getLocal(slotIdx, 0); 248 if (local != null) { 249 names[i] = local.getName(); 250 } 251 JavaKind kind = method.getSignature().getParameterKind(i); 252 slotIdx += kind.getSlotCount(); 253 } 254 } 255 } 256 return true; 257 } 258 } 259 260 /** 261 * Times instantiations of all templates derived from this snippet. 262 */ 263 private final TimerKey instantiationTimer; 264 265 /** 266 * Counts instantiations of all templates derived from this snippet. 267 */ 268 private final CounterKey instantiationCounter; 269 270 protected abstract Lazy lazy(); 271 272 protected SnippetInfo(ResolvedJavaMethod method, LocationIdentity[] privateLocations, Object receiver) { 273 this.method = method; 274 this.privateLocations = privateLocations; 275 instantiationCounter = DebugContext.counter("SnippetInstantiationCount[%s]", method.getName()); 276 instantiationTimer = DebugContext.timer("SnippetInstantiationTime[%s]", method.getName()); 277 this.receiver = receiver; 278 } 279 280 public ResolvedJavaMethod getMethod() { 281 return method; 282 } 283 284 public int getParameterCount() { 285 return lazy().constantParameters.length; 286 } 287 288 public void setOriginalMethod(ResolvedJavaMethod original) { 289 this.original = original; 290 } 291 292 public boolean isConstantParameter(int paramIdx) { 293 return lazy().constantParameters[paramIdx]; 294 } 295 296 public boolean isVarargsParameter(int paramIdx) { 297 return lazy().varargsParameters[paramIdx]; 298 } 299 300 public boolean isNonNullParameter(int paramIdx) { 301 return lazy().nonNullParameters[paramIdx]; 302 } 303 304 public String getParameterName(int paramIdx) { 305 String[] names = lazy().names; 306 if (names != null) { 307 return names[paramIdx]; 308 } 309 return null; 310 } 311 312 @Override 313 public String toString() { 314 return getClass().getSimpleName() + ":" + method.format("%h.%n"); 315 } 316 } 317 318 protected static class LazySnippetInfo extends SnippetInfo { 319 protected final AtomicReference<Lazy> lazy = new AtomicReference<>(null); 320 321 protected LazySnippetInfo(ResolvedJavaMethod method, LocationIdentity[] privateLocations, Object receiver) { 322 super(method, privateLocations, receiver); 323 } 324 325 @Override 326 protected Lazy lazy() { 327 if (lazy.get() == null) { 328 lazy.compareAndSet(null, new Lazy(method)); 329 } 330 return lazy.get(); 331 } 332 } 333 334 protected static class EagerSnippetInfo extends SnippetInfo { 335 protected final Lazy lazy; 336 337 protected EagerSnippetInfo(ResolvedJavaMethod method, LocationIdentity[] privateLocations, Object receiver) { 338 super(method, privateLocations, receiver); 339 lazy = new Lazy(method); 340 } 341 342 @Override 343 protected Lazy lazy() { 344 return lazy; 345 } 346 } 347 348 /** 349 * Values that are bound to the snippet method parameters. The methods {@link #add}, 350 * {@link #addConst}, and {@link #addVarargs} must be called in the same order as in the 351 * signature of the snippet method. The parameter name is passed to the add methods for 352 * assertion checking, i.e., to enforce that the order matches. Which method needs to be called 353 * depends on the annotation of the snippet method parameter: 354 * <ul> 355 * <li>Use {@link #add} for a parameter without an annotation. The value is bound when the 356 * {@link SnippetTemplate} is {@link SnippetTemplate#instantiate instantiated}. 357 * <li>Use {@link #addConst} for a parameter annotated with {@link ConstantParameter}. The value 358 * is bound when the {@link SnippetTemplate} is {@link SnippetTemplate#SnippetTemplate created}. 359 * <li>Use {@link #addVarargs} for an array parameter annotated with {@link VarargsParameter}. A 360 * separate {@link SnippetTemplate} is {@link SnippetTemplate#SnippetTemplate created} for every 361 * distinct array length. The actual values are bound when the {@link SnippetTemplate} is 362 * {@link SnippetTemplate#instantiate instantiated} 363 * </ul> 364 */ 365 public static class Arguments implements Formattable { 366 367 protected final SnippetInfo info; 368 protected final CacheKey cacheKey; 369 protected final Object[] values; 370 protected final Stamp[] constStamps; 371 protected boolean cacheable; 372 373 protected int nextParamIdx; 374 375 public Arguments(SnippetInfo info, GuardsStage guardsStage, LoweringTool.LoweringStage loweringStage) { 376 this.info = info; 377 this.cacheKey = new CacheKey(info, guardsStage, loweringStage); 378 this.values = new Object[info.getParameterCount()]; 379 this.constStamps = new Stamp[info.getParameterCount()]; 380 this.cacheable = true; 381 if (info.hasReceiver()) { 382 addConst("this", info.getReceiver()); 383 } 384 } 385 386 public Arguments add(String name, Object value) { 387 assert check(name, false, false); 388 values[nextParamIdx] = value; 389 nextParamIdx++; 390 return this; 391 } 392 393 public Arguments addConst(String name, Object value) { 394 assert value != null; 395 return addConst(name, value, null); 396 } 397 398 public Arguments addConst(String name, Object value, Stamp stamp) { 399 assert check(name, true, false); 400 values[nextParamIdx] = value; 401 constStamps[nextParamIdx] = stamp; 402 cacheKey.setParam(nextParamIdx, value); 403 nextParamIdx++; 404 return this; 405 } 406 407 public Arguments addVarargs(String name, Class<?> componentType, Stamp argStamp, Object value) { 408 assert check(name, false, true); 409 Varargs varargs = new Varargs(componentType, argStamp, value); 410 values[nextParamIdx] = varargs; 411 // A separate template is necessary for every distinct array length 412 cacheKey.setParam(nextParamIdx, varargs.length); 413 nextParamIdx++; 414 return this; 415 } 416 417 public void setCacheable(boolean cacheable) { 418 this.cacheable = cacheable; 419 } 420 421 private boolean check(String name, boolean constParam, boolean varargsParam) { 422 assert nextParamIdx < info.getParameterCount() : "too many parameters: " + name + " " + this; 423 assert info.getParameterName(nextParamIdx) == null || info.getParameterName(nextParamIdx).equals(name) : "wrong parameter name at " + nextParamIdx + " : " + name + " " + this; 424 assert constParam == info.isConstantParameter(nextParamIdx) : "Parameter " + (constParam ? "not " : "") + "annotated with @" + ConstantParameter.class.getSimpleName() + ": " + name + 425 " " + this; 426 assert varargsParam == info.isVarargsParameter(nextParamIdx) : "Parameter " + (varargsParam ? "not " : "") + "annotated with @" + VarargsParameter.class.getSimpleName() + ": " + name + 427 " " + this; 428 return true; 429 } 430 431 @Override 432 public String toString() { 433 StringBuilder result = new StringBuilder(); 434 result.append("Parameters<").append(info.method.format("%h.%n")).append(" ["); 435 String sep = ""; 436 for (int i = 0; i < info.getParameterCount(); i++) { 437 result.append(sep); 438 if (info.isConstantParameter(i)) { 439 result.append("const "); 440 } else if (info.isVarargsParameter(i)) { 441 result.append("varargs "); 442 } 443 result.append(info.getParameterName(i)).append(" = ").append(values[i]); 444 sep = ", "; 445 } 446 result.append(">"); 447 return result.toString(); 448 } 449 450 @Override 451 public void formatTo(Formatter formatter, int flags, int width, int precision) { 452 if ((flags & ALTERNATE) == 0) { 453 formatter.format(applyFormattingFlagsAndWidth(toString(), flags, width)); 454 } else { 455 StringBuilder sb = new StringBuilder(); 456 sb.append(info.method.getName()).append('('); 457 String sep = ""; 458 for (int i = 0; i < info.getParameterCount(); i++) { 459 if (info.isConstantParameter(i)) { 460 sb.append(sep); 461 if (info.getParameterName(i) != null) { 462 sb.append(info.getParameterName(i)); 463 } else { 464 sb.append(i); 465 } 466 sb.append('=').append(values[i]); 467 sep = ", "; 468 } 469 } 470 sb.append(")"); 471 String string = sb.toString(); 472 if (string.indexOf('%') != -1) { 473 // Quote any % signs 474 string = string.replace("%", "%%"); 475 } 476 formatter.format(applyFormattingFlagsAndWidth(string, flags & ~ALTERNATE, width)); 477 } 478 } 479 } 480 481 /** 482 * Wrapper for the prototype value of a {@linkplain VarargsParameter varargs} parameter. 483 */ 484 static class Varargs { 485 486 protected final Class<?> componentType; 487 protected final Stamp stamp; 488 protected final Object value; 489 protected final int length; 490 491 protected Varargs(Class<?> componentType, Stamp stamp, Object value) { 492 this.componentType = componentType; 493 this.stamp = stamp; 494 this.value = value; 495 if (value instanceof List) { 496 this.length = ((List<?>) value).size(); 497 } else { 498 this.length = Array.getLength(value); 499 } 500 } 501 502 @Override 503 public String toString() { 504 if (value instanceof boolean[]) { 505 return Arrays.toString((boolean[]) value); 506 } 507 if (value instanceof byte[]) { 508 return Arrays.toString((byte[]) value); 509 } 510 if (value instanceof char[]) { 511 return Arrays.toString((char[]) value); 512 } 513 if (value instanceof short[]) { 514 return Arrays.toString((short[]) value); 515 } 516 if (value instanceof int[]) { 517 return Arrays.toString((int[]) value); 518 } 519 if (value instanceof long[]) { 520 return Arrays.toString((long[]) value); 521 } 522 if (value instanceof float[]) { 523 return Arrays.toString((float[]) value); 524 } 525 if (value instanceof double[]) { 526 return Arrays.toString((double[]) value); 527 } 528 if (value instanceof Object[]) { 529 return Arrays.toString((Object[]) value); 530 } 531 return String.valueOf(value); 532 } 533 } 534 535 @NodeInfo(cycles = CYCLES_IGNORED, size = SIZE_IGNORED) 536 static final class VarargsPlaceholderNode extends FloatingNode implements ArrayLengthProvider { 537 538 public static final NodeClass<VarargsPlaceholderNode> TYPE = NodeClass.create(VarargsPlaceholderNode.class); 539 protected final Varargs varargs; 540 541 protected VarargsPlaceholderNode(Varargs varargs, MetaAccessProvider metaAccess) { 542 super(TYPE, StampFactory.objectNonNull(TypeReference.createExactTrusted(metaAccess.lookupJavaType(varargs.componentType).getArrayClass()))); 543 this.varargs = varargs; 544 } 545 546 @Override 547 public ValueNode findLength(FindLengthMode mode, ConstantReflectionProvider constantReflection) { 548 return ConstantNode.forInt(varargs.length); 549 } 550 } 551 552 static class CacheKey { 553 554 private final ResolvedJavaMethod method; 555 private final Object[] values; 556 private final GuardsStage guardsStage; 557 private final LoweringTool.LoweringStage loweringStage; 558 private int hash; 559 560 protected CacheKey(SnippetInfo info, GuardsStage guardsStage, LoweringTool.LoweringStage loweringStage) { 561 this.method = info.method; 562 this.guardsStage = guardsStage; 563 this.loweringStage = loweringStage; 564 this.values = new Object[info.getParameterCount()]; 565 this.hash = info.method.hashCode() + 31 * guardsStage.ordinal(); 566 } 567 568 protected void setParam(int paramIdx, Object value) { 569 values[paramIdx] = value; 570 hash = (hash * 31) ^ (value == null ? 0 : value.hashCode()); 571 } 572 573 @Override 574 public boolean equals(Object obj) { 575 if (!(obj instanceof CacheKey)) { 576 return false; 577 } 578 CacheKey other = (CacheKey) obj; 579 if (!method.equals(other.method)) { 580 return false; 581 } 582 if (guardsStage != other.guardsStage || loweringStage != other.loweringStage) { 583 return false; 584 } 585 for (int i = 0; i < values.length; i++) { 586 if (values[i] != null && !values[i].equals(other.values[i])) { 587 return false; 588 } 589 } 590 return true; 591 } 592 593 @Override 594 public int hashCode() { 595 return hash; 596 } 597 } 598 599 private static final TimerKey SnippetTemplateCreationTime = DebugContext.timer("SnippetTemplateCreationTime"); 600 private static final CounterKey SnippetTemplates = DebugContext.counter("SnippetTemplateCount"); 601 602 static class Options { 603 @Option(help = "Use a LRU cache for snippet templates.")// 604 public static final OptionKey<Boolean> UseSnippetTemplateCache = new OptionKey<>(true); 605 606 @Option(help = "")// 607 static final OptionKey<Integer> MaxTemplatesPerSnippet = new OptionKey<>(50); 608 } 609 610 /** 611 * Base class for snippet classes. It provides a cache for {@link SnippetTemplate}s. 612 */ 613 public abstract static class AbstractTemplates implements org.graalvm.compiler.api.replacements.SnippetTemplateCache { 614 615 protected final OptionValues options; 616 protected final Providers providers; 617 protected final SnippetReflectionProvider snippetReflection; 618 protected final Iterable<DebugHandlersFactory> factories; 619 protected final TargetDescription target; 620 private final Map<CacheKey, SnippetTemplate> templates; 621 622 protected AbstractTemplates(OptionValues options, Iterable<DebugHandlersFactory> factories, Providers providers, SnippetReflectionProvider snippetReflection, TargetDescription target) { 623 this.options = options; 624 this.providers = providers; 625 this.snippetReflection = snippetReflection; 626 this.target = target; 627 this.factories = factories; 628 if (Options.UseSnippetTemplateCache.getValue(options)) { 629 int size = Options.MaxTemplatesPerSnippet.getValue(options); 630 this.templates = Collections.synchronizedMap(new LRUCache<>(size, size)); 631 } else { 632 this.templates = null; 633 } 634 } 635 636 public static Method findMethod(Class<? extends Snippets> declaringClass, String methodName, Method except) { 637 for (Method m : declaringClass.getDeclaredMethods()) { 638 if (m.getName().equals(methodName) && !m.equals(except)) { 639 return m; 640 } 641 } 642 return null; 643 } 644 645 protected SnippetInfo snippet(Class<? extends Snippets> declaringClass, String methodName, LocationIdentity... initialPrivateLocations) { 646 return snippet(declaringClass, methodName, null, initialPrivateLocations); 647 } 648 649 /** 650 * Finds the unique method in {@code declaringClass} named {@code methodName} annotated by 651 * {@link Snippet} and returns a {@link SnippetInfo} value describing it. There must be 652 * exactly one snippet method in {@code declaringClass}. 653 */ 654 protected SnippetInfo snippet(Class<? extends Snippets> declaringClass, String methodName, Object receiver, LocationIdentity... initialPrivateLocations) { 655 assert methodName != null; 656 Method method = findMethod(declaringClass, methodName, null); 657 assert method != null : "did not find @" + Snippet.class.getSimpleName() + " method in " + declaringClass + " named " + methodName; 658 assert method.getAnnotation(Snippet.class) != null : method + " must be annotated with @" + Snippet.class.getSimpleName(); 659 assert findMethod(declaringClass, methodName, method) == null : "found more than one method named " + methodName + " in " + declaringClass; 660 ResolvedJavaMethod javaMethod = providers.getMetaAccess().lookupJavaMethod(method); 661 providers.getReplacements().registerSnippet(javaMethod, GraalOptions.TrackNodeSourcePosition.getValue(options)); 662 LocationIdentity[] privateLocations = GraalOptions.SnippetCounters.getValue(options) ? SnippetCounterNode.addSnippetCounters(initialPrivateLocations) : initialPrivateLocations; 663 if (GraalOptions.EagerSnippets.getValue(options)) { 664 return new EagerSnippetInfo(javaMethod, privateLocations, receiver); 665 } else { 666 return new LazySnippetInfo(javaMethod, privateLocations, receiver); 667 } 668 } 669 670 static final AtomicInteger nextSnippetTemplateId = new AtomicInteger(); 671 672 private DebugContext openDebugContext(DebugContext outer, Arguments args) { 673 if (DebugStubsAndSnippets.getValue(options)) { 674 Description description = new Description(args.cacheKey.method, "SnippetTemplate_" + nextSnippetTemplateId.incrementAndGet()); 675 return DebugContext.create(options, description, outer.getGlobalMetrics(), DEFAULT_LOG_STREAM, factories); 676 } 677 return DebugContext.DISABLED; 678 } 679 680 /** 681 * Gets a template for a given key, creating it first if necessary. 682 */ 683 @SuppressWarnings("try") 684 protected SnippetTemplate template(ValueNode replacee, final Arguments args) { 685 StructuredGraph graph = replacee.graph(); 686 DebugContext outer = graph.getDebug(); 687 SnippetTemplate template = Options.UseSnippetTemplateCache.getValue(options) && args.cacheable ? templates.get(args.cacheKey) : null; 688 if (template == null || (graph.trackNodeSourcePosition() && !template.snippet.trackNodeSourcePosition())) { 689 try (DebugContext debug = openDebugContext(outer, args)) { 690 try (DebugCloseable a = SnippetTemplateCreationTime.start(debug); DebugContext.Scope s = debug.scope("SnippetSpecialization", args.info.method)) { 691 SnippetTemplates.increment(debug); 692 OptionValues snippetOptions = new OptionValues(options, GraalOptions.TraceInlining, GraalOptions.TraceInliningForStubsAndSnippets.getValue(options)); 693 template = new SnippetTemplate(snippetOptions, debug, providers, snippetReflection, args, graph.trackNodeSourcePosition(), replacee); 694 if (Options.UseSnippetTemplateCache.getValue(snippetOptions) && args.cacheable) { 695 templates.put(args.cacheKey, template); 696 } 697 } catch (Throwable e) { 698 throw debug.handle(e); 699 } 700 } 701 } 702 return template; 703 } 704 } 705 706 private static final class LRUCache<K, V> extends LinkedHashMap<K, V> { 707 private static final long serialVersionUID = 1L; 708 private final int maxCacheSize; 709 710 LRUCache(int initialCapacity, int maxCacheSize) { 711 super(initialCapacity, 0.75F, true); 712 this.maxCacheSize = maxCacheSize; 713 } 714 715 @Override 716 protected boolean removeEldestEntry(java.util.Map.Entry<K, V> eldest) { 717 return size() > maxCacheSize; 718 } 719 } 720 721 // These values must be compared with equals() not '==' to support replay compilation. 722 private static final Object UNUSED_PARAMETER = "UNUSED_PARAMETER"; 723 private static final Object CONSTANT_PARAMETER = "CONSTANT_PARAMETER"; 724 725 /** 726 * Determines if any parameter of a given method is annotated with {@link ConstantParameter}. 727 */ 728 public static boolean hasConstantParameter(ResolvedJavaMethod method) { 729 for (ConstantParameter p : method.getParameterAnnotations(ConstantParameter.class)) { 730 if (p != null) { 731 return true; 732 } 733 } 734 return false; 735 } 736 737 private final SnippetReflectionProvider snippetReflection; 738 739 /** 740 * Creates a snippet template. 741 */ 742 @SuppressWarnings("try") 743 protected SnippetTemplate(OptionValues options, DebugContext debug, final Providers providers, SnippetReflectionProvider snippetReflection, Arguments args, boolean trackNodeSourcePosition, 744 Node replacee) { 745 this.snippetReflection = snippetReflection; 746 this.info = args.info; 747 748 Object[] constantArgs = getConstantArgs(args); 749 boolean shouldTrackNodeSourcePosition1 = trackNodeSourcePosition || (providers.getCodeCache() != null && providers.getCodeCache().shouldDebugNonSafepoints()); 750 StructuredGraph snippetGraph = providers.getReplacements().getSnippet(args.info.method, args.info.original, constantArgs, shouldTrackNodeSourcePosition1, replacee.getNodeSourcePosition()); 751 752 ResolvedJavaMethod method = snippetGraph.method(); 753 Signature signature = method.getSignature(); 754 755 PhaseContext phaseContext = new PhaseContext(providers); 756 757 // Copy snippet graph, replacing constant parameters with given arguments 758 final StructuredGraph snippetCopy = new StructuredGraph.Builder(options, debug).name(snippetGraph.name).method(snippetGraph.method()).trackNodeSourcePosition( 759 snippetGraph.trackNodeSourcePosition()).setIsSubstitution(true).build(); 760 assert !GraalOptions.TrackNodeSourcePosition.getValue(options) || snippetCopy.trackNodeSourcePosition(); 761 try (DebugContext.Scope scope = debug.scope("SpecializeSnippet", snippetCopy)) { 762 if (!snippetGraph.isUnsafeAccessTrackingEnabled()) { 763 snippetCopy.disableUnsafeAccessTracking(); 764 } 765 766 EconomicMap<Node, Node> nodeReplacements = EconomicMap.create(Equivalence.IDENTITY); 767 nodeReplacements.put(snippetGraph.start(), snippetCopy.start()); 768 769 MetaAccessProvider metaAccess = providers.getMetaAccess(); 770 assert checkTemplate(metaAccess, args, method, signature); 771 772 int parameterCount = args.info.getParameterCount(); 773 VarargsPlaceholderNode[] placeholders = new VarargsPlaceholderNode[parameterCount]; 774 775 for (int i = 0; i < parameterCount; i++) { 776 ParameterNode parameter = snippetGraph.getParameter(i); 777 if (parameter != null) { 778 if (args.info.isConstantParameter(i)) { 779 Object arg = args.values[i]; 780 JavaKind kind = signature.getParameterKind(i); 781 ConstantNode constantNode; 782 if (arg instanceof Constant) { 783 Stamp stamp = args.constStamps[i]; 784 if (stamp == null) { 785 assert arg instanceof JavaConstant : "could not determine type of constant " + arg; 786 constantNode = ConstantNode.forConstant((JavaConstant) arg, metaAccess, snippetCopy); 787 } else { 788 constantNode = ConstantNode.forConstant(stamp, (Constant) arg, metaAccess, snippetCopy); 789 } 790 } else { 791 constantNode = ConstantNode.forConstant(snippetReflection.forBoxed(kind, arg), metaAccess, snippetCopy); 792 } 793 nodeReplacements.put(parameter, constantNode); 794 } else if (args.info.isVarargsParameter(i)) { 795 Varargs varargs = (Varargs) args.values[i]; 796 VarargsPlaceholderNode placeholder = snippetCopy.unique(new VarargsPlaceholderNode(varargs, providers.getMetaAccess())); 797 nodeReplacements.put(parameter, placeholder); 798 placeholders[i] = placeholder; 799 } else if (args.info.isNonNullParameter(i)) { 800 parameter.setStamp(parameter.stamp(NodeView.DEFAULT).join(StampFactory.objectNonNull())); 801 } 802 } 803 } 804 try (InliningLog.UpdateScope updateScope = snippetCopy.getInliningLog().openDefaultUpdateScope()) { 805 UnmodifiableEconomicMap<Node, Node> duplicates = snippetCopy.addDuplicates(snippetGraph.getNodes(), snippetGraph, snippetGraph.getNodeCount(), nodeReplacements); 806 if (updateScope != null) { 807 snippetCopy.getInliningLog().replaceLog(duplicates, snippetGraph.getInliningLog()); 808 } 809 } 810 811 debug.dump(DebugContext.INFO_LEVEL, snippetCopy, "Before specialization"); 812 813 // Gather the template parameters 814 parameters = new Object[parameterCount]; 815 for (int i = 0; i < parameterCount; i++) { 816 if (args.info.isConstantParameter(i)) { 817 parameters[i] = CONSTANT_PARAMETER; 818 } else if (args.info.isVarargsParameter(i)) { 819 assert snippetCopy.getParameter(i) == null; 820 Varargs varargs = (Varargs) args.values[i]; 821 int length = varargs.length; 822 ParameterNode[] params = new ParameterNode[length]; 823 Stamp stamp = varargs.stamp; 824 for (int j = 0; j < length; j++) { 825 // Use a decimal friendly numbering make it more obvious how values map 826 assert parameterCount < 10000; 827 int idx = (i + 1) * 10000 + j; 828 assert idx >= parameterCount : "collision in parameter numbering"; 829 ParameterNode local = snippetCopy.addOrUnique(new ParameterNode(idx, StampPair.createSingle(stamp))); 830 params[j] = local; 831 } 832 parameters[i] = params; 833 834 VarargsPlaceholderNode placeholder = placeholders[i]; 835 if (placeholder != null) { 836 for (Node usage : placeholder.usages().snapshot()) { 837 if (usage instanceof LoadIndexedNode) { 838 LoadIndexedNode loadIndexed = (LoadIndexedNode) usage; 839 debug.dump(DebugContext.INFO_LEVEL, snippetCopy, "Before replacing %s", loadIndexed); 840 LoadSnippetVarargParameterNode loadSnippetParameter = snippetCopy.add( 841 new LoadSnippetVarargParameterNode(params, loadIndexed.index(), loadIndexed.stamp(NodeView.DEFAULT))); 842 snippetCopy.replaceFixedWithFixed(loadIndexed, loadSnippetParameter); 843 debug.dump(DebugContext.INFO_LEVEL, snippetCopy, "After replacing %s", loadIndexed); 844 } else if (usage instanceof StoreIndexedNode) { 845 /* 846 * The template lowering doesn't really treat this as an array so 847 * you can't store back into the varargs. Allocate your own array if 848 * you really need this and EA should eliminate it. 849 */ 850 throw new GraalError("Can't store into VarargsParameter array"); 851 } 852 } 853 } 854 } else { 855 ParameterNode local = snippetCopy.getParameter(i); 856 if (local == null) { 857 // Parameter value was eliminated 858 parameters[i] = UNUSED_PARAMETER; 859 } else { 860 parameters[i] = local; 861 } 862 } 863 } 864 865 explodeLoops(snippetCopy, phaseContext); 866 867 GuardsStage guardsStage = args.cacheKey.guardsStage; 868 // Perform lowering on the snippet 869 if (!guardsStage.allowsFloatingGuards()) { 870 new GuardLoweringPhase().apply(snippetCopy, null); 871 } 872 snippetCopy.setGuardsStage(guardsStage); 873 try (DebugContext.Scope s = debug.scope("LoweringSnippetTemplate", snippetCopy)) { 874 new LoweringPhase(new CanonicalizerPhase(), args.cacheKey.loweringStage).apply(snippetCopy, phaseContext); 875 } catch (Throwable e) { 876 throw debug.handle(e); 877 } 878 879 ArrayList<StateSplit> curSideEffectNodes = new ArrayList<>(); 880 ArrayList<DeoptimizingNode> curDeoptNodes = new ArrayList<>(); 881 ArrayList<ValueNode> curPlaceholderStampedNodes = new ArrayList<>(); 882 for (Node node : snippetCopy.getNodes()) { 883 if (node instanceof ValueNode) { 884 ValueNode valueNode = (ValueNode) node; 885 if (valueNode.stamp(NodeView.DEFAULT) == PlaceholderStamp.singleton()) { 886 curPlaceholderStampedNodes.add(valueNode); 887 } 888 } 889 890 if (node instanceof StateSplit) { 891 StateSplit stateSplit = (StateSplit) node; 892 FrameState frameState = stateSplit.stateAfter(); 893 if (stateSplit.hasSideEffect()) { 894 curSideEffectNodes.add((StateSplit) node); 895 } 896 if (frameState != null) { 897 stateSplit.setStateAfter(null); 898 } 899 } 900 if (node instanceof DeoptimizingNode) { 901 DeoptimizingNode deoptNode = (DeoptimizingNode) node; 902 if (deoptNode.canDeoptimize()) { 903 curDeoptNodes.add(deoptNode); 904 } 905 } 906 } 907 908 new DeadCodeEliminationPhase(Required).apply(snippetCopy); 909 910 assert checkAllVarargPlaceholdersAreDeleted(parameterCount, placeholders); 911 912 new FloatingReadPhase(true, true).apply(snippetCopy); 913 new RemoveValueProxyPhase().apply(snippetCopy); 914 915 MemoryAnchorNode anchor = snippetCopy.add(new MemoryAnchorNode()); 916 snippetCopy.start().replaceAtUsages(InputType.Memory, anchor); 917 918 this.snippet = snippetCopy; 919 920 StartNode entryPointNode = snippet.start(); 921 if (anchor.hasNoUsages()) { 922 anchor.safeDelete(); 923 this.memoryAnchor = null; 924 } else { 925 // Find out if all the return memory maps point to the anchor (i.e., there's no kill 926 // anywhere) 927 boolean needsMemoryMaps = false; 928 for (ReturnNode retNode : snippet.getNodes(ReturnNode.TYPE)) { 929 MemoryMapNode memoryMap = retNode.getMemoryMap(); 930 if (memoryMap.getLocations().size() > 1 || memoryMap.getLastLocationAccess(LocationIdentity.any()) != anchor) { 931 needsMemoryMaps = true; 932 break; 933 } 934 } 935 boolean needsAnchor; 936 if (needsMemoryMaps) { 937 needsAnchor = true; 938 } else { 939 // Check that all those memory maps where the only usages of the anchor 940 needsAnchor = anchor.usages().filter(isNotA(MemoryMapNode.class)).isNotEmpty(); 941 // Remove the useless memory map 942 MemoryMapNode memoryMap = null; 943 for (ReturnNode retNode : snippet.getNodes(ReturnNode.TYPE)) { 944 if (memoryMap == null) { 945 memoryMap = retNode.getMemoryMap(); 946 } else { 947 assert memoryMap == retNode.getMemoryMap(); 948 } 949 retNode.setMemoryMap(null); 950 } 951 memoryMap.safeDelete(); 952 } 953 if (needsAnchor) { 954 snippetCopy.addAfterFixed(snippetCopy.start(), anchor); 955 this.memoryAnchor = anchor; 956 } else { 957 anchor.safeDelete(); 958 this.memoryAnchor = null; 959 } 960 } 961 debug.dump(DebugContext.INFO_LEVEL, snippet, "SnippetTemplate after fixing memory anchoring"); 962 963 List<ReturnNode> returnNodes = snippet.getNodes(ReturnNode.TYPE).snapshot(); 964 if (returnNodes.isEmpty()) { 965 this.returnNode = null; 966 } else if (returnNodes.size() == 1) { 967 this.returnNode = returnNodes.get(0); 968 } else { 969 AbstractMergeNode merge = snippet.add(new MergeNode()); 970 List<MemoryMapNode> memMaps = new ArrayList<>(); 971 for (ReturnNode retNode : returnNodes) { 972 MemoryMapNode memoryMapNode = retNode.getMemoryMap(); 973 if (memoryMapNode != null) { 974 memMaps.add(memoryMapNode); 975 } 976 } 977 978 ValueNode returnValue = InliningUtil.mergeReturns(merge, returnNodes); 979 this.returnNode = snippet.add(new ReturnNode(returnValue)); 980 if (!memMaps.isEmpty()) { 981 MemoryMapImpl mmap = FloatingReadPhase.mergeMemoryMaps(merge, memMaps); 982 MemoryMapNode memoryMap = snippet.unique(new MemoryMapNode(mmap.getMap())); 983 this.returnNode.setMemoryMap(memoryMap); 984 for (MemoryMapNode mm : memMaps) { 985 if (mm != memoryMap && mm.isAlive()) { 986 assert mm.hasNoUsages(); 987 GraphUtil.killWithUnusedFloatingInputs(mm); 988 } 989 } 990 } 991 merge.setNext(this.returnNode); 992 } 993 994 assert verifyIntrinsicsProcessed(snippetCopy); 995 996 this.sideEffectNodes = curSideEffectNodes; 997 this.deoptNodes = curDeoptNodes; 998 this.placeholderStampedNodes = curPlaceholderStampedNodes; 999 1000 nodes = new ArrayList<>(snippet.getNodeCount()); 1001 for (Node node : snippet.getNodes()) { 1002 if (node != entryPointNode && node != entryPointNode.stateAfter()) { 1003 nodes.add(node); 1004 } 1005 } 1006 1007 if (debug.areMetricsEnabled()) { 1008 DebugContext.counter("SnippetTemplateNodeCount[%#s]", args).add(debug, nodes.size()); 1009 } 1010 debug.dump(DebugContext.INFO_LEVEL, snippet, "SnippetTemplate final state"); 1011 this.snippet.freeze(); 1012 1013 } catch (Throwable ex) { 1014 throw debug.handle(ex); 1015 } 1016 } 1017 1018 private static boolean verifyIntrinsicsProcessed(StructuredGraph snippetCopy) { 1019 for (MethodCallTargetNode target : snippetCopy.getNodes(MethodCallTargetNode.TYPE)) { 1020 ResolvedJavaMethod targetMethod = target.targetMethod(); 1021 if (targetMethod != null) { 1022 assert targetMethod.getAnnotation(Fold.class) == null && targetMethod.getAnnotation(NodeIntrinsic.class) == null : "plugin should have been processed"; 1023 } 1024 } 1025 return true; 1026 } 1027 1028 public static void explodeLoops(final StructuredGraph snippetCopy, PhaseContext phaseContext) { 1029 // Do any required loop explosion 1030 boolean exploded = false; 1031 do { 1032 exploded = false; 1033 ExplodeLoopNode explodeLoop = snippetCopy.getNodes().filter(ExplodeLoopNode.class).first(); 1034 if (explodeLoop != null) { // Earlier canonicalization may have removed the loop 1035 // altogether 1036 LoopBeginNode loopBegin = explodeLoop.findLoopBegin(); 1037 if (loopBegin != null) { 1038 LoopEx loop = new LoopsData(snippetCopy).loop(loopBegin); 1039 Mark mark = snippetCopy.getMark(); 1040 LoopTransformations.fullUnroll(loop, phaseContext, new CanonicalizerPhase()); 1041 new CanonicalizerPhase().applyIncremental(snippetCopy, phaseContext, mark); 1042 loop.deleteUnusedNodes(); 1043 } 1044 GraphUtil.removeFixedWithUnusedInputs(explodeLoop); 1045 exploded = true; 1046 } 1047 } while (exploded); 1048 } 1049 1050 protected Object[] getConstantArgs(Arguments args) { 1051 Object[] constantArgs = args.values.clone(); 1052 for (int i = 0; i < args.info.getParameterCount(); i++) { 1053 if (!args.info.isConstantParameter(i)) { 1054 constantArgs[i] = null; 1055 } else { 1056 assert constantArgs[i] != null : "Can't pass raw null through as argument"; 1057 } 1058 } 1059 return constantArgs; 1060 } 1061 1062 private static boolean checkAllVarargPlaceholdersAreDeleted(int parameterCount, VarargsPlaceholderNode[] placeholders) { 1063 for (int i = 0; i < parameterCount; i++) { 1064 if (placeholders[i] != null) { 1065 assert placeholders[i].isDeleted() : placeholders[i]; 1066 } 1067 } 1068 return true; 1069 } 1070 1071 private static boolean checkConstantArgument(MetaAccessProvider metaAccess, final ResolvedJavaMethod method, Signature signature, int paramIndex, String name, Object arg, JavaKind kind) { 1072 ResolvedJavaType type = signature.getParameterType(paramIndex, method.getDeclaringClass()).resolve(method.getDeclaringClass()); 1073 if (metaAccess.lookupJavaType(WordBase.class).isAssignableFrom(type)) { 1074 assert arg instanceof JavaConstant : method + ": word constant parameters must be passed boxed in a Constant value: " + arg; 1075 return true; 1076 } 1077 if (kind != JavaKind.Object) { 1078 assert arg != null && kind.toBoxedJavaClass() == arg.getClass() : method + ": wrong value kind for " + name + ": expected " + kind + ", got " + 1079 (arg == null ? "null" : arg.getClass().getSimpleName()); 1080 } 1081 return true; 1082 } 1083 1084 private static boolean checkVarargs(MetaAccessProvider metaAccess, final ResolvedJavaMethod method, Signature signature, int i, String name, Varargs varargs) { 1085 ResolvedJavaType type = (ResolvedJavaType) signature.getParameterType(i, method.getDeclaringClass()); 1086 assert type.isArray() : "varargs parameter must be an array type"; 1087 assert type.getComponentType().isAssignableFrom(metaAccess.lookupJavaType(varargs.componentType)) : "componentType for " + name + " not matching " + type.toJavaName() + " instance: " + 1088 varargs.componentType; 1089 return true; 1090 } 1091 1092 /** 1093 * The graph built from the snippet method. 1094 */ 1095 private final StructuredGraph snippet; 1096 1097 private final SnippetInfo info; 1098 1099 /** 1100 * The named parameters of this template that must be bound to values during instantiation. For 1101 * a parameter that is still live after specialization, the value in this map is either a 1102 * {@link ParameterNode} instance or a {@link ParameterNode} array. For an eliminated parameter, 1103 * the value is identical to the key. 1104 */ 1105 private final Object[] parameters; 1106 1107 /** 1108 * The return node (if any) of the snippet. 1109 */ 1110 private final ReturnNode returnNode; 1111 1112 /** 1113 * The memory anchor (if any) of the snippet. 1114 */ 1115 private final MemoryAnchorNode memoryAnchor; 1116 1117 /** 1118 * Nodes that inherit the {@link StateSplit#stateAfter()} from the replacee during 1119 * instantiation. 1120 */ 1121 private final ArrayList<StateSplit> sideEffectNodes; 1122 1123 /** 1124 * Nodes that inherit a deoptimization {@link FrameState} from the replacee during 1125 * instantiation. 1126 */ 1127 private final ArrayList<DeoptimizingNode> deoptNodes; 1128 1129 /** 1130 * Nodes that have a stamp originating from a {@link Placeholder}. 1131 */ 1132 private final ArrayList<ValueNode> placeholderStampedNodes; 1133 1134 /** 1135 * The nodes to be inlined when this specialization is instantiated. 1136 */ 1137 private final ArrayList<Node> nodes; 1138 1139 /** 1140 * Gets the instantiation-time bindings to this template's parameters. 1141 * 1142 * @return the map that will be used to bind arguments to parameters when inlining this template 1143 */ 1144 private EconomicMap<Node, Node> bind(StructuredGraph replaceeGraph, MetaAccessProvider metaAccess, Arguments args) { 1145 EconomicMap<Node, Node> replacements = EconomicMap.create(Equivalence.IDENTITY); 1146 assert args.info.getParameterCount() == parameters.length : "number of args (" + args.info.getParameterCount() + ") != number of parameters (" + parameters.length + ")"; 1147 for (int i = 0; i < parameters.length; i++) { 1148 Object parameter = parameters[i]; 1149 assert parameter != null : this + " has no parameter named " + args.info.getParameterName(i); 1150 Object argument = args.values[i]; 1151 if (parameter instanceof ParameterNode) { 1152 if (argument instanceof ValueNode) { 1153 replacements.put((ParameterNode) parameter, (ValueNode) argument); 1154 } else { 1155 JavaKind kind = ((ParameterNode) parameter).getStackKind(); 1156 assert argument != null || kind == JavaKind.Object : this + " cannot accept null for non-object parameter named " + args.info.getParameterName(i); 1157 JavaConstant constant = forBoxed(argument, kind); 1158 replacements.put((ParameterNode) parameter, ConstantNode.forConstant(constant, metaAccess, replaceeGraph)); 1159 } 1160 } else if (parameter instanceof ParameterNode[]) { 1161 ParameterNode[] params = (ParameterNode[]) parameter; 1162 Varargs varargs = (Varargs) argument; 1163 int length = params.length; 1164 List<?> list = null; 1165 Object array = null; 1166 if (varargs.value instanceof List) { 1167 list = (List<?>) varargs.value; 1168 assert list.size() == length : length + " != " + list.size(); 1169 } else { 1170 array = varargs.value; 1171 assert array != null && array.getClass().isArray(); 1172 assert Array.getLength(array) == length : length + " != " + Array.getLength(array); 1173 } 1174 1175 for (int j = 0; j < length; j++) { 1176 ParameterNode param = params[j]; 1177 assert param != null; 1178 Object value = list != null ? list.get(j) : Array.get(array, j); 1179 if (value instanceof ValueNode) { 1180 replacements.put(param, (ValueNode) value); 1181 } else { 1182 JavaConstant constant = forBoxed(value, param.getStackKind()); 1183 ConstantNode element = ConstantNode.forConstant(constant, metaAccess, replaceeGraph); 1184 replacements.put(param, element); 1185 } 1186 } 1187 } else { 1188 assert parameter.equals(CONSTANT_PARAMETER) || parameter.equals(UNUSED_PARAMETER) : "unexpected entry for parameter: " + args.info.getParameterName(i) + " -> " + parameter; 1189 } 1190 } 1191 return replacements; 1192 } 1193 1194 /** 1195 * Converts a Java boxed value to a {@link JavaConstant} of the right kind. This adjusts for the 1196 * limitation that a {@link Local}'s kind is a {@linkplain JavaKind#getStackKind() stack kind} 1197 * and so cannot be used for re-boxing primitives smaller than an int. 1198 * 1199 * @param argument a Java boxed value 1200 * @param localKind the kind of the {@link Local} to which {@code argument} will be bound 1201 */ 1202 protected JavaConstant forBoxed(Object argument, JavaKind localKind) { 1203 assert localKind == localKind.getStackKind(); 1204 if (localKind == JavaKind.Int) { 1205 return JavaConstant.forBoxedPrimitive(argument); 1206 } 1207 return snippetReflection.forBoxed(localKind, argument); 1208 } 1209 1210 /** 1211 * Logic for replacing a snippet-lowered node at its usages with the return value of the 1212 * snippet. An alternative to the {@linkplain SnippetTemplate#DEFAULT_REPLACER default} 1213 * replacement logic can be used to handle mismatches between the stamp of the node being 1214 * lowered and the stamp of the snippet's return value. 1215 */ 1216 public interface UsageReplacer { 1217 /** 1218 * Replaces all usages of {@code oldNode} with direct or indirect usages of {@code newNode}. 1219 */ 1220 void replace(ValueNode oldNode, ValueNode newNode); 1221 } 1222 1223 /** 1224 * Represents the default {@link UsageReplacer usage replacer} logic which simply delegates to 1225 * {@link Node#replaceAtUsages(Node)}. 1226 */ 1227 public static final UsageReplacer DEFAULT_REPLACER = new UsageReplacer() { 1228 1229 @Override 1230 public void replace(ValueNode oldNode, ValueNode newNode) { 1231 if (newNode == null) { 1232 assert oldNode.hasNoUsages(); 1233 } else { 1234 oldNode.replaceAtUsages(newNode); 1235 } 1236 } 1237 }; 1238 1239 private boolean assertSnippetKills(ValueNode replacee) { 1240 if (!replacee.graph().isAfterFloatingReadPhase()) { 1241 // no floating reads yet, ignore locations created while lowering 1242 return true; 1243 } 1244 if (returnNode == null) { 1245 // The snippet terminates control flow 1246 return true; 1247 } 1248 MemoryMapNode memoryMap = returnNode.getMemoryMap(); 1249 if (memoryMap == null || memoryMap.isEmpty()) { 1250 // there are no kills in the snippet graph 1251 return true; 1252 } 1253 1254 EconomicSet<LocationIdentity> kills = EconomicSet.create(Equivalence.DEFAULT); 1255 kills.addAll(memoryMap.getLocations()); 1256 1257 if (replacee instanceof MemoryCheckpoint.Single) { 1258 // check if some node in snippet graph also kills the same location 1259 LocationIdentity locationIdentity = ((MemoryCheckpoint.Single) replacee).getLocationIdentity(); 1260 if (locationIdentity.isAny()) { 1261 assert !(memoryMap.getLastLocationAccess(any()) instanceof MemoryAnchorNode) : replacee + " kills ANY_LOCATION, but snippet does not"; 1262 // if the replacee kills ANY_LOCATION, the snippet can kill arbitrary locations 1263 return true; 1264 } 1265 assert kills.contains(locationIdentity) : replacee + " kills " + locationIdentity + ", but snippet doesn't contain a kill to this location"; 1266 kills.remove(locationIdentity); 1267 } 1268 assert !(replacee instanceof MemoryCheckpoint.Multi) : replacee + " multi not supported (yet)"; 1269 1270 // remove ANY_LOCATION if it's just a kill by the start node 1271 if (memoryMap.getLastLocationAccess(any()) instanceof MemoryAnchorNode) { 1272 kills.remove(any()); 1273 } 1274 1275 // node can only lower to a ANY_LOCATION kill if the replacee also kills ANY_LOCATION 1276 assert !kills.contains(any()) : "snippet graph contains a kill to ANY_LOCATION, but replacee (" + replacee + ") doesn't kill ANY_LOCATION. kills: " + kills; 1277 1278 /* 1279 * Kills to private locations are safe, since there can be no floating read to these 1280 * locations except reads that are introduced by the snippet itself or related snippets in 1281 * the same lowering round. These reads are anchored to a MemoryAnchor at the beginning of 1282 * their snippet, so they can not float above a kill in another instance of the same 1283 * snippet. 1284 */ 1285 for (LocationIdentity p : this.info.privateLocations) { 1286 kills.remove(p); 1287 } 1288 1289 assert kills.isEmpty() : "snippet graph kills non-private locations " + kills + " that replacee (" + replacee + ") doesn't kill"; 1290 return true; 1291 } 1292 1293 private static class MemoryInputMap implements MemoryMap { 1294 1295 private final LocationIdentity locationIdentity; 1296 private final MemoryNode lastLocationAccess; 1297 1298 MemoryInputMap(ValueNode replacee) { 1299 if (replacee instanceof MemoryAccess) { 1300 MemoryAccess access = (MemoryAccess) replacee; 1301 locationIdentity = access.getLocationIdentity(); 1302 lastLocationAccess = access.getLastLocationAccess(); 1303 } else { 1304 locationIdentity = null; 1305 lastLocationAccess = null; 1306 } 1307 } 1308 1309 @Override 1310 public MemoryNode getLastLocationAccess(LocationIdentity location) { 1311 if (locationIdentity != null && locationIdentity.equals(location)) { 1312 return lastLocationAccess; 1313 } else { 1314 return null; 1315 } 1316 } 1317 1318 @Override 1319 public Collection<LocationIdentity> getLocations() { 1320 if (locationIdentity == null) { 1321 return Collections.emptySet(); 1322 } else { 1323 return Collections.singleton(locationIdentity); 1324 } 1325 } 1326 } 1327 1328 private class MemoryOutputMap extends MemoryInputMap { 1329 1330 private final UnmodifiableEconomicMap<Node, Node> duplicates; 1331 1332 MemoryOutputMap(ValueNode replacee, UnmodifiableEconomicMap<Node, Node> duplicates) { 1333 super(replacee); 1334 this.duplicates = duplicates; 1335 } 1336 1337 @Override 1338 public MemoryNode getLastLocationAccess(LocationIdentity locationIdentity) { 1339 MemoryMapNode memoryMap = returnNode.getMemoryMap(); 1340 assert memoryMap != null : "no memory map stored for this snippet graph (snippet doesn't have a ReturnNode?)"; 1341 MemoryNode lastLocationAccess = memoryMap.getLastLocationAccess(locationIdentity); 1342 assert lastLocationAccess != null : locationIdentity; 1343 if (lastLocationAccess == memoryAnchor) { 1344 return super.getLastLocationAccess(locationIdentity); 1345 } else { 1346 return (MemoryNode) duplicates.get(ValueNodeUtil.asNode(lastLocationAccess)); 1347 } 1348 } 1349 1350 @Override 1351 public Collection<LocationIdentity> getLocations() { 1352 return returnNode.getMemoryMap().getLocations(); 1353 } 1354 } 1355 1356 private void rewireMemoryGraph(ValueNode replacee, UnmodifiableEconomicMap<Node, Node> duplicates) { 1357 if (replacee.graph().isAfterFloatingReadPhase()) { 1358 // rewire outgoing memory edges 1359 replaceMemoryUsages(replacee, new MemoryOutputMap(replacee, duplicates)); 1360 1361 if (returnNode != null) { 1362 ReturnNode ret = (ReturnNode) duplicates.get(returnNode); 1363 if (ret != null) { 1364 MemoryMapNode memoryMap = ret.getMemoryMap(); 1365 if (memoryMap != null) { 1366 ret.setMemoryMap(null); 1367 memoryMap.safeDelete(); 1368 } 1369 } 1370 } 1371 if (memoryAnchor != null) { 1372 // rewire incoming memory edges 1373 MemoryAnchorNode memoryDuplicate = (MemoryAnchorNode) duplicates.get(memoryAnchor); 1374 replaceMemoryUsages(memoryDuplicate, new MemoryInputMap(replacee)); 1375 1376 if (memoryDuplicate.hasNoUsages()) { 1377 if (memoryDuplicate.next() != null) { 1378 memoryDuplicate.graph().removeFixed(memoryDuplicate); 1379 } else { 1380 // this was a dummy memory node used when instantiating pure data-flow 1381 // snippets: it was not attached to the control flow. 1382 memoryDuplicate.safeDelete(); 1383 } 1384 } 1385 } 1386 } 1387 } 1388 1389 private static LocationIdentity getLocationIdentity(Node node) { 1390 if (node instanceof MemoryAccess) { 1391 return ((MemoryAccess) node).getLocationIdentity(); 1392 } else if (node instanceof MemoryProxy) { 1393 return ((MemoryProxy) node).getLocationIdentity(); 1394 } else if (node instanceof MemoryPhiNode) { 1395 return ((MemoryPhiNode) node).getLocationIdentity(); 1396 } else { 1397 return null; 1398 } 1399 } 1400 1401 private void replaceMemoryUsages(ValueNode node, MemoryMap map) { 1402 for (Node usage : node.usages().snapshot()) { 1403 if (usage instanceof MemoryMapNode) { 1404 continue; 1405 } 1406 1407 LocationIdentity location = getLocationIdentity(usage); 1408 if (location != null) { 1409 for (Position pos : usage.inputPositions()) { 1410 if (pos.getInputType() == InputType.Memory && pos.get(usage) == node) { 1411 MemoryNode replacement = map.getLastLocationAccess(location); 1412 if (replacement == null) { 1413 assert mayRemoveLocation || LocationIdentity.any().equals(location) || 1414 CollectionsUtil.anyMatch(info.privateLocations, Predicate.isEqual(location)) : "Snippet " + 1415 info.method.format("%h.%n") + " contains access to the non-private location " + 1416 location + ", but replacee doesn't access this location." + map.getLocations(); 1417 } else { 1418 pos.set(usage, replacement.asNode()); 1419 } 1420 } 1421 } 1422 } 1423 } 1424 } 1425 1426 /** 1427 * Replaces a given fixed node with this specialized snippet. 1428 * 1429 * @param metaAccess 1430 * @param replacee the node that will be replaced 1431 * @param replacer object that replaces the usages of {@code replacee} 1432 * @param args the arguments to be bound to the flattened positional parameters of the snippet 1433 * @return the map of duplicated nodes (original -> duplicate) 1434 */ 1435 @SuppressWarnings("try") 1436 public UnmodifiableEconomicMap<Node, Node> instantiate(MetaAccessProvider metaAccess, FixedNode replacee, UsageReplacer replacer, Arguments args) { 1437 return instantiate(metaAccess, replacee, replacer, args, true); 1438 } 1439 1440 /** 1441 * Replaces a given fixed node with this specialized snippet. 1442 * 1443 * @param metaAccess 1444 * @param replacee the node that will be replaced 1445 * @param replacer object that replaces the usages of {@code replacee} 1446 * @param args the arguments to be bound to the flattened positional parameters of the snippet 1447 * @param killReplacee is true, the replacee node is deleted 1448 * @return the map of duplicated nodes (original -> duplicate) 1449 */ 1450 @SuppressWarnings("try") 1451 public UnmodifiableEconomicMap<Node, Node> instantiate(MetaAccessProvider metaAccess, FixedNode replacee, UsageReplacer replacer, Arguments args, boolean killReplacee) { 1452 DebugContext debug = replacee.getDebug(); 1453 assert assertSnippetKills(replacee); 1454 try (DebugCloseable a = args.info.instantiationTimer.start(debug)) { 1455 args.info.instantiationCounter.increment(debug); 1456 // Inline the snippet nodes, replacing parameters with the given args in the process 1457 StartNode entryPointNode = snippet.start(); 1458 FixedNode firstCFGNode = entryPointNode.next(); 1459 StructuredGraph replaceeGraph = replacee.graph(); 1460 EconomicMap<Node, Node> replacements = bind(replaceeGraph, metaAccess, args); 1461 replacements.put(entryPointNode, AbstractBeginNode.prevBegin(replacee)); 1462 UnmodifiableEconomicMap<Node, Node> duplicates = inlineSnippet(replacee, debug, replaceeGraph, replacements); 1463 1464 // Re-wire the control flow graph around the replacee 1465 FixedNode firstCFGNodeDuplicate = (FixedNode) duplicates.get(firstCFGNode); 1466 replacee.replaceAtPredecessor(firstCFGNodeDuplicate); 1467 1468 rewireFrameStates(replacee, duplicates); 1469 1470 if (replacee instanceof DeoptimizingNode) { 1471 DeoptimizingNode replaceeDeopt = (DeoptimizingNode) replacee; 1472 1473 FrameState stateBefore = null; 1474 FrameState stateDuring = null; 1475 FrameState stateAfter = null; 1476 if (replaceeDeopt.canDeoptimize()) { 1477 if (replaceeDeopt instanceof DeoptimizingNode.DeoptBefore) { 1478 stateBefore = ((DeoptimizingNode.DeoptBefore) replaceeDeopt).stateBefore(); 1479 } 1480 if (replaceeDeopt instanceof DeoptimizingNode.DeoptDuring) { 1481 stateDuring = ((DeoptimizingNode.DeoptDuring) replaceeDeopt).stateDuring(); 1482 } 1483 if (replaceeDeopt instanceof DeoptimizingNode.DeoptAfter) { 1484 stateAfter = ((DeoptimizingNode.DeoptAfter) replaceeDeopt).stateAfter(); 1485 } 1486 } 1487 1488 for (DeoptimizingNode deoptNode : deoptNodes) { 1489 DeoptimizingNode deoptDup = (DeoptimizingNode) duplicates.get(deoptNode.asNode()); 1490 if (deoptDup.canDeoptimize()) { 1491 if (deoptDup instanceof DeoptimizingNode.DeoptBefore) { 1492 ((DeoptimizingNode.DeoptBefore) deoptDup).setStateBefore(stateBefore); 1493 } 1494 if (deoptDup instanceof DeoptimizingNode.DeoptDuring) { 1495 DeoptimizingNode.DeoptDuring deoptDupDuring = (DeoptimizingNode.DeoptDuring) deoptDup; 1496 if (stateDuring != null) { 1497 deoptDupDuring.setStateDuring(stateDuring); 1498 } else if (stateAfter != null) { 1499 deoptDupDuring.computeStateDuring(stateAfter); 1500 } else if (stateBefore != null) { 1501 assert !deoptDupDuring.hasSideEffect() : "can't use stateBefore as stateDuring for state split " + deoptDupDuring; 1502 deoptDupDuring.setStateDuring(stateBefore); 1503 } 1504 } 1505 if (deoptDup instanceof DeoptimizingNode.DeoptAfter) { 1506 DeoptimizingNode.DeoptAfter deoptDupAfter = (DeoptimizingNode.DeoptAfter) deoptDup; 1507 if (stateAfter != null) { 1508 deoptDupAfter.setStateAfter(stateAfter); 1509 } else { 1510 assert !deoptDupAfter.hasSideEffect() : "can't use stateBefore as stateAfter for state split " + deoptDupAfter; 1511 deoptDupAfter.setStateAfter(stateBefore); 1512 } 1513 1514 } 1515 } 1516 } 1517 } 1518 1519 updateStamps(replacee, duplicates); 1520 1521 rewireMemoryGraph(replacee, duplicates); 1522 1523 // Replace all usages of the replacee with the value returned by the snippet 1524 ValueNode returnValue = null; 1525 if (returnNode != null && !(replacee instanceof ControlSinkNode)) { 1526 ReturnNode returnDuplicate = (ReturnNode) duplicates.get(returnNode); 1527 returnValue = returnDuplicate.result(); 1528 if (returnValue == null && replacee.usages().isNotEmpty() && replacee instanceof MemoryCheckpoint) { 1529 replacer.replace(replacee, null); 1530 } else { 1531 assert returnValue != null || replacee.hasNoUsages(); 1532 replacer.replace(replacee, returnValue); 1533 } 1534 if (returnDuplicate.isAlive()) { 1535 FixedNode next = null; 1536 if (replacee instanceof FixedWithNextNode) { 1537 FixedWithNextNode fwn = (FixedWithNextNode) replacee; 1538 next = fwn.next(); 1539 fwn.setNext(null); 1540 } 1541 returnDuplicate.replaceAndDelete(next); 1542 } 1543 } 1544 1545 if (killReplacee) { 1546 // Remove the replacee from its graph 1547 GraphUtil.killCFG(replacee); 1548 } 1549 1550 debug.dump(DebugContext.DETAILED_LEVEL, replaceeGraph, "After lowering %s with %s", replacee, this); 1551 return duplicates; 1552 } 1553 } 1554 1555 private UnmodifiableEconomicMap<Node, Node> inlineSnippet(Node replacee, DebugContext debug, StructuredGraph replaceeGraph, EconomicMap<Node, Node> replacements) { 1556 Mark mark = replaceeGraph.getMark(); 1557 try (InliningLog.UpdateScope scope = replaceeGraph.getInliningLog().openUpdateScope((oldNode, newNode) -> { 1558 InliningLog log = replaceeGraph.getInliningLog(); 1559 if (oldNode == null) { 1560 log.trackNewCallsite(newNode); 1561 } 1562 })) { 1563 UnmodifiableEconomicMap<Node, Node> duplicates = replaceeGraph.addDuplicates(nodes, snippet, snippet.getNodeCount(), replacements); 1564 if (scope != null) { 1565 replaceeGraph.getInliningLog().addLog(duplicates, snippet.getInliningLog()); 1566 } 1567 NodeSourcePosition position = replacee.getNodeSourcePosition(); 1568 InliningUtil.updateSourcePosition(replaceeGraph, duplicates, mark, position, true); 1569 debug.dump(DebugContext.DETAILED_LEVEL, replaceeGraph, "After inlining snippet %s", snippet.method()); 1570 return duplicates; 1571 } 1572 } 1573 1574 private void propagateStamp(Node node) { 1575 if (node instanceof PhiNode) { 1576 PhiNode phi = (PhiNode) node; 1577 if (phi.inferStamp()) { 1578 for (Node usage : node.usages()) { 1579 propagateStamp(usage); 1580 } 1581 } 1582 } 1583 } 1584 1585 private void updateStamps(ValueNode replacee, UnmodifiableEconomicMap<Node, Node> duplicates) { 1586 for (ValueNode node : placeholderStampedNodes) { 1587 ValueNode dup = (ValueNode) duplicates.get(node); 1588 Stamp replaceeStamp = replacee.stamp(NodeView.DEFAULT); 1589 if (node instanceof Placeholder) { 1590 Placeholder placeholderDup = (Placeholder) dup; 1591 placeholderDup.makeReplacement(replaceeStamp); 1592 } else { 1593 dup.setStamp(replaceeStamp); 1594 } 1595 } 1596 for (ParameterNode paramNode : snippet.getNodes(ParameterNode.TYPE)) { 1597 for (Node usage : paramNode.usages()) { 1598 Node usageDup = duplicates.get(usage); 1599 propagateStamp(usageDup); 1600 } 1601 } 1602 } 1603 1604 /** 1605 * Gets a copy of the specialized graph. 1606 */ 1607 public StructuredGraph copySpecializedGraph(DebugContext debugForCopy) { 1608 return (StructuredGraph) snippet.copy(debugForCopy); 1609 } 1610 1611 /** 1612 * Replaces a given floating node with this specialized snippet. 1613 * 1614 * @param metaAccess 1615 * @param replacee the node that will be replaced 1616 * @param replacer object that replaces the usages of {@code replacee} 1617 * @param tool lowering tool used to insert the snippet into the control-flow 1618 * @param args the arguments to be bound to the flattened positional parameters of the snippet 1619 */ 1620 @SuppressWarnings("try") 1621 public void instantiate(MetaAccessProvider metaAccess, FloatingNode replacee, UsageReplacer replacer, LoweringTool tool, Arguments args) { 1622 DebugContext debug = replacee.getDebug(); 1623 assert assertSnippetKills(replacee); 1624 try (DebugCloseable a = args.info.instantiationTimer.start(debug)) { 1625 args.info.instantiationCounter.increment(debug); 1626 1627 // Inline the snippet nodes, replacing parameters with the given args in the process 1628 StartNode entryPointNode = snippet.start(); 1629 FixedNode firstCFGNode = entryPointNode.next(); 1630 StructuredGraph replaceeGraph = replacee.graph(); 1631 EconomicMap<Node, Node> replacements = bind(replaceeGraph, metaAccess, args); 1632 replacements.put(entryPointNode, tool.getCurrentGuardAnchor().asNode()); 1633 UnmodifiableEconomicMap<Node, Node> duplicates = inlineSnippet(replacee, debug, replaceeGraph, replacements); 1634 1635 FixedWithNextNode lastFixedNode = tool.lastFixedNode(); 1636 assert lastFixedNode != null && lastFixedNode.isAlive() : replaceeGraph + " lastFixed=" + lastFixedNode; 1637 FixedNode next = lastFixedNode.next(); 1638 lastFixedNode.setNext(null); 1639 FixedNode firstCFGNodeDuplicate = (FixedNode) duplicates.get(firstCFGNode); 1640 replaceeGraph.addAfterFixed(lastFixedNode, firstCFGNodeDuplicate); 1641 1642 rewireFrameStates(replacee, duplicates); 1643 updateStamps(replacee, duplicates); 1644 1645 rewireMemoryGraph(replacee, duplicates); 1646 1647 // Replace all usages of the replacee with the value returned by the snippet 1648 ReturnNode returnDuplicate = (ReturnNode) duplicates.get(returnNode); 1649 ValueNode returnValue = returnDuplicate.result(); 1650 assert returnValue != null || replacee.hasNoUsages(); 1651 replacer.replace(replacee, returnValue); 1652 1653 if (returnDuplicate.isAlive()) { 1654 returnDuplicate.replaceAndDelete(next); 1655 } 1656 1657 debug.dump(DebugContext.DETAILED_LEVEL, replaceeGraph, "After lowering %s with %s", replacee, this); 1658 } 1659 } 1660 1661 /** 1662 * Replaces a given floating node with this specialized snippet. 1663 * 1664 * This snippet must be pure data-flow 1665 * 1666 * @param metaAccess 1667 * @param replacee the node that will be replaced 1668 * @param replacer object that replaces the usages of {@code replacee} 1669 * @param args the arguments to be bound to the flattened positional parameters of the snippet 1670 */ 1671 @SuppressWarnings("try") 1672 public void instantiate(MetaAccessProvider metaAccess, FloatingNode replacee, UsageReplacer replacer, Arguments args) { 1673 DebugContext debug = replacee.getDebug(); 1674 assert assertSnippetKills(replacee); 1675 try (DebugCloseable a = args.info.instantiationTimer.start(debug)) { 1676 args.info.instantiationCounter.increment(debug); 1677 1678 // Inline the snippet nodes, replacing parameters with the given args in the process 1679 StartNode entryPointNode = snippet.start(); 1680 assert entryPointNode.next() == (memoryAnchor == null ? returnNode : memoryAnchor) : entryPointNode.next(); 1681 StructuredGraph replaceeGraph = replacee.graph(); 1682 EconomicMap<Node, Node> replacements = bind(replaceeGraph, metaAccess, args); 1683 MemoryAnchorNode anchorDuplicate = null; 1684 if (memoryAnchor != null) { 1685 anchorDuplicate = replaceeGraph.add(new MemoryAnchorNode()); 1686 replacements.put(memoryAnchor, anchorDuplicate); 1687 } 1688 List<Node> floatingNodes = new ArrayList<>(nodes.size() - 2); 1689 for (Node n : nodes) { 1690 if (n != entryPointNode && n != returnNode) { 1691 floatingNodes.add(n); 1692 } 1693 } 1694 UnmodifiableEconomicMap<Node, Node> duplicates = inlineSnippet(replacee, debug, replaceeGraph, replacements); 1695 1696 rewireFrameStates(replacee, duplicates); 1697 updateStamps(replacee, duplicates); 1698 1699 rewireMemoryGraph(replacee, duplicates); 1700 assert anchorDuplicate == null || anchorDuplicate.isDeleted(); 1701 1702 // Replace all usages of the replacee with the value returned by the snippet 1703 ValueNode returnValue = (ValueNode) duplicates.get(returnNode.result()); 1704 replacer.replace(replacee, returnValue); 1705 1706 debug.dump(DebugContext.DETAILED_LEVEL, replaceeGraph, "After lowering %s with %s", replacee, this); 1707 } 1708 } 1709 1710 protected void rewireFrameStates(ValueNode replacee, UnmodifiableEconomicMap<Node, Node> duplicates) { 1711 if (replacee instanceof StateSplit) { 1712 for (StateSplit sideEffectNode : sideEffectNodes) { 1713 assert ((StateSplit) replacee).hasSideEffect(); 1714 Node sideEffectDup = duplicates.get(sideEffectNode.asNode()); 1715 ((StateSplit) sideEffectDup).setStateAfter(((StateSplit) replacee).stateAfter()); 1716 } 1717 } 1718 } 1719 1720 @Override 1721 public String toString() { 1722 StringBuilder buf = new StringBuilder(snippet.toString()).append('('); 1723 String sep = ""; 1724 for (int i = 0; i < parameters.length; i++) { 1725 String name = "[" + i + "]"; 1726 Object value = parameters[i]; 1727 buf.append(sep); 1728 sep = ", "; 1729 if (value == null) { 1730 buf.append("<null> ").append(name); 1731 } else if (value.equals(UNUSED_PARAMETER)) { 1732 buf.append("<unused> ").append(name); 1733 } else if (value.equals(CONSTANT_PARAMETER)) { 1734 buf.append("<constant> ").append(name); 1735 } else if (value instanceof ParameterNode) { 1736 ParameterNode param = (ParameterNode) value; 1737 buf.append(param.getStackKind().getJavaName()).append(' ').append(name); 1738 } else { 1739 ParameterNode[] params = (ParameterNode[]) value; 1740 String kind = params.length == 0 ? "?" : params[0].getStackKind().getJavaName(); 1741 buf.append(kind).append('[').append(params.length).append("] ").append(name); 1742 } 1743 } 1744 return buf.append(')').toString(); 1745 } 1746 1747 private static boolean checkTemplate(MetaAccessProvider metaAccess, Arguments args, ResolvedJavaMethod method, Signature signature) { 1748 int offset = args.info.hasReceiver() ? 1 : 0; 1749 for (int i = offset; i < args.info.getParameterCount(); i++) { 1750 if (args.info.isConstantParameter(i)) { 1751 JavaKind kind = signature.getParameterKind(i - offset); 1752 assert checkConstantArgument(metaAccess, method, signature, i - offset, args.info.getParameterName(i), args.values[i], kind); 1753 1754 } else if (args.info.isVarargsParameter(i)) { 1755 assert args.values[i] instanceof Varargs; 1756 Varargs varargs = (Varargs) args.values[i]; 1757 assert checkVarargs(metaAccess, method, signature, i, args.info.getParameterName(i), varargs); 1758 } 1759 } 1760 return true; 1761 } 1762 1763 public void setMayRemoveLocation(boolean mayRemoveLocation) { 1764 this.mayRemoveLocation = mayRemoveLocation; 1765 } 1766 }