1 /* 2 * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 25 package org.graalvm.compiler.replacements; 26 27 import static java.util.FormattableFlags.ALTERNATE; 28 import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE; 29 import static org.graalvm.compiler.debug.DebugContext.DEFAULT_LOG_STREAM; 30 import static org.graalvm.compiler.debug.DebugContext.applyFormattingFlagsAndWidth; 31 import static org.graalvm.compiler.debug.DebugOptions.DebugStubsAndSnippets; 32 import static org.graalvm.compiler.graph.iterators.NodePredicates.isNotA; 33 import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_IGNORED; 34 import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_IGNORED; 35 import static org.graalvm.compiler.phases.common.DeadCodeEliminationPhase.Optionality.Required; 36 import static jdk.internal.vm.compiler.word.LocationIdentity.any; 37 38 import java.lang.reflect.Array; 39 import java.lang.reflect.Method; 40 import java.util.ArrayList; 41 import java.util.Arrays; 42 import java.util.Collection; 43 import java.util.Collections; 44 import java.util.Formattable; 45 import java.util.Formatter; 46 import java.util.LinkedHashMap; 47 import java.util.List; 48 import java.util.Map; 49 import java.util.concurrent.atomic.AtomicInteger; 50 import java.util.concurrent.atomic.AtomicReference; 51 import java.util.function.Predicate; 52 53 import jdk.internal.vm.compiler.collections.EconomicMap; 54 import jdk.internal.vm.compiler.collections.EconomicSet; 55 import jdk.internal.vm.compiler.collections.Equivalence; 56 import jdk.internal.vm.compiler.collections.UnmodifiableEconomicMap; 57 import org.graalvm.compiler.api.replacements.Fold; 58 import org.graalvm.compiler.api.replacements.Snippet; 59 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter; 60 import org.graalvm.compiler.api.replacements.Snippet.NonNullParameter; 61 import org.graalvm.compiler.api.replacements.Snippet.VarargsParameter; 62 import org.graalvm.compiler.api.replacements.SnippetReflectionProvider; 63 import org.graalvm.compiler.core.common.GraalOptions; 64 import org.graalvm.compiler.core.common.type.Stamp; 65 import org.graalvm.compiler.core.common.type.StampFactory; 66 import org.graalvm.compiler.core.common.type.StampPair; 67 import org.graalvm.compiler.core.common.type.TypeReference; 68 import org.graalvm.compiler.debug.Assertions; 69 import org.graalvm.compiler.debug.CounterKey; 70 import org.graalvm.compiler.debug.DebugCloseable; 71 import org.graalvm.compiler.debug.DebugContext; 72 import org.graalvm.compiler.debug.DebugContext.Description; 73 import org.graalvm.compiler.debug.DebugHandlersFactory; 74 import org.graalvm.compiler.debug.GraalError; 75 import org.graalvm.compiler.debug.TimerKey; 76 import org.graalvm.compiler.graph.Graph.Mark; 77 import org.graalvm.compiler.graph.Node; 78 import org.graalvm.compiler.graph.Node.NodeIntrinsic; 79 import org.graalvm.compiler.graph.NodeClass; 80 import org.graalvm.compiler.graph.NodeSourcePosition; 81 import org.graalvm.compiler.graph.Position; 82 import org.graalvm.compiler.loop.LoopEx; 83 import org.graalvm.compiler.loop.LoopsData; 84 import org.graalvm.compiler.loop.phases.LoopTransformations; 85 import org.graalvm.compiler.nodeinfo.InputType; 86 import org.graalvm.compiler.nodeinfo.NodeInfo; 87 import org.graalvm.compiler.nodes.AbstractBeginNode; 88 import org.graalvm.compiler.nodes.AbstractMergeNode; 89 import org.graalvm.compiler.nodes.ConstantNode; 90 import org.graalvm.compiler.nodes.ControlSinkNode; 91 import org.graalvm.compiler.nodes.DeoptimizingNode; 92 import org.graalvm.compiler.nodes.FixedNode; 93 import org.graalvm.compiler.nodes.FixedWithNextNode; 94 import org.graalvm.compiler.nodes.FrameState; 95 import org.graalvm.compiler.nodes.InliningLog; 96 import org.graalvm.compiler.nodes.LoopBeginNode; 97 import org.graalvm.compiler.nodes.MergeNode; 98 import org.graalvm.compiler.nodes.NodeView; 99 import org.graalvm.compiler.nodes.ParameterNode; 100 import org.graalvm.compiler.nodes.PhiNode; 101 import org.graalvm.compiler.nodes.PiNode.Placeholder; 102 import org.graalvm.compiler.nodes.PiNode.PlaceholderStamp; 103 import org.graalvm.compiler.nodes.ReturnNode; 104 import org.graalvm.compiler.nodes.StartNode; 105 import org.graalvm.compiler.nodes.StateSplit; 106 import org.graalvm.compiler.nodes.StructuredGraph; 107 import org.graalvm.compiler.nodes.StructuredGraph.GuardsStage; 108 import org.graalvm.compiler.nodes.ValueNode; 109 import org.graalvm.compiler.nodes.ValueNodeUtil; 110 import org.graalvm.compiler.nodes.calc.FloatingNode; 111 import org.graalvm.compiler.nodes.java.LoadIndexedNode; 112 import org.graalvm.compiler.nodes.java.MethodCallTargetNode; 113 import org.graalvm.compiler.nodes.java.StoreIndexedNode; 114 import org.graalvm.compiler.nodes.memory.MemoryAccess; 115 import org.graalvm.compiler.nodes.memory.MemoryAnchorNode; 116 import org.graalvm.compiler.nodes.memory.MemoryCheckpoint; 117 import org.graalvm.compiler.nodes.memory.MemoryMap; 118 import org.graalvm.compiler.nodes.memory.MemoryMapNode; 119 import org.graalvm.compiler.nodes.memory.MemoryNode; 120 import org.graalvm.compiler.nodes.memory.MemoryPhiNode; 121 import org.graalvm.compiler.nodes.spi.ArrayLengthProvider; 122 import org.graalvm.compiler.nodes.spi.LoweringTool; 123 import org.graalvm.compiler.nodes.spi.MemoryProxy; 124 import org.graalvm.compiler.nodes.util.GraphUtil; 125 import org.graalvm.compiler.options.Option; 126 import org.graalvm.compiler.options.OptionKey; 127 import org.graalvm.compiler.options.OptionValues; 128 import org.graalvm.compiler.phases.common.CanonicalizerPhase; 129 import org.graalvm.compiler.phases.common.DeadCodeEliminationPhase; 130 import org.graalvm.compiler.phases.common.FloatingReadPhase; 131 import org.graalvm.compiler.phases.common.FloatingReadPhase.MemoryMapImpl; 132 import org.graalvm.compiler.phases.common.GuardLoweringPhase; 133 import org.graalvm.compiler.phases.common.LoweringPhase; 134 import org.graalvm.compiler.phases.common.RemoveValueProxyPhase; 135 import org.graalvm.compiler.phases.common.inlining.InliningUtil; 136 import org.graalvm.compiler.phases.tiers.PhaseContext; 137 import org.graalvm.compiler.phases.util.Providers; 138 import org.graalvm.compiler.replacements.nodes.ExplodeLoopNode; 139 import org.graalvm.compiler.replacements.nodes.LoadSnippetVarargParameterNode; 140 import org.graalvm.util.CollectionsUtil; 141 import jdk.internal.vm.compiler.word.LocationIdentity; 142 import jdk.internal.vm.compiler.word.WordBase; 143 144 import jdk.vm.ci.code.TargetDescription; 145 import jdk.vm.ci.meta.Constant; 146 import jdk.vm.ci.meta.ConstantReflectionProvider; 147 import jdk.vm.ci.meta.JavaConstant; 148 import jdk.vm.ci.meta.JavaKind; 149 import jdk.vm.ci.meta.Local; 150 import jdk.vm.ci.meta.LocalVariableTable; 151 import jdk.vm.ci.meta.MetaAccessProvider; 152 import jdk.vm.ci.meta.ResolvedJavaMethod; 153 import jdk.vm.ci.meta.ResolvedJavaMethod.Parameter; 154 import jdk.vm.ci.meta.ResolvedJavaType; 155 import jdk.vm.ci.meta.Signature; 156 157 /** 158 * A snippet template is a graph created by parsing a snippet method and then specialized by binding 159 * constants to the snippet's {@link ConstantParameter} parameters. 160 * 161 * Snippet templates can be managed in a cache maintained by {@link AbstractTemplates}. 162 */ 163 public class SnippetTemplate { 164 165 private boolean mayRemoveLocation = false; 166 167 /** 168 * Holds the {@link ResolvedJavaMethod} of the snippet, together with some information about the 169 * method that needs to be computed only once. The {@link SnippetInfo} should be created once 170 * per snippet and then cached. 171 */ 172 public abstract static class SnippetInfo { 173 174 protected final ResolvedJavaMethod method; 175 protected final ResolvedJavaMethod original; 176 protected final LocationIdentity[] privateLocations; 177 protected final Object receiver; 178 179 public Object getReceiver() { 180 return receiver; 181 } 182 183 boolean hasReceiver() { 184 assert hasReceiver(method) == (receiver != null) : "Snippet with the receiver must have it set as constant. Snippet: " + this; 185 return hasReceiver(method); 186 } 187 188 static boolean hasReceiver(ResolvedJavaMethod method) { 189 return method.hasReceiver(); 190 } 191 192 /** 193 * Lazily constructed parts of {@link SnippetInfo}. 194 */ 195 static class Lazy { 196 Lazy(ResolvedJavaMethod method) { 197 int count = method.getSignature().getParameterCount(hasReceiver(method)); 198 constantParameters = new boolean[count]; 199 varargsParameters = new boolean[count]; 200 nonNullParameters = new boolean[count]; 201 int offset = hasReceiver(method) ? 1 : 0; 202 for (int i = offset; i < count; i++) { 203 constantParameters[i] = method.getParameterAnnotation(ConstantParameter.class, i - offset) != null; 204 varargsParameters[i] = method.getParameterAnnotation(VarargsParameter.class, i - offset) != null; 205 nonNullParameters[i] = method.getParameterAnnotation(NonNullParameter.class, i - offset) != null; 206 207 assert !constantParameters[i - offset] || !varargsParameters[i - offset] : "Parameter cannot be annotated with both @" + ConstantParameter.class.getSimpleName() + " and @" + 208 VarargsParameter.class.getSimpleName(); 209 } 210 if (method.hasReceiver()) { 211 // Receiver must be constant. 212 assert !constantParameters[0]; 213 constantParameters[0] = true; 214 } 215 216 // Retrieve the names only when assertions are turned on. Parameter annotations are 217 // unsupported in the native image. 218 assert IS_IN_NATIVE_IMAGE || initNames(method, count); 219 } 220 221 final boolean[] constantParameters; 222 final boolean[] varargsParameters; 223 final boolean[] nonNullParameters; 224 225 /** 226 * The parameter names, taken from the local variables table. Only used for assertion 227 * checking, so use only within an assert statement. 228 */ 229 String[] names; 230 231 private boolean initNames(ResolvedJavaMethod method, int parameterCount) { 232 names = new String[parameterCount]; 233 int offset = 0; 234 if (method.hasReceiver()) { 235 names[0] = "this"; 236 offset = 1; 237 } 238 Parameter[] params = method.getParameters(); 239 if (params != null) { 240 for (int i = offset; i < names.length; i++) { 241 if (params[i - offset].isNamePresent()) { 242 names[i] = params[i - offset].getName(); 243 } 244 } 245 } else { 246 int slotIdx = 0; 247 LocalVariableTable localVariableTable = method.getLocalVariableTable(); 248 if (localVariableTable != null) { 249 for (int i = 0; i < names.length; i++) { 250 Local local = localVariableTable.getLocal(slotIdx, 0); 251 if (local != null) { 252 names[i] = local.getName(); 253 } 254 JavaKind kind = method.getSignature().getParameterKind(i); 255 slotIdx += kind.getSlotCount(); 256 } 257 } 258 } 259 return true; 260 } 261 } 262 263 /** 264 * Times instantiations of all templates derived from this snippet. 265 */ 266 private final TimerKey instantiationTimer; 267 268 /** 269 * Counts instantiations of all templates derived from this snippet. 270 */ 271 private final CounterKey instantiationCounter; 272 273 protected abstract Lazy lazy(); 274 275 protected SnippetInfo(ResolvedJavaMethod method, ResolvedJavaMethod original, LocationIdentity[] privateLocations, Object receiver) { 276 this.method = method; 277 this.original = original; 278 this.privateLocations = privateLocations; 279 instantiationCounter = DebugContext.counter("SnippetInstantiationCount[%s]", method.getName()); 280 instantiationTimer = DebugContext.timer("SnippetInstantiationTime[%s]", method.getName()); 281 this.receiver = receiver; 282 } 283 284 public ResolvedJavaMethod getMethod() { 285 return method; 286 } 287 288 public int getParameterCount() { 289 return lazy().constantParameters.length; 290 } 291 292 public boolean isConstantParameter(int paramIdx) { 293 return lazy().constantParameters[paramIdx]; 294 } 295 296 public boolean isVarargsParameter(int paramIdx) { 297 return lazy().varargsParameters[paramIdx]; 298 } 299 300 public boolean isNonNullParameter(int paramIdx) { 301 return lazy().nonNullParameters[paramIdx]; 302 } 303 304 public String getParameterName(int paramIdx) { 305 String[] names = lazy().names; 306 if (names != null) { 307 return names[paramIdx]; 308 } 309 return null; 310 } 311 312 @Override 313 public String toString() { 314 return getClass().getSimpleName() + ":" + method.format("%h.%n"); 315 } 316 } 317 318 protected static class LazySnippetInfo extends SnippetInfo { 319 protected final AtomicReference<Lazy> lazy = new AtomicReference<>(null); 320 321 protected LazySnippetInfo(ResolvedJavaMethod method, ResolvedJavaMethod original, LocationIdentity[] privateLocations, Object receiver) { 322 super(method, original, privateLocations, receiver); 323 } 324 325 @Override 326 protected Lazy lazy() { 327 if (lazy.get() == null) { 328 lazy.compareAndSet(null, new Lazy(method)); 329 } 330 return lazy.get(); 331 } 332 } 333 334 protected static class EagerSnippetInfo extends SnippetInfo { 335 protected final Lazy lazy; 336 337 protected EagerSnippetInfo(ResolvedJavaMethod method, ResolvedJavaMethod original, LocationIdentity[] privateLocations, Object receiver) { 338 super(method, original, privateLocations, receiver); 339 lazy = new Lazy(method); 340 } 341 342 @Override 343 protected Lazy lazy() { 344 return lazy; 345 } 346 } 347 348 /** 349 * Values that are bound to the snippet method parameters. The methods {@link #add}, 350 * {@link #addConst}, and {@link #addVarargs} must be called in the same order as in the 351 * signature of the snippet method. The parameter name is passed to the add methods for 352 * assertion checking, i.e., to enforce that the order matches. Which method needs to be called 353 * depends on the annotation of the snippet method parameter: 354 * <ul> 355 * <li>Use {@link #add} for a parameter without an annotation. The value is bound when the 356 * {@link SnippetTemplate} is {@link SnippetTemplate#instantiate instantiated}. 357 * <li>Use {@link #addConst} for a parameter annotated with {@link ConstantParameter}. The value 358 * is bound when the {@link SnippetTemplate} is {@link SnippetTemplate#SnippetTemplate created}. 359 * <li>Use {@link #addVarargs} for an array parameter annotated with {@link VarargsParameter}. A 360 * separate {@link SnippetTemplate} is {@link SnippetTemplate#SnippetTemplate created} for every 361 * distinct array length. The actual values are bound when the {@link SnippetTemplate} is 362 * {@link SnippetTemplate#instantiate instantiated} 363 * </ul> 364 */ 365 public static class Arguments implements Formattable { 366 367 protected final SnippetInfo info; 368 protected final CacheKey cacheKey; 369 protected final Object[] values; 370 protected final Stamp[] constStamps; 371 protected boolean cacheable; 372 373 protected int nextParamIdx; 374 375 public Arguments(SnippetInfo info, GuardsStage guardsStage, LoweringTool.LoweringStage loweringStage) { 376 this.info = info; 377 this.cacheKey = new CacheKey(info, guardsStage, loweringStage); 378 this.values = new Object[info.getParameterCount()]; 379 this.constStamps = new Stamp[info.getParameterCount()]; 380 this.cacheable = true; 381 if (info.hasReceiver()) { 382 addConst("this", info.getReceiver()); 383 } 384 } 385 386 public Arguments add(String name, Object value) { 387 assert check(name, false, false); 388 values[nextParamIdx] = value; 389 nextParamIdx++; 390 return this; 391 } 392 393 public Arguments addConst(String name, Object value) { 394 assert value != null; 395 return addConst(name, value, null); 396 } 397 398 public Arguments addConst(String name, Object value, Stamp stamp) { 399 assert check(name, true, false); 400 values[nextParamIdx] = value; 401 constStamps[nextParamIdx] = stamp; 402 cacheKey.setParam(nextParamIdx, value); 403 nextParamIdx++; 404 return this; 405 } 406 407 public Arguments addVarargs(String name, Class<?> componentType, Stamp argStamp, Object value) { 408 assert check(name, false, true); 409 Varargs varargs = new Varargs(componentType, argStamp, value); 410 values[nextParamIdx] = varargs; 411 // A separate template is necessary for every distinct array length 412 cacheKey.setParam(nextParamIdx, varargs.length); 413 nextParamIdx++; 414 return this; 415 } 416 417 public void setCacheable(boolean cacheable) { 418 this.cacheable = cacheable; 419 } 420 421 private boolean check(String name, boolean constParam, boolean varargsParam) { 422 assert nextParamIdx < info.getParameterCount() : "too many parameters: " + name + " " + this; 423 assert info.getParameterName(nextParamIdx) == null || info.getParameterName(nextParamIdx).equals(name) : "wrong parameter name at " + nextParamIdx + " : " + name + " " + this; 424 assert constParam == info.isConstantParameter(nextParamIdx) : "Parameter " + (constParam ? "not " : "") + "annotated with @" + ConstantParameter.class.getSimpleName() + ": " + name + 425 " " + this; 426 assert varargsParam == info.isVarargsParameter(nextParamIdx) : "Parameter " + (varargsParam ? "not " : "") + "annotated with @" + VarargsParameter.class.getSimpleName() + ": " + name + 427 " " + this; 428 return true; 429 } 430 431 @Override 432 public String toString() { 433 StringBuilder result = new StringBuilder(); 434 result.append("Parameters<").append(info.method.format("%h.%n")).append(" ["); 435 String sep = ""; 436 for (int i = 0; i < info.getParameterCount(); i++) { 437 result.append(sep); 438 if (info.isConstantParameter(i)) { 439 result.append("const "); 440 } else if (info.isVarargsParameter(i)) { 441 result.append("varargs "); 442 } 443 result.append(info.getParameterName(i)).append(" = ").append(values[i]); 444 sep = ", "; 445 } 446 result.append(">"); 447 return result.toString(); 448 } 449 450 @Override 451 public void formatTo(Formatter formatter, int flags, int width, int precision) { 452 if ((flags & ALTERNATE) == 0) { 453 formatter.format(applyFormattingFlagsAndWidth(toString(), flags, width)); 454 } else { 455 StringBuilder sb = new StringBuilder(); 456 sb.append(info.method.getName()).append('('); 457 String sep = ""; 458 for (int i = 0; i < info.getParameterCount(); i++) { 459 if (info.isConstantParameter(i)) { 460 sb.append(sep); 461 if (info.getParameterName(i) != null) { 462 sb.append(info.getParameterName(i)); 463 } else { 464 sb.append(i); 465 } 466 sb.append('=').append(values[i]); 467 sep = ", "; 468 } 469 } 470 sb.append(")"); 471 String string = sb.toString(); 472 if (string.indexOf('%') != -1) { 473 // Quote any % signs 474 string = string.replace("%", "%%"); 475 } 476 formatter.format(applyFormattingFlagsAndWidth(string, flags & ~ALTERNATE, width)); 477 } 478 } 479 } 480 481 /** 482 * Wrapper for the prototype value of a {@linkplain VarargsParameter varargs} parameter. 483 */ 484 static class Varargs { 485 486 protected final Class<?> componentType; 487 protected final Stamp stamp; 488 protected final Object value; 489 protected final int length; 490 491 protected Varargs(Class<?> componentType, Stamp stamp, Object value) { 492 this.componentType = componentType; 493 this.stamp = stamp; 494 this.value = value; 495 if (value instanceof List) { 496 this.length = ((List<?>) value).size(); 497 } else { 498 this.length = Array.getLength(value); 499 } 500 } 501 502 @Override 503 public String toString() { 504 if (value instanceof boolean[]) { 505 return Arrays.toString((boolean[]) value); 506 } 507 if (value instanceof byte[]) { 508 return Arrays.toString((byte[]) value); 509 } 510 if (value instanceof char[]) { 511 return Arrays.toString((char[]) value); 512 } 513 if (value instanceof short[]) { 514 return Arrays.toString((short[]) value); 515 } 516 if (value instanceof int[]) { 517 return Arrays.toString((int[]) value); 518 } 519 if (value instanceof long[]) { 520 return Arrays.toString((long[]) value); 521 } 522 if (value instanceof float[]) { 523 return Arrays.toString((float[]) value); 524 } 525 if (value instanceof double[]) { 526 return Arrays.toString((double[]) value); 527 } 528 if (value instanceof Object[]) { 529 return Arrays.toString((Object[]) value); 530 } 531 return String.valueOf(value); 532 } 533 } 534 535 @NodeInfo(cycles = CYCLES_IGNORED, size = SIZE_IGNORED) 536 static final class VarargsPlaceholderNode extends FloatingNode implements ArrayLengthProvider { 537 538 public static final NodeClass<VarargsPlaceholderNode> TYPE = NodeClass.create(VarargsPlaceholderNode.class); 539 protected final Varargs varargs; 540 541 protected VarargsPlaceholderNode(Varargs varargs, MetaAccessProvider metaAccess) { 542 super(TYPE, StampFactory.objectNonNull(TypeReference.createExactTrusted(metaAccess.lookupJavaType(varargs.componentType).getArrayClass()))); 543 this.varargs = varargs; 544 } 545 546 @Override 547 public ValueNode findLength(FindLengthMode mode, ConstantReflectionProvider constantReflection) { 548 return ConstantNode.forInt(varargs.length); 549 } 550 } 551 552 static class CacheKey { 553 554 private final ResolvedJavaMethod method; 555 private final Object[] values; 556 private final GuardsStage guardsStage; 557 private final LoweringTool.LoweringStage loweringStage; 558 private int hash; 559 560 protected CacheKey(SnippetInfo info, GuardsStage guardsStage, LoweringTool.LoweringStage loweringStage) { 561 this.method = info.method; 562 this.guardsStage = guardsStage; 563 this.loweringStage = loweringStage; 564 this.values = new Object[info.getParameterCount()]; 565 this.hash = info.method.hashCode() + 31 * guardsStage.ordinal(); 566 } 567 568 protected void setParam(int paramIdx, Object value) { 569 values[paramIdx] = value; 570 hash = (hash * 31) ^ (value == null ? 0 : value.hashCode()); 571 } 572 573 @Override 574 public boolean equals(Object obj) { 575 if (!(obj instanceof CacheKey)) { 576 return false; 577 } 578 CacheKey other = (CacheKey) obj; 579 if (!method.equals(other.method)) { 580 return false; 581 } 582 if (guardsStage != other.guardsStage || loweringStage != other.loweringStage) { 583 return false; 584 } 585 for (int i = 0; i < values.length; i++) { 586 if (values[i] != null && !values[i].equals(other.values[i])) { 587 return false; 588 } 589 } 590 return true; 591 } 592 593 @Override 594 public int hashCode() { 595 return hash; 596 } 597 } 598 599 private static final TimerKey SnippetTemplateCreationTime = DebugContext.timer("SnippetTemplateCreationTime"); 600 private static final CounterKey SnippetTemplates = DebugContext.counter("SnippetTemplateCount"); 601 602 static class Options { 603 @Option(help = "Use a LRU cache for snippet templates.")// 604 public static final OptionKey<Boolean> UseSnippetTemplateCache = new OptionKey<>(true); 605 606 @Option(help = "")// 607 static final OptionKey<Integer> MaxTemplatesPerSnippet = new OptionKey<>(50); 608 } 609 610 /** 611 * Base class for snippet classes. It provides a cache for {@link SnippetTemplate}s. 612 */ 613 public abstract static class AbstractTemplates implements org.graalvm.compiler.api.replacements.SnippetTemplateCache { 614 615 protected final OptionValues options; 616 protected final Providers providers; 617 protected final SnippetReflectionProvider snippetReflection; 618 protected final Iterable<DebugHandlersFactory> factories; 619 protected final TargetDescription target; 620 private final Map<CacheKey, SnippetTemplate> templates; 621 622 protected AbstractTemplates(OptionValues options, Iterable<DebugHandlersFactory> factories, Providers providers, SnippetReflectionProvider snippetReflection, TargetDescription target) { 623 this.options = options; 624 this.providers = providers; 625 this.snippetReflection = snippetReflection; 626 this.target = target; 627 this.factories = factories; 628 if (Options.UseSnippetTemplateCache.getValue(options)) { 629 int size = Options.MaxTemplatesPerSnippet.getValue(options); 630 this.templates = Collections.synchronizedMap(new LRUCache<>(size, size)); 631 } else { 632 this.templates = null; 633 } 634 } 635 636 public static Method findMethod(Class<? extends Snippets> declaringClass, String methodName, Method except) { 637 for (Method m : declaringClass.getDeclaredMethods()) { 638 if (m.getName().equals(methodName) && !m.equals(except)) { 639 return m; 640 } 641 } 642 return null; 643 } 644 645 public static ResolvedJavaMethod findMethod(MetaAccessProvider metaAccess, Class<?> declaringClass, String methodName) { 646 ResolvedJavaType type = metaAccess.lookupJavaType(declaringClass); 647 ResolvedJavaMethod result = null; 648 for (ResolvedJavaMethod m : type.getDeclaredMethods()) { 649 if (m.getName().equals(methodName)) { 650 if (!Assertions.assertionsEnabled()) { 651 return m; 652 } else { 653 assert result == null : "multiple definitions found"; 654 result = m; 655 } 656 } 657 } 658 if (result == null) { 659 throw new GraalError("Could not find method in " + declaringClass + " named " + methodName); 660 } 661 return result; 662 } 663 664 protected SnippetInfo snippet(Class<? extends Snippets> declaringClass, String methodName, LocationIdentity... initialPrivateLocations) { 665 return snippet(declaringClass, methodName, null, null, initialPrivateLocations); 666 } 667 668 /** 669 * Finds the unique method in {@code declaringClass} named {@code methodName} annotated by 670 * {@link Snippet} and returns a {@link SnippetInfo} value describing it. There must be 671 * exactly one snippet method in {@code declaringClass} with a given name. 672 */ 673 protected SnippetInfo snippet(Class<? extends Snippets> declaringClass, String methodName, ResolvedJavaMethod original, Object receiver, LocationIdentity... initialPrivateLocations) { 674 assert methodName != null; 675 ResolvedJavaMethod javaMethod = findMethod(providers.getMetaAccess(), declaringClass, methodName); 676 assert javaMethod != null : "did not find @" + Snippet.class.getSimpleName() + " method in " + declaringClass + " named " + methodName; 677 assert javaMethod.getAnnotation(Snippet.class) != null : javaMethod + " must be annotated with @" + Snippet.class.getSimpleName(); 678 providers.getReplacements().registerSnippet(javaMethod, original, receiver, GraalOptions.TrackNodeSourcePosition.getValue(options)); 679 LocationIdentity[] privateLocations = GraalOptions.SnippetCounters.getValue(options) ? SnippetCounterNode.addSnippetCounters(initialPrivateLocations) : initialPrivateLocations; 680 if (GraalOptions.EagerSnippets.getValue(options)) { 681 return new EagerSnippetInfo(javaMethod, original, privateLocations, receiver); 682 } else { 683 return new LazySnippetInfo(javaMethod, original, privateLocations, receiver); 684 } 685 } 686 687 static final AtomicInteger nextSnippetTemplateId = new AtomicInteger(); 688 689 private DebugContext openDebugContext(DebugContext outer, Arguments args) { 690 if (DebugStubsAndSnippets.getValue(options)) { 691 Description description = new Description(args.cacheKey.method, "SnippetTemplate_" + nextSnippetTemplateId.incrementAndGet()); 692 return DebugContext.create(options, description, outer.getGlobalMetrics(), DEFAULT_LOG_STREAM, factories); 693 } 694 return DebugContext.DISABLED; 695 } 696 697 /** 698 * Gets a template for a given key, creating it first if necessary. 699 */ 700 @SuppressWarnings("try") 701 protected SnippetTemplate template(ValueNode replacee, final Arguments args) { 702 StructuredGraph graph = replacee.graph(); 703 DebugContext outer = graph.getDebug(); 704 SnippetTemplate template = Options.UseSnippetTemplateCache.getValue(options) && args.cacheable ? templates.get(args.cacheKey) : null; 705 if (template == null || (graph.trackNodeSourcePosition() && !template.snippet.trackNodeSourcePosition())) { 706 try (DebugContext debug = openDebugContext(outer, args)) { 707 try (DebugCloseable a = SnippetTemplateCreationTime.start(debug); DebugContext.Scope s = debug.scope("SnippetSpecialization", args.info.method)) { 708 SnippetTemplates.increment(debug); 709 OptionValues snippetOptions = new OptionValues(options, GraalOptions.TraceInlining, GraalOptions.TraceInliningForStubsAndSnippets.getValue(options)); 710 template = new SnippetTemplate(snippetOptions, debug, providers, snippetReflection, args, graph.trackNodeSourcePosition(), replacee); 711 if (Options.UseSnippetTemplateCache.getValue(snippetOptions) && args.cacheable) { 712 templates.put(args.cacheKey, template); 713 } 714 } catch (Throwable e) { 715 throw debug.handle(e); 716 } 717 } 718 } 719 return template; 720 } 721 } 722 723 private static final class LRUCache<K, V> extends LinkedHashMap<K, V> { 724 private static final long serialVersionUID = 1L; 725 private final int maxCacheSize; 726 727 LRUCache(int initialCapacity, int maxCacheSize) { 728 super(initialCapacity, 0.75F, true); 729 this.maxCacheSize = maxCacheSize; 730 } 731 732 @Override 733 protected boolean removeEldestEntry(java.util.Map.Entry<K, V> eldest) { 734 return size() > maxCacheSize; 735 } 736 } 737 738 // These values must be compared with equals() not '==' to support replay compilation. 739 private static final Object UNUSED_PARAMETER = "UNUSED_PARAMETER"; 740 private static final Object CONSTANT_PARAMETER = "CONSTANT_PARAMETER"; 741 742 /** 743 * Determines if any parameter of a given method is annotated with {@link ConstantParameter}. 744 */ 745 public static boolean hasConstantParameter(ResolvedJavaMethod method) { 746 for (ConstantParameter p : method.getParameterAnnotations(ConstantParameter.class)) { 747 if (p != null) { 748 return true; 749 } 750 } 751 return false; 752 } 753 754 private final SnippetReflectionProvider snippetReflection; 755 756 /** 757 * Creates a snippet template. 758 */ 759 @SuppressWarnings("try") 760 protected SnippetTemplate(OptionValues options, DebugContext debug, final Providers providers, SnippetReflectionProvider snippetReflection, Arguments args, boolean trackNodeSourcePosition, 761 Node replacee) { 762 this.snippetReflection = snippetReflection; 763 this.info = args.info; 764 765 Object[] constantArgs = getConstantArgs(args); 766 boolean shouldTrackNodeSourcePosition1 = trackNodeSourcePosition || (providers.getCodeCache() != null && providers.getCodeCache().shouldDebugNonSafepoints()); 767 StructuredGraph snippetGraph = providers.getReplacements().getSnippet(args.info.method, args.info.original, constantArgs, shouldTrackNodeSourcePosition1, replacee.getNodeSourcePosition()); 768 769 ResolvedJavaMethod method = snippetGraph.method(); 770 Signature signature = method.getSignature(); 771 772 PhaseContext phaseContext = new PhaseContext(providers); 773 774 // Copy snippet graph, replacing constant parameters with given arguments 775 final StructuredGraph snippetCopy = new StructuredGraph.Builder(options, debug).name(snippetGraph.name).method(snippetGraph.method()).trackNodeSourcePosition( 776 snippetGraph.trackNodeSourcePosition()).setIsSubstitution(true).build(); 777 assert !GraalOptions.TrackNodeSourcePosition.getValue(options) || snippetCopy.trackNodeSourcePosition(); 778 try (DebugContext.Scope scope = debug.scope("SpecializeSnippet", snippetCopy)) { 779 if (!snippetGraph.isUnsafeAccessTrackingEnabled()) { 780 snippetCopy.disableUnsafeAccessTracking(); 781 } 782 783 EconomicMap<Node, Node> nodeReplacements = EconomicMap.create(Equivalence.IDENTITY); 784 nodeReplacements.put(snippetGraph.start(), snippetCopy.start()); 785 786 MetaAccessProvider metaAccess = providers.getMetaAccess(); 787 assert checkTemplate(metaAccess, args, method, signature); 788 789 int parameterCount = args.info.getParameterCount(); 790 VarargsPlaceholderNode[] placeholders = new VarargsPlaceholderNode[parameterCount]; 791 792 for (int i = 0; i < parameterCount; i++) { 793 ParameterNode parameter = snippetGraph.getParameter(i); 794 if (parameter != null) { 795 if (args.info.isConstantParameter(i)) { 796 Object arg = args.values[i]; 797 JavaKind kind = signature.getParameterKind(i); 798 ConstantNode constantNode; 799 if (arg instanceof Constant) { 800 Stamp stamp = args.constStamps[i]; 801 if (stamp == null) { 802 assert arg instanceof JavaConstant : "could not determine type of constant " + arg; 803 constantNode = ConstantNode.forConstant((JavaConstant) arg, metaAccess, snippetCopy); 804 } else { 805 constantNode = ConstantNode.forConstant(stamp, (Constant) arg, metaAccess, snippetCopy); 806 } 807 } else { 808 constantNode = ConstantNode.forConstant(snippetReflection.forBoxed(kind, arg), metaAccess, snippetCopy); 809 } 810 nodeReplacements.put(parameter, constantNode); 811 } else if (args.info.isVarargsParameter(i)) { 812 Varargs varargs = (Varargs) args.values[i]; 813 VarargsPlaceholderNode placeholder = snippetCopy.unique(new VarargsPlaceholderNode(varargs, providers.getMetaAccess())); 814 nodeReplacements.put(parameter, placeholder); 815 placeholders[i] = placeholder; 816 } else if (args.info.isNonNullParameter(i)) { 817 parameter.setStamp(parameter.stamp(NodeView.DEFAULT).join(StampFactory.objectNonNull())); 818 } 819 } 820 } 821 try (InliningLog.UpdateScope updateScope = snippetCopy.getInliningLog().openDefaultUpdateScope()) { 822 UnmodifiableEconomicMap<Node, Node> duplicates = snippetCopy.addDuplicates(snippetGraph.getNodes(), snippetGraph, snippetGraph.getNodeCount(), nodeReplacements); 823 if (updateScope != null) { 824 snippetCopy.getInliningLog().replaceLog(duplicates, snippetGraph.getInliningLog()); 825 } 826 } 827 828 debug.dump(DebugContext.INFO_LEVEL, snippetCopy, "Before specialization"); 829 830 // Gather the template parameters 831 parameters = new Object[parameterCount]; 832 for (int i = 0; i < parameterCount; i++) { 833 if (args.info.isConstantParameter(i)) { 834 parameters[i] = CONSTANT_PARAMETER; 835 } else if (args.info.isVarargsParameter(i)) { 836 assert snippetCopy.getParameter(i) == null; 837 Varargs varargs = (Varargs) args.values[i]; 838 int length = varargs.length; 839 ParameterNode[] params = new ParameterNode[length]; 840 Stamp stamp = varargs.stamp; 841 for (int j = 0; j < length; j++) { 842 // Use a decimal friendly numbering make it more obvious how values map 843 assert parameterCount < 10000; 844 int idx = (i + 1) * 10000 + j; 845 assert idx >= parameterCount : "collision in parameter numbering"; 846 ParameterNode local = snippetCopy.addOrUnique(new ParameterNode(idx, StampPair.createSingle(stamp))); 847 params[j] = local; 848 } 849 parameters[i] = params; 850 851 VarargsPlaceholderNode placeholder = placeholders[i]; 852 if (placeholder != null) { 853 for (Node usage : placeholder.usages().snapshot()) { 854 if (usage instanceof LoadIndexedNode) { 855 LoadIndexedNode loadIndexed = (LoadIndexedNode) usage; 856 debug.dump(DebugContext.INFO_LEVEL, snippetCopy, "Before replacing %s", loadIndexed); 857 LoadSnippetVarargParameterNode loadSnippetParameter = snippetCopy.add( 858 new LoadSnippetVarargParameterNode(params, loadIndexed.index(), loadIndexed.stamp(NodeView.DEFAULT))); 859 snippetCopy.replaceFixedWithFixed(loadIndexed, loadSnippetParameter); 860 debug.dump(DebugContext.INFO_LEVEL, snippetCopy, "After replacing %s", loadIndexed); 861 } else if (usage instanceof StoreIndexedNode) { 862 /* 863 * The template lowering doesn't really treat this as an array so 864 * you can't store back into the varargs. Allocate your own array if 865 * you really need this and EA should eliminate it. 866 */ 867 throw new GraalError("Can't store into VarargsParameter array"); 868 } 869 } 870 } 871 } else { 872 ParameterNode local = snippetCopy.getParameter(i); 873 if (local == null) { 874 // Parameter value was eliminated 875 parameters[i] = UNUSED_PARAMETER; 876 } else { 877 parameters[i] = local; 878 } 879 } 880 } 881 882 explodeLoops(snippetCopy, phaseContext); 883 884 GuardsStage guardsStage = args.cacheKey.guardsStage; 885 // Perform lowering on the snippet 886 if (!guardsStage.allowsFloatingGuards()) { 887 new GuardLoweringPhase().apply(snippetCopy, null); 888 } 889 snippetCopy.setGuardsStage(guardsStage); 890 try (DebugContext.Scope s = debug.scope("LoweringSnippetTemplate", snippetCopy)) { 891 new LoweringPhase(new CanonicalizerPhase(), args.cacheKey.loweringStage).apply(snippetCopy, phaseContext); 892 } catch (Throwable e) { 893 throw debug.handle(e); 894 } 895 896 ArrayList<StateSplit> curSideEffectNodes = new ArrayList<>(); 897 ArrayList<DeoptimizingNode> curDeoptNodes = new ArrayList<>(); 898 ArrayList<ValueNode> curPlaceholderStampedNodes = new ArrayList<>(); 899 for (Node node : snippetCopy.getNodes()) { 900 if (node instanceof ValueNode) { 901 ValueNode valueNode = (ValueNode) node; 902 if (valueNode.stamp(NodeView.DEFAULT) == PlaceholderStamp.singleton()) { 903 curPlaceholderStampedNodes.add(valueNode); 904 } 905 } 906 907 if (node instanceof StateSplit) { 908 StateSplit stateSplit = (StateSplit) node; 909 FrameState frameState = stateSplit.stateAfter(); 910 if (stateSplit.hasSideEffect()) { 911 curSideEffectNodes.add((StateSplit) node); 912 } 913 if (frameState != null) { 914 stateSplit.setStateAfter(null); 915 } 916 } 917 if (node instanceof DeoptimizingNode) { 918 DeoptimizingNode deoptNode = (DeoptimizingNode) node; 919 if (deoptNode.canDeoptimize()) { 920 curDeoptNodes.add(deoptNode); 921 } 922 } 923 } 924 925 new DeadCodeEliminationPhase(Required).apply(snippetCopy); 926 927 assert checkAllVarargPlaceholdersAreDeleted(parameterCount, placeholders); 928 929 new FloatingReadPhase(true, true).apply(snippetCopy); 930 new RemoveValueProxyPhase().apply(snippetCopy); 931 932 MemoryAnchorNode anchor = snippetCopy.add(new MemoryAnchorNode()); 933 snippetCopy.start().replaceAtUsages(InputType.Memory, anchor); 934 935 this.snippet = snippetCopy; 936 937 StartNode entryPointNode = snippet.start(); 938 if (anchor.hasNoUsages()) { 939 anchor.safeDelete(); 940 this.memoryAnchor = null; 941 } else { 942 // Find out if all the return memory maps point to the anchor (i.e., there's no kill 943 // anywhere) 944 boolean needsMemoryMaps = false; 945 for (ReturnNode retNode : snippet.getNodes(ReturnNode.TYPE)) { 946 MemoryMapNode memoryMap = retNode.getMemoryMap(); 947 if (memoryMap.getLocations().size() > 1 || memoryMap.getLastLocationAccess(LocationIdentity.any()) != anchor) { 948 needsMemoryMaps = true; 949 break; 950 } 951 } 952 boolean needsAnchor; 953 if (needsMemoryMaps) { 954 needsAnchor = true; 955 } else { 956 // Check that all those memory maps where the only usages of the anchor 957 needsAnchor = anchor.usages().filter(isNotA(MemoryMapNode.class)).isNotEmpty(); 958 // Remove the useless memory map 959 MemoryMapNode memoryMap = null; 960 for (ReturnNode retNode : snippet.getNodes(ReturnNode.TYPE)) { 961 if (memoryMap == null) { 962 memoryMap = retNode.getMemoryMap(); 963 } else { 964 assert memoryMap == retNode.getMemoryMap(); 965 } 966 retNode.setMemoryMap(null); 967 } 968 memoryMap.safeDelete(); 969 } 970 if (needsAnchor) { 971 snippetCopy.addAfterFixed(snippetCopy.start(), anchor); 972 this.memoryAnchor = anchor; 973 } else { 974 anchor.safeDelete(); 975 this.memoryAnchor = null; 976 } 977 } 978 debug.dump(DebugContext.INFO_LEVEL, snippet, "SnippetTemplate after fixing memory anchoring"); 979 980 List<ReturnNode> returnNodes = snippet.getNodes(ReturnNode.TYPE).snapshot(); 981 if (returnNodes.isEmpty()) { 982 this.returnNode = null; 983 } else if (returnNodes.size() == 1) { 984 this.returnNode = returnNodes.get(0); 985 } else { 986 AbstractMergeNode merge = snippet.add(new MergeNode()); 987 List<MemoryMapNode> memMaps = new ArrayList<>(); 988 for (ReturnNode retNode : returnNodes) { 989 MemoryMapNode memoryMapNode = retNode.getMemoryMap(); 990 if (memoryMapNode != null) { 991 memMaps.add(memoryMapNode); 992 } 993 } 994 995 ValueNode returnValue = InliningUtil.mergeReturns(merge, returnNodes); 996 this.returnNode = snippet.add(new ReturnNode(returnValue)); 997 if (!memMaps.isEmpty()) { 998 MemoryMapImpl mmap = FloatingReadPhase.mergeMemoryMaps(merge, memMaps); 999 MemoryMapNode memoryMap = snippet.unique(new MemoryMapNode(mmap.getMap())); 1000 this.returnNode.setMemoryMap(memoryMap); 1001 for (MemoryMapNode mm : memMaps) { 1002 if (mm != memoryMap && mm.isAlive()) { 1003 assert mm.hasNoUsages(); 1004 GraphUtil.killWithUnusedFloatingInputs(mm); 1005 } 1006 } 1007 } 1008 merge.setNext(this.returnNode); 1009 } 1010 1011 assert verifyIntrinsicsProcessed(snippetCopy); 1012 1013 this.sideEffectNodes = curSideEffectNodes; 1014 this.deoptNodes = curDeoptNodes; 1015 this.placeholderStampedNodes = curPlaceholderStampedNodes; 1016 1017 nodes = new ArrayList<>(snippet.getNodeCount()); 1018 for (Node node : snippet.getNodes()) { 1019 if (node != entryPointNode && node != entryPointNode.stateAfter()) { 1020 nodes.add(node); 1021 } 1022 } 1023 1024 if (debug.areMetricsEnabled()) { 1025 DebugContext.counter("SnippetTemplateNodeCount[%#s]", args).add(debug, nodes.size()); 1026 } 1027 debug.dump(DebugContext.INFO_LEVEL, snippet, "SnippetTemplate final state"); 1028 this.snippet.freeze(); 1029 1030 } catch (Throwable ex) { 1031 throw debug.handle(ex); 1032 } 1033 } 1034 1035 private static boolean verifyIntrinsicsProcessed(StructuredGraph snippetCopy) { 1036 for (MethodCallTargetNode target : snippetCopy.getNodes(MethodCallTargetNode.TYPE)) { 1037 ResolvedJavaMethod targetMethod = target.targetMethod(); 1038 if (targetMethod != null) { 1039 assert targetMethod.getAnnotation(Fold.class) == null && targetMethod.getAnnotation(NodeIntrinsic.class) == null : "plugin should have been processed"; 1040 } 1041 } 1042 return true; 1043 } 1044 1045 public static void explodeLoops(final StructuredGraph snippetCopy, PhaseContext phaseContext) { 1046 // Do any required loop explosion 1047 boolean exploded = false; 1048 do { 1049 exploded = false; 1050 ExplodeLoopNode explodeLoop = snippetCopy.getNodes().filter(ExplodeLoopNode.class).first(); 1051 if (explodeLoop != null) { // Earlier canonicalization may have removed the loop 1052 // altogether 1053 LoopBeginNode loopBegin = explodeLoop.findLoopBegin(); 1054 if (loopBegin != null) { 1055 LoopEx loop = new LoopsData(snippetCopy).loop(loopBegin); 1056 Mark mark = snippetCopy.getMark(); 1057 LoopTransformations.fullUnroll(loop, phaseContext, new CanonicalizerPhase()); 1058 new CanonicalizerPhase().applyIncremental(snippetCopy, phaseContext, mark, false); 1059 loop.deleteUnusedNodes(); 1060 } 1061 GraphUtil.removeFixedWithUnusedInputs(explodeLoop); 1062 exploded = true; 1063 } 1064 } while (exploded); 1065 } 1066 1067 protected Object[] getConstantArgs(Arguments args) { 1068 Object[] constantArgs = args.values.clone(); 1069 for (int i = 0; i < args.info.getParameterCount(); i++) { 1070 if (!args.info.isConstantParameter(i)) { 1071 constantArgs[i] = null; 1072 } else { 1073 assert constantArgs[i] != null : "Can't pass raw null through as argument"; 1074 } 1075 } 1076 return constantArgs; 1077 } 1078 1079 private static boolean checkAllVarargPlaceholdersAreDeleted(int parameterCount, VarargsPlaceholderNode[] placeholders) { 1080 for (int i = 0; i < parameterCount; i++) { 1081 if (placeholders[i] != null) { 1082 assert placeholders[i].isDeleted() : placeholders[i]; 1083 } 1084 } 1085 return true; 1086 } 1087 1088 private static boolean checkConstantArgument(MetaAccessProvider metaAccess, final ResolvedJavaMethod method, Signature signature, int paramIndex, String name, Object arg, JavaKind kind) { 1089 ResolvedJavaType type = signature.getParameterType(paramIndex, method.getDeclaringClass()).resolve(method.getDeclaringClass()); 1090 if (metaAccess.lookupJavaType(WordBase.class).isAssignableFrom(type)) { 1091 assert arg instanceof JavaConstant : method + ": word constant parameters must be passed boxed in a Constant value: " + arg; 1092 return true; 1093 } 1094 if (kind != JavaKind.Object) { 1095 assert arg != null && kind.toBoxedJavaClass() == arg.getClass() : method + ": wrong value kind for " + name + ": expected " + kind + ", got " + 1096 (arg == null ? "null" : arg.getClass().getSimpleName()); 1097 } 1098 return true; 1099 } 1100 1101 private static boolean checkVarargs(MetaAccessProvider metaAccess, final ResolvedJavaMethod method, Signature signature, int i, String name, Varargs varargs) { 1102 ResolvedJavaType type = (ResolvedJavaType) signature.getParameterType(i, method.getDeclaringClass()); 1103 assert type.isArray() : "varargs parameter must be an array type"; 1104 assert type.getComponentType().isAssignableFrom(metaAccess.lookupJavaType(varargs.componentType)) : "componentType for " + name + " not matching " + type.toJavaName() + " instance: " + 1105 varargs.componentType; 1106 return true; 1107 } 1108 1109 /** 1110 * The graph built from the snippet method. 1111 */ 1112 private final StructuredGraph snippet; 1113 1114 private final SnippetInfo info; 1115 1116 /** 1117 * The named parameters of this template that must be bound to values during instantiation. For 1118 * a parameter that is still live after specialization, the value in this map is either a 1119 * {@link ParameterNode} instance or a {@link ParameterNode} array. For an eliminated parameter, 1120 * the value is identical to the key. 1121 */ 1122 private final Object[] parameters; 1123 1124 /** 1125 * The return node (if any) of the snippet. 1126 */ 1127 private final ReturnNode returnNode; 1128 1129 /** 1130 * The memory anchor (if any) of the snippet. 1131 */ 1132 private final MemoryAnchorNode memoryAnchor; 1133 1134 /** 1135 * Nodes that inherit the {@link StateSplit#stateAfter()} from the replacee during 1136 * instantiation. 1137 */ 1138 private final ArrayList<StateSplit> sideEffectNodes; 1139 1140 /** 1141 * Nodes that inherit a deoptimization {@link FrameState} from the replacee during 1142 * instantiation. 1143 */ 1144 private final ArrayList<DeoptimizingNode> deoptNodes; 1145 1146 /** 1147 * Nodes that have a stamp originating from a {@link Placeholder}. 1148 */ 1149 private final ArrayList<ValueNode> placeholderStampedNodes; 1150 1151 /** 1152 * The nodes to be inlined when this specialization is instantiated. 1153 */ 1154 private final ArrayList<Node> nodes; 1155 1156 /** 1157 * Gets the instantiation-time bindings to this template's parameters. 1158 * 1159 * @return the map that will be used to bind arguments to parameters when inlining this template 1160 */ 1161 private EconomicMap<Node, Node> bind(StructuredGraph replaceeGraph, MetaAccessProvider metaAccess, Arguments args) { 1162 EconomicMap<Node, Node> replacements = EconomicMap.create(Equivalence.IDENTITY); 1163 assert args.info.getParameterCount() == parameters.length : "number of args (" + args.info.getParameterCount() + ") != number of parameters (" + parameters.length + ")"; 1164 for (int i = 0; i < parameters.length; i++) { 1165 Object parameter = parameters[i]; 1166 assert parameter != null : this + " has no parameter named " + args.info.getParameterName(i); 1167 Object argument = args.values[i]; 1168 if (parameter instanceof ParameterNode) { 1169 if (argument instanceof ValueNode) { 1170 replacements.put((ParameterNode) parameter, (ValueNode) argument); 1171 } else { 1172 JavaKind kind = ((ParameterNode) parameter).getStackKind(); 1173 assert argument != null || kind == JavaKind.Object : this + " cannot accept null for non-object parameter named " + args.info.getParameterName(i); 1174 JavaConstant constant = forBoxed(argument, kind); 1175 replacements.put((ParameterNode) parameter, ConstantNode.forConstant(constant, metaAccess, replaceeGraph)); 1176 } 1177 } else if (parameter instanceof ParameterNode[]) { 1178 ParameterNode[] params = (ParameterNode[]) parameter; 1179 Varargs varargs = (Varargs) argument; 1180 int length = params.length; 1181 List<?> list = null; 1182 Object array = null; 1183 if (varargs.value instanceof List) { 1184 list = (List<?>) varargs.value; 1185 assert list.size() == length : length + " != " + list.size(); 1186 } else { 1187 array = varargs.value; 1188 assert array != null && array.getClass().isArray(); 1189 assert Array.getLength(array) == length : length + " != " + Array.getLength(array); 1190 } 1191 1192 for (int j = 0; j < length; j++) { 1193 ParameterNode param = params[j]; 1194 assert param != null; 1195 Object value = list != null ? list.get(j) : Array.get(array, j); 1196 if (value instanceof ValueNode) { 1197 replacements.put(param, (ValueNode) value); 1198 } else { 1199 JavaConstant constant = forBoxed(value, param.getStackKind()); 1200 ConstantNode element = ConstantNode.forConstant(constant, metaAccess, replaceeGraph); 1201 replacements.put(param, element); 1202 } 1203 } 1204 } else { 1205 assert parameter.equals(CONSTANT_PARAMETER) || parameter.equals(UNUSED_PARAMETER) : "unexpected entry for parameter: " + args.info.getParameterName(i) + " -> " + parameter; 1206 } 1207 } 1208 return replacements; 1209 } 1210 1211 /** 1212 * Converts a Java boxed value to a {@link JavaConstant} of the right kind. This adjusts for the 1213 * limitation that a {@link Local}'s kind is a {@linkplain JavaKind#getStackKind() stack kind} 1214 * and so cannot be used for re-boxing primitives smaller than an int. 1215 * 1216 * @param argument a Java boxed value 1217 * @param localKind the kind of the {@link Local} to which {@code argument} will be bound 1218 */ 1219 protected JavaConstant forBoxed(Object argument, JavaKind localKind) { 1220 assert localKind == localKind.getStackKind(); 1221 if (localKind == JavaKind.Int) { 1222 return JavaConstant.forBoxedPrimitive(argument); 1223 } 1224 return snippetReflection.forBoxed(localKind, argument); 1225 } 1226 1227 /** 1228 * Logic for replacing a snippet-lowered node at its usages with the return value of the 1229 * snippet. An alternative to the {@linkplain SnippetTemplate#DEFAULT_REPLACER default} 1230 * replacement logic can be used to handle mismatches between the stamp of the node being 1231 * lowered and the stamp of the snippet's return value. 1232 */ 1233 public interface UsageReplacer { 1234 /** 1235 * Replaces all usages of {@code oldNode} with direct or indirect usages of {@code newNode}. 1236 */ 1237 void replace(ValueNode oldNode, ValueNode newNode); 1238 } 1239 1240 /** 1241 * Represents the default {@link UsageReplacer usage replacer} logic which simply delegates to 1242 * {@link Node#replaceAtUsages(Node)}. 1243 */ 1244 public static final UsageReplacer DEFAULT_REPLACER = new UsageReplacer() { 1245 1246 @Override 1247 public void replace(ValueNode oldNode, ValueNode newNode) { 1248 if (newNode == null) { 1249 assert oldNode.hasNoUsages(); 1250 } else { 1251 oldNode.replaceAtUsages(newNode); 1252 } 1253 } 1254 }; 1255 1256 private boolean assertSnippetKills(ValueNode replacee) { 1257 if (!replacee.graph().isAfterFloatingReadPhase()) { 1258 // no floating reads yet, ignore locations created while lowering 1259 return true; 1260 } 1261 if (returnNode == null) { 1262 // The snippet terminates control flow 1263 return true; 1264 } 1265 MemoryMapNode memoryMap = returnNode.getMemoryMap(); 1266 if (memoryMap == null || memoryMap.isEmpty()) { 1267 // there are no kills in the snippet graph 1268 return true; 1269 } 1270 1271 EconomicSet<LocationIdentity> kills = EconomicSet.create(Equivalence.DEFAULT); 1272 kills.addAll(memoryMap.getLocations()); 1273 1274 if (replacee instanceof MemoryCheckpoint.Single) { 1275 // check if some node in snippet graph also kills the same location 1276 LocationIdentity locationIdentity = ((MemoryCheckpoint.Single) replacee).getLocationIdentity(); 1277 if (locationIdentity.isAny()) { 1278 assert !(memoryMap.getLastLocationAccess(any()) instanceof MemoryAnchorNode) : replacee + " kills ANY_LOCATION, but snippet does not"; 1279 // if the replacee kills ANY_LOCATION, the snippet can kill arbitrary locations 1280 return true; 1281 } 1282 assert kills.contains(locationIdentity) : replacee + " kills " + locationIdentity + ", but snippet doesn't contain a kill to this location"; 1283 kills.remove(locationIdentity); 1284 } 1285 assert !(replacee instanceof MemoryCheckpoint.Multi) : replacee + " multi not supported (yet)"; 1286 1287 // remove ANY_LOCATION if it's just a kill by the start node 1288 if (memoryMap.getLastLocationAccess(any()) instanceof MemoryAnchorNode) { 1289 kills.remove(any()); 1290 } 1291 1292 // node can only lower to a ANY_LOCATION kill if the replacee also kills ANY_LOCATION 1293 assert !kills.contains(any()) : "snippet graph contains a kill to ANY_LOCATION, but replacee (" + replacee + ") doesn't kill ANY_LOCATION. kills: " + kills; 1294 1295 /* 1296 * Kills to private locations are safe, since there can be no floating read to these 1297 * locations except reads that are introduced by the snippet itself or related snippets in 1298 * the same lowering round. These reads are anchored to a MemoryAnchor at the beginning of 1299 * their snippet, so they can not float above a kill in another instance of the same 1300 * snippet. 1301 */ 1302 for (LocationIdentity p : this.info.privateLocations) { 1303 kills.remove(p); 1304 } 1305 1306 assert kills.isEmpty() : "snippet graph kills non-private locations " + kills + " that replacee (" + replacee + ") doesn't kill"; 1307 return true; 1308 } 1309 1310 private static class MemoryInputMap implements MemoryMap { 1311 1312 private final LocationIdentity locationIdentity; 1313 private final MemoryNode lastLocationAccess; 1314 1315 MemoryInputMap(ValueNode replacee) { 1316 if (replacee instanceof MemoryAccess) { 1317 MemoryAccess access = (MemoryAccess) replacee; 1318 locationIdentity = access.getLocationIdentity(); 1319 lastLocationAccess = access.getLastLocationAccess(); 1320 } else { 1321 locationIdentity = null; 1322 lastLocationAccess = null; 1323 } 1324 } 1325 1326 @Override 1327 public MemoryNode getLastLocationAccess(LocationIdentity location) { 1328 if (locationIdentity != null && locationIdentity.equals(location)) { 1329 return lastLocationAccess; 1330 } else { 1331 return null; 1332 } 1333 } 1334 1335 @Override 1336 public Collection<LocationIdentity> getLocations() { 1337 if (locationIdentity == null) { 1338 return Collections.emptySet(); 1339 } else { 1340 return Collections.singleton(locationIdentity); 1341 } 1342 } 1343 } 1344 1345 private class MemoryOutputMap extends MemoryInputMap { 1346 1347 private final UnmodifiableEconomicMap<Node, Node> duplicates; 1348 1349 MemoryOutputMap(ValueNode replacee, UnmodifiableEconomicMap<Node, Node> duplicates) { 1350 super(replacee); 1351 this.duplicates = duplicates; 1352 } 1353 1354 @Override 1355 public MemoryNode getLastLocationAccess(LocationIdentity locationIdentity) { 1356 MemoryMapNode memoryMap = returnNode.getMemoryMap(); 1357 assert memoryMap != null : "no memory map stored for this snippet graph (snippet doesn't have a ReturnNode?)"; 1358 MemoryNode lastLocationAccess = memoryMap.getLastLocationAccess(locationIdentity); 1359 assert lastLocationAccess != null : locationIdentity; 1360 if (lastLocationAccess == memoryAnchor) { 1361 return super.getLastLocationAccess(locationIdentity); 1362 } else { 1363 return (MemoryNode) duplicates.get(ValueNodeUtil.asNode(lastLocationAccess)); 1364 } 1365 } 1366 1367 @Override 1368 public Collection<LocationIdentity> getLocations() { 1369 return returnNode.getMemoryMap().getLocations(); 1370 } 1371 } 1372 1373 private void rewireMemoryGraph(ValueNode replacee, UnmodifiableEconomicMap<Node, Node> duplicates) { 1374 if (replacee.graph().isAfterFloatingReadPhase()) { 1375 // rewire outgoing memory edges 1376 replaceMemoryUsages(replacee, new MemoryOutputMap(replacee, duplicates)); 1377 1378 if (returnNode != null) { 1379 ReturnNode ret = (ReturnNode) duplicates.get(returnNode); 1380 if (ret != null) { 1381 MemoryMapNode memoryMap = ret.getMemoryMap(); 1382 if (memoryMap != null) { 1383 ret.setMemoryMap(null); 1384 memoryMap.safeDelete(); 1385 } 1386 } 1387 } 1388 if (memoryAnchor != null) { 1389 // rewire incoming memory edges 1390 MemoryAnchorNode memoryDuplicate = (MemoryAnchorNode) duplicates.get(memoryAnchor); 1391 replaceMemoryUsages(memoryDuplicate, new MemoryInputMap(replacee)); 1392 1393 if (memoryDuplicate.hasNoUsages()) { 1394 if (memoryDuplicate.next() != null) { 1395 memoryDuplicate.graph().removeFixed(memoryDuplicate); 1396 } else { 1397 // this was a dummy memory node used when instantiating pure data-flow 1398 // snippets: it was not attached to the control flow. 1399 memoryDuplicate.safeDelete(); 1400 } 1401 } 1402 } 1403 } 1404 } 1405 1406 private static LocationIdentity getLocationIdentity(Node node) { 1407 if (node instanceof MemoryAccess) { 1408 return ((MemoryAccess) node).getLocationIdentity(); 1409 } else if (node instanceof MemoryProxy) { 1410 return ((MemoryProxy) node).getLocationIdentity(); 1411 } else if (node instanceof MemoryPhiNode) { 1412 return ((MemoryPhiNode) node).getLocationIdentity(); 1413 } else { 1414 return null; 1415 } 1416 } 1417 1418 private void replaceMemoryUsages(ValueNode node, MemoryMap map) { 1419 for (Node usage : node.usages().snapshot()) { 1420 if (usage instanceof MemoryMapNode) { 1421 continue; 1422 } 1423 1424 LocationIdentity location = getLocationIdentity(usage); 1425 if (location != null) { 1426 for (Position pos : usage.inputPositions()) { 1427 if (pos.getInputType() == InputType.Memory && pos.get(usage) == node) { 1428 MemoryNode replacement = map.getLastLocationAccess(location); 1429 if (replacement == null) { 1430 assert mayRemoveLocation || LocationIdentity.any().equals(location) || 1431 CollectionsUtil.anyMatch(info.privateLocations, Predicate.isEqual(location)) : "Snippet " + 1432 info.method.format("%h.%n") + " contains access to the non-private location " + 1433 location + ", but replacee doesn't access this location." + map.getLocations(); 1434 } else { 1435 pos.set(usage, replacement.asNode()); 1436 } 1437 } 1438 } 1439 } 1440 } 1441 } 1442 1443 /** 1444 * Replaces a given fixed node with this specialized snippet. 1445 * 1446 * @param metaAccess 1447 * @param replacee the node that will be replaced 1448 * @param replacer object that replaces the usages of {@code replacee} 1449 * @param args the arguments to be bound to the flattened positional parameters of the snippet 1450 * @return the map of duplicated nodes (original -> duplicate) 1451 */ 1452 @SuppressWarnings("try") 1453 public UnmodifiableEconomicMap<Node, Node> instantiate(MetaAccessProvider metaAccess, FixedNode replacee, UsageReplacer replacer, Arguments args) { 1454 return instantiate(metaAccess, replacee, replacer, args, true); 1455 } 1456 1457 /** 1458 * Replaces a given fixed node with this specialized snippet. 1459 * 1460 * @param metaAccess 1461 * @param replacee the node that will be replaced 1462 * @param replacer object that replaces the usages of {@code replacee} 1463 * @param args the arguments to be bound to the flattened positional parameters of the snippet 1464 * @param killReplacee is true, the replacee node is deleted 1465 * @return the map of duplicated nodes (original -> duplicate) 1466 */ 1467 @SuppressWarnings("try") 1468 public UnmodifiableEconomicMap<Node, Node> instantiate(MetaAccessProvider metaAccess, FixedNode replacee, UsageReplacer replacer, Arguments args, boolean killReplacee) { 1469 DebugContext debug = replacee.getDebug(); 1470 assert assertSnippetKills(replacee); 1471 try (DebugCloseable a = args.info.instantiationTimer.start(debug)) { 1472 args.info.instantiationCounter.increment(debug); 1473 // Inline the snippet nodes, replacing parameters with the given args in the process 1474 StartNode entryPointNode = snippet.start(); 1475 FixedNode firstCFGNode = entryPointNode.next(); 1476 StructuredGraph replaceeGraph = replacee.graph(); 1477 EconomicMap<Node, Node> replacements = bind(replaceeGraph, metaAccess, args); 1478 replacements.put(entryPointNode, AbstractBeginNode.prevBegin(replacee)); 1479 UnmodifiableEconomicMap<Node, Node> duplicates = inlineSnippet(replacee, debug, replaceeGraph, replacements); 1480 1481 // Re-wire the control flow graph around the replacee 1482 FixedNode firstCFGNodeDuplicate = (FixedNode) duplicates.get(firstCFGNode); 1483 replacee.replaceAtPredecessor(firstCFGNodeDuplicate); 1484 1485 rewireFrameStates(replacee, duplicates); 1486 1487 if (replacee instanceof DeoptimizingNode) { 1488 DeoptimizingNode replaceeDeopt = (DeoptimizingNode) replacee; 1489 1490 FrameState stateBefore = null; 1491 FrameState stateDuring = null; 1492 FrameState stateAfter = null; 1493 if (replaceeDeopt.canDeoptimize()) { 1494 if (replaceeDeopt instanceof DeoptimizingNode.DeoptBefore) { 1495 stateBefore = ((DeoptimizingNode.DeoptBefore) replaceeDeopt).stateBefore(); 1496 } 1497 if (replaceeDeopt instanceof DeoptimizingNode.DeoptDuring) { 1498 stateDuring = ((DeoptimizingNode.DeoptDuring) replaceeDeopt).stateDuring(); 1499 } 1500 if (replaceeDeopt instanceof DeoptimizingNode.DeoptAfter) { 1501 stateAfter = ((DeoptimizingNode.DeoptAfter) replaceeDeopt).stateAfter(); 1502 } 1503 } 1504 1505 for (DeoptimizingNode deoptNode : deoptNodes) { 1506 DeoptimizingNode deoptDup = (DeoptimizingNode) duplicates.get(deoptNode.asNode()); 1507 if (deoptDup.canDeoptimize()) { 1508 if (deoptDup instanceof DeoptimizingNode.DeoptBefore) { 1509 ((DeoptimizingNode.DeoptBefore) deoptDup).setStateBefore(stateBefore); 1510 } 1511 if (deoptDup instanceof DeoptimizingNode.DeoptDuring) { 1512 DeoptimizingNode.DeoptDuring deoptDupDuring = (DeoptimizingNode.DeoptDuring) deoptDup; 1513 if (stateDuring != null) { 1514 deoptDupDuring.setStateDuring(stateDuring); 1515 } else if (stateAfter != null) { 1516 deoptDupDuring.computeStateDuring(stateAfter); 1517 } else if (stateBefore != null) { 1518 assert !deoptDupDuring.hasSideEffect() : "can't use stateBefore as stateDuring for state split " + deoptDupDuring; 1519 deoptDupDuring.setStateDuring(stateBefore); 1520 } 1521 } 1522 if (deoptDup instanceof DeoptimizingNode.DeoptAfter) { 1523 DeoptimizingNode.DeoptAfter deoptDupAfter = (DeoptimizingNode.DeoptAfter) deoptDup; 1524 if (stateAfter != null) { 1525 deoptDupAfter.setStateAfter(stateAfter); 1526 } else { 1527 assert !deoptDupAfter.hasSideEffect() : "can't use stateBefore as stateAfter for state split " + deoptDupAfter; 1528 deoptDupAfter.setStateAfter(stateBefore); 1529 } 1530 1531 } 1532 } 1533 } 1534 } 1535 1536 updateStamps(replacee, duplicates); 1537 1538 rewireMemoryGraph(replacee, duplicates); 1539 1540 // Replace all usages of the replacee with the value returned by the snippet 1541 ValueNode returnValue = null; 1542 if (returnNode != null && !(replacee instanceof ControlSinkNode)) { 1543 ReturnNode returnDuplicate = (ReturnNode) duplicates.get(returnNode); 1544 returnValue = returnDuplicate.result(); 1545 if (returnValue == null && replacee.usages().isNotEmpty() && replacee instanceof MemoryCheckpoint) { 1546 replacer.replace(replacee, null); 1547 } else { 1548 assert returnValue != null || replacee.hasNoUsages(); 1549 replacer.replace(replacee, returnValue); 1550 } 1551 if (returnDuplicate.isAlive()) { 1552 FixedNode next = null; 1553 if (replacee instanceof FixedWithNextNode) { 1554 FixedWithNextNode fwn = (FixedWithNextNode) replacee; 1555 next = fwn.next(); 1556 fwn.setNext(null); 1557 } 1558 returnDuplicate.replaceAndDelete(next); 1559 } 1560 } 1561 1562 if (killReplacee) { 1563 // Remove the replacee from its graph 1564 GraphUtil.killCFG(replacee); 1565 } 1566 1567 debug.dump(DebugContext.DETAILED_LEVEL, replaceeGraph, "After lowering %s with %s", replacee, this); 1568 return duplicates; 1569 } 1570 } 1571 1572 private UnmodifiableEconomicMap<Node, Node> inlineSnippet(Node replacee, DebugContext debug, StructuredGraph replaceeGraph, EconomicMap<Node, Node> replacements) { 1573 Mark mark = replaceeGraph.getMark(); 1574 try (InliningLog.UpdateScope scope = replaceeGraph.getInliningLog().openUpdateScope((oldNode, newNode) -> { 1575 InliningLog log = replaceeGraph.getInliningLog(); 1576 if (oldNode == null) { 1577 log.trackNewCallsite(newNode); 1578 } 1579 })) { 1580 UnmodifiableEconomicMap<Node, Node> duplicates = replaceeGraph.addDuplicates(nodes, snippet, snippet.getNodeCount(), replacements); 1581 if (scope != null) { 1582 replaceeGraph.getInliningLog().addLog(duplicates, snippet.getInliningLog()); 1583 } 1584 NodeSourcePosition position = replacee.getNodeSourcePosition(); 1585 InliningUtil.updateSourcePosition(replaceeGraph, duplicates, mark, position, true); 1586 debug.dump(DebugContext.DETAILED_LEVEL, replaceeGraph, "After inlining snippet %s", snippet.method()); 1587 return duplicates; 1588 } 1589 } 1590 1591 private void propagateStamp(Node node) { 1592 if (node instanceof PhiNode) { 1593 PhiNode phi = (PhiNode) node; 1594 if (phi.inferStamp()) { 1595 for (Node usage : node.usages()) { 1596 propagateStamp(usage); 1597 } 1598 } 1599 } 1600 } 1601 1602 private void updateStamps(ValueNode replacee, UnmodifiableEconomicMap<Node, Node> duplicates) { 1603 for (ValueNode node : placeholderStampedNodes) { 1604 ValueNode dup = (ValueNode) duplicates.get(node); 1605 Stamp replaceeStamp = replacee.stamp(NodeView.DEFAULT); 1606 if (node instanceof Placeholder) { 1607 Placeholder placeholderDup = (Placeholder) dup; 1608 placeholderDup.makeReplacement(replaceeStamp); 1609 } else { 1610 dup.setStamp(replaceeStamp); 1611 } 1612 } 1613 for (ParameterNode paramNode : snippet.getNodes(ParameterNode.TYPE)) { 1614 for (Node usage : paramNode.usages()) { 1615 Node usageDup = duplicates.get(usage); 1616 propagateStamp(usageDup); 1617 } 1618 } 1619 } 1620 1621 /** 1622 * Gets a copy of the specialized graph. 1623 */ 1624 public StructuredGraph copySpecializedGraph(DebugContext debugForCopy) { 1625 return (StructuredGraph) snippet.copy(debugForCopy); 1626 } 1627 1628 /** 1629 * Replaces a given floating node with this specialized snippet. 1630 * 1631 * @param metaAccess 1632 * @param replacee the node that will be replaced 1633 * @param replacer object that replaces the usages of {@code replacee} 1634 * @param tool lowering tool used to insert the snippet into the control-flow 1635 * @param args the arguments to be bound to the flattened positional parameters of the snippet 1636 */ 1637 @SuppressWarnings("try") 1638 public void instantiate(MetaAccessProvider metaAccess, FloatingNode replacee, UsageReplacer replacer, LoweringTool tool, Arguments args) { 1639 DebugContext debug = replacee.getDebug(); 1640 assert assertSnippetKills(replacee); 1641 try (DebugCloseable a = args.info.instantiationTimer.start(debug)) { 1642 args.info.instantiationCounter.increment(debug); 1643 1644 // Inline the snippet nodes, replacing parameters with the given args in the process 1645 StartNode entryPointNode = snippet.start(); 1646 FixedNode firstCFGNode = entryPointNode.next(); 1647 StructuredGraph replaceeGraph = replacee.graph(); 1648 EconomicMap<Node, Node> replacements = bind(replaceeGraph, metaAccess, args); 1649 replacements.put(entryPointNode, tool.getCurrentGuardAnchor().asNode()); 1650 UnmodifiableEconomicMap<Node, Node> duplicates = inlineSnippet(replacee, debug, replaceeGraph, replacements); 1651 1652 FixedWithNextNode lastFixedNode = tool.lastFixedNode(); 1653 assert lastFixedNode != null && lastFixedNode.isAlive() : replaceeGraph + " lastFixed=" + lastFixedNode; 1654 FixedNode next = lastFixedNode.next(); 1655 lastFixedNode.setNext(null); 1656 FixedNode firstCFGNodeDuplicate = (FixedNode) duplicates.get(firstCFGNode); 1657 replaceeGraph.addAfterFixed(lastFixedNode, firstCFGNodeDuplicate); 1658 1659 rewireFrameStates(replacee, duplicates); 1660 updateStamps(replacee, duplicates); 1661 1662 rewireMemoryGraph(replacee, duplicates); 1663 1664 // Replace all usages of the replacee with the value returned by the snippet 1665 ReturnNode returnDuplicate = (ReturnNode) duplicates.get(returnNode); 1666 ValueNode returnValue = returnDuplicate.result(); 1667 assert returnValue != null || replacee.hasNoUsages(); 1668 replacer.replace(replacee, returnValue); 1669 1670 if (returnDuplicate.isAlive()) { 1671 returnDuplicate.replaceAndDelete(next); 1672 } 1673 1674 debug.dump(DebugContext.DETAILED_LEVEL, replaceeGraph, "After lowering %s with %s", replacee, this); 1675 } 1676 } 1677 1678 /** 1679 * Replaces a given floating node with this specialized snippet. 1680 * 1681 * This snippet must be pure data-flow 1682 * 1683 * @param metaAccess 1684 * @param replacee the node that will be replaced 1685 * @param replacer object that replaces the usages of {@code replacee} 1686 * @param args the arguments to be bound to the flattened positional parameters of the snippet 1687 */ 1688 @SuppressWarnings("try") 1689 public void instantiate(MetaAccessProvider metaAccess, FloatingNode replacee, UsageReplacer replacer, Arguments args) { 1690 DebugContext debug = replacee.getDebug(); 1691 assert assertSnippetKills(replacee); 1692 try (DebugCloseable a = args.info.instantiationTimer.start(debug)) { 1693 args.info.instantiationCounter.increment(debug); 1694 1695 // Inline the snippet nodes, replacing parameters with the given args in the process 1696 StartNode entryPointNode = snippet.start(); 1697 assert entryPointNode.next() == (memoryAnchor == null ? returnNode : memoryAnchor) : entryPointNode.next(); 1698 StructuredGraph replaceeGraph = replacee.graph(); 1699 EconomicMap<Node, Node> replacements = bind(replaceeGraph, metaAccess, args); 1700 MemoryAnchorNode anchorDuplicate = null; 1701 if (memoryAnchor != null) { 1702 anchorDuplicate = replaceeGraph.add(new MemoryAnchorNode()); 1703 replacements.put(memoryAnchor, anchorDuplicate); 1704 } 1705 List<Node> floatingNodes = new ArrayList<>(nodes.size() - 2); 1706 for (Node n : nodes) { 1707 if (n != entryPointNode && n != returnNode) { 1708 floatingNodes.add(n); 1709 } 1710 } 1711 UnmodifiableEconomicMap<Node, Node> duplicates = inlineSnippet(replacee, debug, replaceeGraph, replacements); 1712 1713 rewireFrameStates(replacee, duplicates); 1714 updateStamps(replacee, duplicates); 1715 1716 rewireMemoryGraph(replacee, duplicates); 1717 assert anchorDuplicate == null || anchorDuplicate.isDeleted(); 1718 1719 // Replace all usages of the replacee with the value returned by the snippet 1720 ValueNode returnValue = (ValueNode) duplicates.get(returnNode.result()); 1721 replacer.replace(replacee, returnValue); 1722 1723 debug.dump(DebugContext.DETAILED_LEVEL, replaceeGraph, "After lowering %s with %s", replacee, this); 1724 } 1725 } 1726 1727 protected void rewireFrameStates(ValueNode replacee, UnmodifiableEconomicMap<Node, Node> duplicates) { 1728 if (replacee instanceof StateSplit) { 1729 for (StateSplit sideEffectNode : sideEffectNodes) { 1730 assert ((StateSplit) replacee).hasSideEffect(); 1731 Node sideEffectDup = duplicates.get(sideEffectNode.asNode()); 1732 ((StateSplit) sideEffectDup).setStateAfter(((StateSplit) replacee).stateAfter()); 1733 } 1734 } 1735 } 1736 1737 @Override 1738 public String toString() { 1739 StringBuilder buf = new StringBuilder(snippet.toString()).append('('); 1740 String sep = ""; 1741 for (int i = 0; i < parameters.length; i++) { 1742 String name = "[" + i + "]"; 1743 Object value = parameters[i]; 1744 buf.append(sep); 1745 sep = ", "; 1746 if (value == null) { 1747 buf.append("<null> ").append(name); 1748 } else if (value.equals(UNUSED_PARAMETER)) { 1749 buf.append("<unused> ").append(name); 1750 } else if (value.equals(CONSTANT_PARAMETER)) { 1751 buf.append("<constant> ").append(name); 1752 } else if (value instanceof ParameterNode) { 1753 ParameterNode param = (ParameterNode) value; 1754 buf.append(param.getStackKind().getJavaName()).append(' ').append(name); 1755 } else { 1756 ParameterNode[] params = (ParameterNode[]) value; 1757 String kind = params.length == 0 ? "?" : params[0].getStackKind().getJavaName(); 1758 buf.append(kind).append('[').append(params.length).append("] ").append(name); 1759 } 1760 } 1761 return buf.append(')').toString(); 1762 } 1763 1764 private static boolean checkTemplate(MetaAccessProvider metaAccess, Arguments args, ResolvedJavaMethod method, Signature signature) { 1765 int offset = args.info.hasReceiver() ? 1 : 0; 1766 for (int i = offset; i < args.info.getParameterCount(); i++) { 1767 if (args.info.isConstantParameter(i)) { 1768 JavaKind kind = signature.getParameterKind(i - offset); 1769 assert checkConstantArgument(metaAccess, method, signature, i - offset, args.info.getParameterName(i), args.values[i], kind); 1770 1771 } else if (args.info.isVarargsParameter(i)) { 1772 assert args.values[i] instanceof Varargs; 1773 Varargs varargs = (Varargs) args.values[i]; 1774 assert checkVarargs(metaAccess, method, signature, i, args.info.getParameterName(i), varargs); 1775 } 1776 } 1777 return true; 1778 } 1779 1780 public void setMayRemoveLocation(boolean mayRemoveLocation) { 1781 this.mayRemoveLocation = mayRemoveLocation; 1782 } 1783 }