1 /* 2 * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 25 package org.graalvm.compiler.phases.common.inlining; 26 27 import static jdk.vm.ci.meta.DeoptimizationAction.InvalidateReprofile; 28 import static jdk.vm.ci.meta.DeoptimizationReason.NullCheckException; 29 import static org.graalvm.compiler.core.common.GraalOptions.HotSpotPrintInlining; 30 31 import java.lang.reflect.Constructor; 32 import java.util.ArrayDeque; 33 import java.util.ArrayList; 34 import java.util.List; 35 import java.util.Objects; 36 import java.util.function.Consumer; 37 38 import jdk.internal.vm.compiler.collections.EconomicMap; 39 import jdk.internal.vm.compiler.collections.EconomicSet; 40 import jdk.internal.vm.compiler.collections.Equivalence; 41 import jdk.internal.vm.compiler.collections.UnmodifiableEconomicMap; 42 import jdk.internal.vm.compiler.collections.UnmodifiableMapCursor; 43 import org.graalvm.compiler.api.replacements.MethodSubstitution; 44 import org.graalvm.compiler.core.common.GraalOptions; 45 import org.graalvm.compiler.core.common.type.Stamp; 46 import org.graalvm.compiler.core.common.type.StampFactory; 47 import org.graalvm.compiler.core.common.type.TypeReference; 48 import org.graalvm.compiler.core.common.util.Util; 49 import org.graalvm.compiler.debug.DebugCloseable; 50 import org.graalvm.compiler.debug.DebugContext; 51 import org.graalvm.compiler.debug.GraalError; 52 import org.graalvm.compiler.graph.GraalGraphError; 53 import org.graalvm.compiler.graph.Graph.DuplicationReplacement; 54 import org.graalvm.compiler.graph.Graph.Mark; 55 import org.graalvm.compiler.graph.Graph.NodeEventScope; 56 import org.graalvm.compiler.graph.Node; 57 import org.graalvm.compiler.graph.NodeInputList; 58 import org.graalvm.compiler.graph.NodeMap; 59 import org.graalvm.compiler.graph.NodeSourcePosition; 60 import org.graalvm.compiler.graph.NodeWorkList; 61 import org.graalvm.compiler.nodeinfo.Verbosity; 62 import org.graalvm.compiler.nodes.AbstractBeginNode; 63 import org.graalvm.compiler.nodes.AbstractEndNode; 64 import org.graalvm.compiler.nodes.AbstractMergeNode; 65 import org.graalvm.compiler.nodes.BeginNode; 66 import org.graalvm.compiler.nodes.CallTargetNode; 67 import org.graalvm.compiler.nodes.CallTargetNode.InvokeKind; 68 import org.graalvm.compiler.nodes.DeoptimizeNode; 69 import org.graalvm.compiler.nodes.DeoptimizingGuard; 70 import org.graalvm.compiler.nodes.EndNode; 71 import org.graalvm.compiler.nodes.FixedGuardNode; 72 import org.graalvm.compiler.nodes.FixedNode; 73 import org.graalvm.compiler.nodes.FixedWithNextNode; 74 import org.graalvm.compiler.nodes.FrameState; 75 import org.graalvm.compiler.nodes.InliningLog; 76 import org.graalvm.compiler.nodes.Invoke; 77 import org.graalvm.compiler.nodes.InvokeNode; 78 import org.graalvm.compiler.nodes.InvokeWithExceptionNode; 79 import org.graalvm.compiler.nodes.KillingBeginNode; 80 import org.graalvm.compiler.nodes.LogicNode; 81 import org.graalvm.compiler.nodes.MergeNode; 82 import org.graalvm.compiler.nodes.NodeView; 83 import org.graalvm.compiler.nodes.ParameterNode; 84 import org.graalvm.compiler.nodes.PhiNode; 85 import org.graalvm.compiler.nodes.PiNode; 86 import org.graalvm.compiler.nodes.ReturnNode; 87 import org.graalvm.compiler.nodes.StartNode; 88 import org.graalvm.compiler.nodes.StateSplit; 89 import org.graalvm.compiler.nodes.StructuredGraph; 90 import org.graalvm.compiler.nodes.StructuredGraph.GuardsStage; 91 import org.graalvm.compiler.nodes.UnwindNode; 92 import org.graalvm.compiler.nodes.ValueNode; 93 import org.graalvm.compiler.nodes.calc.IsNullNode; 94 import org.graalvm.compiler.nodes.extended.ForeignCallNode; 95 import org.graalvm.compiler.nodes.extended.GuardingNode; 96 import org.graalvm.compiler.nodes.java.ExceptionObjectNode; 97 import org.graalvm.compiler.nodes.java.MethodCallTargetNode; 98 import org.graalvm.compiler.nodes.java.MonitorExitNode; 99 import org.graalvm.compiler.nodes.java.MonitorIdNode; 100 import org.graalvm.compiler.nodes.spi.Replacements; 101 import org.graalvm.compiler.nodes.type.StampTool; 102 import org.graalvm.compiler.nodes.util.GraphUtil; 103 import org.graalvm.compiler.phases.common.inlining.info.InlineInfo; 104 import org.graalvm.compiler.phases.common.util.HashSetNodeEventListener; 105 import org.graalvm.compiler.phases.util.ValueMergeUtil; 106 107 import jdk.vm.ci.code.BytecodeFrame; 108 import jdk.vm.ci.meta.Assumptions; 109 import jdk.vm.ci.meta.DeoptimizationAction; 110 import jdk.vm.ci.meta.DeoptimizationReason; 111 import jdk.vm.ci.meta.JavaKind; 112 import jdk.vm.ci.meta.ResolvedJavaMethod; 113 import jdk.vm.ci.meta.ResolvedJavaType; 114 115 public class InliningUtil extends ValueMergeUtil { 116 117 private static final String inliningDecisionsScopeString = "InliningDecisions"; 118 119 /** 120 * Print a HotSpot-style inlining message to the console. 121 */ 122 private static void printInlining(final InlineInfo info, final int inliningDepth, final boolean success, final String msg, final Object... args) { 123 printInlining(info.methodAt(0), info.invoke(), inliningDepth, success, msg, args); 124 } 125 126 /** 127 * @see #printInlining 128 */ 129 private static void printInlining(final ResolvedJavaMethod method, final Invoke invoke, final int inliningDepth, final boolean success, final String msg, final Object... args) { 130 if (HotSpotPrintInlining.getValue(invoke.asNode().getOptions())) { 131 Util.printInlining(method, invoke.bci(), inliningDepth, success, msg, args); 132 } 133 } 134 135 /** 136 * Trace a decision to inline a method. 137 * 138 * This prints a HotSpot-style inlining message to the console, and it also logs the decision to 139 * the logging stream. 140 * 141 * Phases that perform inlining should use this method to trace the inlining decisions, and use 142 * the {@link #traceNotInlinedMethod} methods only for debugging purposes. 143 */ 144 public static void traceInlinedMethod(InlineInfo info, int inliningDepth, boolean allowLogging, String msg, Object... args) { 145 traceMethod(info, inliningDepth, allowLogging, true, msg, args); 146 } 147 148 /** 149 * Trace a decision to inline a method. 150 * 151 * This prints a HotSpot-style inlining message to the console, and it also logs the decision to 152 * the logging stream. 153 * 154 * Phases that perform inlining should use this method to trace the inlining decisions, and use 155 * the {@link #traceNotInlinedMethod} methods only for debugging purposes. 156 */ 157 public static void traceInlinedMethod(Invoke invoke, int inliningDepth, boolean allowLogging, ResolvedJavaMethod method, String msg, Object... args) { 158 traceMethod(invoke, inliningDepth, allowLogging, true, method, msg, args); 159 } 160 161 /** 162 * Trace a decision to not inline a method. 163 * 164 * This prints a HotSpot-style inlining message to the console, and it also logs the decision to 165 * the logging stream. 166 * 167 * Phases that perform inlining should use this method to trace the inlining decisions, and use 168 * the {@link #traceNotInlinedMethod} methods only for debugging purposes. 169 */ 170 public static void traceNotInlinedMethod(InlineInfo info, int inliningDepth, String msg, Object... args) { 171 traceMethod(info, inliningDepth, true, false, msg, args); 172 } 173 174 /** 175 * Trace a decision about not inlining a method. 176 * 177 * This prints a HotSpot-style inlining message to the console, and it also logs the decision to 178 * the logging stream. 179 * 180 * Phases that perform inlining should use this method to trace the inlining decisions, and use 181 * the {@link #traceNotInlinedMethod} methods only for debugging purposes. 182 */ 183 public static void traceNotInlinedMethod(Invoke invoke, int inliningDepth, ResolvedJavaMethod method, String msg, Object... args) { 184 traceMethod(invoke, inliningDepth, true, false, method, msg, args); 185 } 186 187 private static void traceMethod(Invoke invoke, int inliningDepth, boolean allowLogging, boolean success, ResolvedJavaMethod method, String msg, Object... args) { 188 if (allowLogging) { 189 DebugContext debug = invoke.asNode().getDebug(); 190 printInlining(method, invoke, inliningDepth, success, msg, args); 191 if (shouldLogMethod(debug)) { 192 String methodString = methodName(method, invoke); 193 logMethod(debug, methodString, success, msg, args); 194 } 195 } 196 } 197 198 private static void traceMethod(InlineInfo info, int inliningDepth, boolean allowLogging, boolean success, String msg, final Object... args) { 199 if (allowLogging) { 200 printInlining(info, inliningDepth, success, msg, args); 201 DebugContext debug = info.graph().getDebug(); 202 if (shouldLogMethod(debug)) { 203 logMethod(debug, methodName(info), success, msg, args); 204 } 205 } 206 } 207 208 /** 209 * Output a generic inlining decision to the logging stream (e.g. inlining termination 210 * condition). 211 * 212 * Used for debugging purposes. 213 */ 214 public static void logInliningDecision(DebugContext debug, final String msg, final Object... args) { 215 logInlining(debug, msg, args); 216 } 217 218 /** 219 * Output a decision about not inlining a method to the logging stream, for debugging purposes. 220 */ 221 public static void logNotInlinedMethod(Invoke invoke, String msg) { 222 DebugContext debug = invoke.asNode().getDebug(); 223 if (shouldLogMethod(debug)) { 224 String methodString = invoke.toString(); 225 if (invoke.callTarget() == null) { 226 methodString += " callTarget=null"; 227 } else { 228 String targetName = invoke.callTarget().targetName(); 229 if (!methodString.endsWith(targetName)) { 230 methodString += " " + targetName; 231 } 232 } 233 logMethod(debug, methodString, false, msg, new Object[0]); 234 } 235 } 236 237 private static void logMethod(DebugContext debug, final String methodString, final boolean success, final String msg, final Object... args) { 238 String inliningMsg = "inlining " + methodString + ": " + msg; 239 if (!success) { 240 inliningMsg = "not " + inliningMsg; 241 } 242 logInlining(debug, inliningMsg, args); 243 } 244 245 @SuppressWarnings("try") 246 private static void logInlining(DebugContext debug, final String msg, final Object... args) { 247 try (DebugContext.Scope s = debug.scope(inliningDecisionsScopeString)) { 248 // Can't use log here since we are varargs 249 if (debug.isLogEnabled()) { 250 debug.logv(msg, args); 251 } 252 } 253 } 254 255 @SuppressWarnings("try") 256 private static boolean shouldLogMethod(DebugContext debug) { 257 try (DebugContext.Scope s = debug.scope(inliningDecisionsScopeString)) { 258 return debug.isLogEnabled(); 259 } 260 } 261 262 private static String methodName(ResolvedJavaMethod method, Invoke invoke) { 263 if (invoke != null && invoke.stateAfter() != null) { 264 return methodName(invoke.stateAfter(), invoke.bci()) + ": " + method.format("%H.%n(%p):%r") + " (" + method.getCodeSize() + " bytes)"; 265 } else { 266 return method.format("%H.%n(%p):%r") + " (" + method.getCodeSize() + " bytes)"; 267 } 268 } 269 270 private static String methodName(InlineInfo info) { 271 if (info == null) { 272 return "null"; 273 } else if (info.invoke() != null && info.invoke().stateAfter() != null) { 274 return methodName(info.invoke().stateAfter(), info.invoke().bci()) + ": " + info.toString(); 275 } else { 276 return info.toString(); 277 } 278 } 279 280 private static String methodName(FrameState frameState, int bci) { 281 StringBuilder sb = new StringBuilder(); 282 if (frameState.outerFrameState() != null) { 283 sb.append(methodName(frameState.outerFrameState(), frameState.outerFrameState().bci)); 284 sb.append("->"); 285 } 286 ResolvedJavaMethod method = frameState.getMethod(); 287 sb.append(method != null ? method.format("%h.%n") : "?"); 288 sb.append("@").append(bci); 289 return sb.toString(); 290 } 291 292 public static void replaceInvokeCallTarget(Invoke invoke, StructuredGraph graph, InvokeKind invokeKind, ResolvedJavaMethod targetMethod) { 293 MethodCallTargetNode oldCallTarget = (MethodCallTargetNode) invoke.callTarget(); 294 MethodCallTargetNode newCallTarget = graph.add(new MethodCallTargetNode(invokeKind, targetMethod, oldCallTarget.arguments().toArray(new ValueNode[0]), oldCallTarget.returnStamp(), 295 oldCallTarget.getProfile())); 296 invoke.asNode().replaceFirstInput(oldCallTarget, newCallTarget); 297 } 298 299 public static PiNode createAnchoredReceiver(StructuredGraph graph, GuardingNode anchor, ResolvedJavaType commonType, ValueNode receiver, boolean exact) { 300 return createAnchoredReceiver(graph, anchor, receiver, 301 exact ? StampFactory.objectNonNull(TypeReference.createExactTrusted(commonType)) : StampFactory.objectNonNull(TypeReference.createTrusted(graph.getAssumptions(), commonType))); 302 } 303 304 private static PiNode createAnchoredReceiver(StructuredGraph graph, GuardingNode anchor, ValueNode receiver, Stamp stamp) { 305 // to avoid that floating reads on receiver fields float above the type check 306 return graph.unique(new PiNode(receiver, stamp, (ValueNode) anchor)); 307 } 308 309 /** 310 * @return null iff the check succeeds, otherwise a (non-null) descriptive message. 311 */ 312 public static String checkInvokeConditions(Invoke invoke) { 313 if (invoke.predecessor() == null || !invoke.asNode().isAlive()) { 314 return "the invoke is dead code"; 315 } 316 if (!(invoke.callTarget() instanceof MethodCallTargetNode)) { 317 return "the invoke has already been lowered, or has been created as a low-level node"; 318 } 319 MethodCallTargetNode callTarget = (MethodCallTargetNode) invoke.callTarget(); 320 if (callTarget.targetMethod() == null) { 321 return "target method is null"; 322 } 323 assert invoke.stateAfter() != null : invoke; 324 if (!invoke.useForInlining()) { 325 return "the invoke is marked to be not used for inlining"; 326 } 327 ValueNode receiver = callTarget.receiver(); 328 if (receiver != null && receiver.isConstant() && receiver.isNullConstant()) { 329 return "receiver is null"; 330 } 331 return null; 332 } 333 334 /** 335 * Performs an actual inlining, thereby replacing the given invoke with the given 336 * {@code inlineGraph}. 337 * 338 * @param invoke the invoke that will be replaced 339 * @param inlineGraph the graph that the invoke will be replaced with 340 * @param receiverNullCheck true if a null check needs to be generated for non-static inlinings, 341 * false if no such check is required 342 * @param inlineeMethod the actual method being inlined. Maybe be null for snippets. 343 */ 344 @SuppressWarnings("try") 345 public static UnmodifiableEconomicMap<Node, Node> inline(Invoke invoke, StructuredGraph inlineGraph, boolean receiverNullCheck, ResolvedJavaMethod inlineeMethod) { 346 try { 347 return inline(invoke, inlineGraph, receiverNullCheck, inlineeMethod, "reason not specified", "phase not specified"); 348 } catch (GraalError ex) { 349 ex.addContext("inlining into", invoke.asNode().graph().method()); 350 ex.addContext("inlinee", inlineGraph.method()); 351 throw ex; 352 } 353 } 354 355 /** 356 * Performs an actual inlining, thereby replacing the given invoke with the given 357 * {@code inlineGraph}. 358 * 359 * @param invoke the invoke that will be replaced 360 * @param inlineGraph the graph that the invoke will be replaced with 361 * @param receiverNullCheck true if a null check needs to be generated for non-static inlinings, 362 * false if no such check is required 363 * @param inlineeMethod the actual method being inlined. Maybe be null for snippets. 364 * @param reason the reason for inlining, used in tracing 365 * @param phase the phase that invoked inlining 366 */ 367 @SuppressWarnings("try") 368 public static UnmodifiableEconomicMap<Node, Node> inline(Invoke invoke, StructuredGraph inlineGraph, boolean receiverNullCheck, ResolvedJavaMethod inlineeMethod, String reason, String phase) { 369 FixedNode invokeNode = invoke.asNode(); 370 StructuredGraph graph = invokeNode.graph(); 371 final NodeInputList<ValueNode> parameters = invoke.callTarget().arguments(); 372 373 assert inlineGraph.getGuardsStage().ordinal() >= graph.getGuardsStage().ordinal(); 374 assert !invokeNode.graph().isAfterFloatingReadPhase() : "inline isn't handled correctly after floating reads phase"; 375 376 if (receiverNullCheck && !((MethodCallTargetNode) invoke.callTarget()).isStatic()) { 377 nonNullReceiver(invoke); 378 } 379 380 ArrayList<Node> nodes = new ArrayList<>(inlineGraph.getNodes().count()); 381 ArrayList<ReturnNode> returnNodes = new ArrayList<>(4); 382 ArrayList<Invoke> partialIntrinsicExits = new ArrayList<>(); 383 UnwindNode unwindNode = null; 384 final StartNode entryPointNode = inlineGraph.start(); 385 FixedNode firstCFGNode = entryPointNode.next(); 386 if (firstCFGNode == null) { 387 throw new IllegalStateException("Inlined graph is in invalid state: " + inlineGraph); 388 } 389 for (Node node : inlineGraph.getNodes()) { 390 if (node == entryPointNode || (node == entryPointNode.stateAfter() && node.usages().count() == 1) || node instanceof ParameterNode) { 391 // Do nothing. 392 } else { 393 nodes.add(node); 394 if (node instanceof ReturnNode) { 395 returnNodes.add((ReturnNode) node); 396 } else if (node instanceof Invoke) { 397 Invoke invokeInInlineGraph = (Invoke) node; 398 if (invokeInInlineGraph.bci() == BytecodeFrame.UNKNOWN_BCI) { 399 ResolvedJavaMethod target1 = inlineeMethod; 400 ResolvedJavaMethod target2 = invokeInInlineGraph.callTarget().targetMethod(); 401 assert target1.equals(target2) : String.format("invoke in inlined method expected to be partial intrinsic exit (i.e., call to %s), not a call to %s", 402 target1.format("%H.%n(%p)"), target2.format("%H.%n(%p)")); 403 partialIntrinsicExits.add(invokeInInlineGraph); 404 } 405 } else if (node instanceof UnwindNode) { 406 assert unwindNode == null; 407 unwindNode = (UnwindNode) node; 408 } 409 } 410 } 411 412 final AbstractBeginNode prevBegin = AbstractBeginNode.prevBegin(invokeNode); 413 DuplicationReplacement localReplacement = new DuplicationReplacement() { 414 415 @Override 416 public Node replacement(Node node) { 417 if (node instanceof ParameterNode) { 418 return parameters.get(((ParameterNode) node).index()); 419 } else if (node == entryPointNode) { 420 return prevBegin; 421 } 422 return node; 423 } 424 }; 425 426 assert invokeNode.successors().first() != null : invoke; 427 assert invokeNode.predecessor() != null; 428 429 Mark mark = graph.getMark(); 430 // Instead, attach the inlining log of the child graph to the current inlining log. 431 EconomicMap<Node, Node> duplicates; 432 try (InliningLog.UpdateScope scope = graph.getInliningLog().openDefaultUpdateScope()) { 433 duplicates = graph.addDuplicates(nodes, inlineGraph, inlineGraph.getNodeCount(), localReplacement); 434 if (scope != null) { 435 graph.getInliningLog().addDecision(invoke, true, phase, duplicates, inlineGraph.getInliningLog(), reason); 436 } 437 } 438 439 FrameState stateAfter = invoke.stateAfter(); 440 assert stateAfter == null || stateAfter.isAlive(); 441 442 FrameState stateAtExceptionEdge = null; 443 if (invoke instanceof InvokeWithExceptionNode) { 444 InvokeWithExceptionNode invokeWithException = ((InvokeWithExceptionNode) invoke); 445 if (unwindNode != null) { 446 ExceptionObjectNode obj = (ExceptionObjectNode) invokeWithException.exceptionEdge(); 447 stateAtExceptionEdge = obj.stateAfter(); 448 } 449 } 450 451 updateSourcePositions(invoke, inlineGraph, duplicates, !Objects.equals(inlineGraph.method(), inlineeMethod), mark); 452 if (stateAfter != null) { 453 processFrameStates(invoke, inlineGraph, duplicates, stateAtExceptionEdge, returnNodes.size() > 1); 454 int callerLockDepth = stateAfter.nestedLockDepth(); 455 if (callerLockDepth != 0) { 456 for (MonitorIdNode original : inlineGraph.getNodes(MonitorIdNode.TYPE)) { 457 MonitorIdNode monitor = (MonitorIdNode) duplicates.get(original); 458 processMonitorId(invoke.stateAfter(), monitor); 459 } 460 } 461 } else { 462 assert checkContainsOnlyInvalidOrAfterFrameState(duplicates); 463 } 464 465 firstCFGNode = (FixedNode) duplicates.get(firstCFGNode); 466 for (int i = 0; i < returnNodes.size(); i++) { 467 returnNodes.set(i, (ReturnNode) duplicates.get(returnNodes.get(i))); 468 } 469 for (Invoke exit : partialIntrinsicExits) { 470 // A partial intrinsic exit must be replaced with a call to 471 // the intrinsified method. 472 Invoke dup = (Invoke) duplicates.get(exit.asNode()); 473 if (dup instanceof InvokeNode) { 474 InvokeNode repl = graph.add(new InvokeNode(invoke.callTarget(), invoke.bci())); 475 dup.intrinsify(repl.asNode()); 476 } else { 477 ((InvokeWithExceptionNode) dup).replaceWithNewBci(invoke.bci()); 478 } 479 } 480 if (unwindNode != null) { 481 unwindNode = (UnwindNode) duplicates.get(unwindNode); 482 } 483 484 finishInlining(invoke, graph, firstCFGNode, returnNodes, unwindNode, inlineGraph.getAssumptions(), inlineGraph); 485 GraphUtil.killCFG(invokeNode); 486 487 return duplicates; 488 } 489 490 /** 491 * Inline {@code inlineGraph} into the current replacing the node {@code Invoke} and return the 492 * set of nodes which should be canonicalized. The set should only contain nodes which modified 493 * by the inlining since the current graph and {@code inlineGraph} are expected to already be 494 * canonical. 495 * 496 * @param invoke 497 * @param inlineGraph 498 * @param receiverNullCheck 499 * @param inlineeMethod 500 * @return the set of nodes to canonicalize 501 */ 502 @SuppressWarnings("try") 503 public static EconomicSet<Node> inlineForCanonicalization(Invoke invoke, StructuredGraph inlineGraph, boolean receiverNullCheck, ResolvedJavaMethod inlineeMethod, String reason, String phase) { 504 return inlineForCanonicalization(invoke, inlineGraph, receiverNullCheck, inlineeMethod, null, reason, phase); 505 } 506 507 @SuppressWarnings("try") 508 public static EconomicSet<Node> inlineForCanonicalization(Invoke invoke, StructuredGraph inlineGraph, boolean receiverNullCheck, ResolvedJavaMethod inlineeMethod, 509 Consumer<UnmodifiableEconomicMap<Node, Node>> duplicatesConsumer, String reason, String phase) { 510 HashSetNodeEventListener listener = new HashSetNodeEventListener(); 511 /* 512 * This code relies on the fact that Graph.addDuplicates doesn't trigger the 513 * NodeEventListener to track only nodes which were modified into the process of inlining 514 * the graph into the current graph. 515 */ 516 try (NodeEventScope nes = invoke.asNode().graph().trackNodeEvents(listener)) { 517 UnmodifiableEconomicMap<Node, Node> duplicates = InliningUtil.inline(invoke, inlineGraph, receiverNullCheck, inlineeMethod, reason, phase); 518 if (duplicatesConsumer != null) { 519 duplicatesConsumer.accept(duplicates); 520 } 521 } 522 return listener.getNodes(); 523 } 524 525 @SuppressWarnings("try") 526 private static ValueNode finishInlining(Invoke invoke, StructuredGraph graph, FixedNode firstNode, List<ReturnNode> returnNodes, UnwindNode unwindNode, Assumptions inlinedAssumptions, 527 StructuredGraph inlineGraph) { 528 FixedNode invokeNode = invoke.asNode(); 529 FrameState stateAfter = invoke.stateAfter(); 530 assert stateAfter == null || stateAfter.isAlive(); 531 532 invokeNode.replaceAtPredecessor(firstNode); 533 534 if (invoke instanceof InvokeWithExceptionNode) { 535 InvokeWithExceptionNode invokeWithException = ((InvokeWithExceptionNode) invoke); 536 if (unwindNode != null && unwindNode.isAlive()) { 537 assert unwindNode.predecessor() != null; 538 assert invokeWithException.exceptionEdge().successors().count() == 1; 539 ExceptionObjectNode obj = (ExceptionObjectNode) invokeWithException.exceptionEdge(); 540 obj.replaceAtUsages(unwindNode.exception()); 541 Node n = obj.next(); 542 obj.setNext(null); 543 unwindNode.replaceAndDelete(n); 544 545 obj.replaceAtPredecessor(null); 546 obj.safeDelete(); 547 } else { 548 invokeWithException.killExceptionEdge(); 549 } 550 551 // get rid of memory kill 552 AbstractBeginNode begin = invokeWithException.next(); 553 if (begin instanceof KillingBeginNode) { 554 try (DebugCloseable position = begin.withNodeSourcePosition()) { 555 AbstractBeginNode newBegin = new BeginNode(); 556 graph.addAfterFixed(begin, graph.add(newBegin)); 557 begin.replaceAtUsages(newBegin); 558 graph.removeFixed(begin); 559 } 560 } 561 } else { 562 if (unwindNode != null && unwindNode.isAlive()) { 563 try (DebugCloseable position = unwindNode.withNodeSourcePosition()) { 564 DeoptimizeNode deoptimizeNode = addDeoptimizeNode(graph, DeoptimizationAction.InvalidateRecompile, DeoptimizationReason.NotCompiledExceptionHandler); 565 unwindNode.replaceAndDelete(deoptimizeNode); 566 } 567 } 568 } 569 570 ValueNode returnValue; 571 if (!returnNodes.isEmpty()) { 572 FixedNode n = invoke.next(); 573 invoke.setNext(null); 574 if (returnNodes.size() == 1) { 575 ReturnNode returnNode = returnNodes.get(0); 576 returnValue = returnNode.result(); 577 invokeNode.replaceAtUsages(returnValue); 578 returnNode.replaceAndDelete(n); 579 } else { 580 MergeNode merge = graph.add(new MergeNode()); 581 merge.setStateAfter(stateAfter); 582 returnValue = mergeReturns(merge, returnNodes); 583 invokeNode.replaceAtUsages(returnValue); 584 if (merge.isPhiAtMerge(returnValue)) { 585 fixFrameStates(graph, merge, (PhiNode) returnValue); 586 } 587 merge.setNext(n); 588 } 589 } else { 590 returnValue = null; 591 invokeNode.replaceAtUsages(null); 592 GraphUtil.killCFG(invoke.next()); 593 } 594 595 // Copy assumptions from inlinee to caller 596 Assumptions assumptions = graph.getAssumptions(); 597 if (assumptions != null) { 598 if (inlinedAssumptions != null) { 599 assumptions.record(inlinedAssumptions); 600 } 601 } else { 602 assert inlinedAssumptions == null : String.format("cannot inline graph (%s) which makes assumptions into a graph (%s) that doesn't", inlineGraph, graph); 603 } 604 605 // Copy inlined methods from inlinee to caller 606 graph.updateMethods(inlineGraph); 607 608 // Update the set of accessed fields 609 if (GraalOptions.GeneratePIC.getValue(graph.getOptions())) { 610 graph.updateFields(inlineGraph); 611 } 612 613 if (inlineGraph.hasUnsafeAccess()) { 614 graph.markUnsafeAccess(); 615 } 616 assert inlineGraph.getSpeculationLog() == null || inlineGraph.getSpeculationLog() == graph.getSpeculationLog() : "Only the root graph should have a speculation log"; 617 618 return returnValue; 619 } 620 621 private static void fixFrameStates(StructuredGraph graph, MergeNode originalMerge, PhiNode returnPhi) { 622 // It is possible that some of the frame states that came from AFTER_BCI reference a Phi 623 // node that was created to merge multiple returns. This can create cycles 624 // (see GR-3949 and GR-3957). 625 // To detect this, we follow the control paths starting from the merge node, 626 // split the Phi node inputs at merges and assign the proper input to each frame state. 627 NodeMap<Node> seen = new NodeMap<>(graph); 628 ArrayDeque<Node> workList = new ArrayDeque<>(); 629 ArrayDeque<ValueNode> valueList = new ArrayDeque<>(); 630 workList.push(originalMerge); 631 valueList.push(returnPhi); 632 while (!workList.isEmpty()) { 633 Node current = workList.pop(); 634 ValueNode currentValue = valueList.pop(); 635 if (seen.containsKey(current)) { 636 continue; 637 } 638 seen.put(current, current); 639 if (current instanceof StateSplit && current != originalMerge) { 640 StateSplit stateSplit = (StateSplit) current; 641 FrameState state = stateSplit.stateAfter(); 642 if (state != null && state.values().contains(returnPhi)) { 643 int index = 0; 644 FrameState duplicate = state.duplicate(); 645 for (ValueNode value : state.values()) { 646 if (value == returnPhi) { 647 duplicate.values().set(index, currentValue); 648 } 649 index++; 650 } 651 stateSplit.setStateAfter(duplicate); 652 GraphUtil.tryKillUnused(state); 653 } 654 } 655 if (current instanceof AbstractMergeNode) { 656 AbstractMergeNode currentMerge = (AbstractMergeNode) current; 657 for (EndNode pred : currentMerge.cfgPredecessors()) { 658 ValueNode newValue = currentValue; 659 if (currentMerge.isPhiAtMerge(currentValue)) { 660 PhiNode currentPhi = (PhiNode) currentValue; 661 newValue = currentPhi.valueAt(pred); 662 } 663 workList.push(pred); 664 valueList.push(newValue); 665 } 666 } else if (current.predecessor() != null) { 667 workList.push(current.predecessor()); 668 valueList.push(currentValue); 669 } 670 } 671 } 672 673 @SuppressWarnings("try") 674 private static void updateSourcePositions(Invoke invoke, StructuredGraph inlineGraph, UnmodifiableEconomicMap<Node, Node> duplicates, boolean isSub, Mark mark) { 675 FixedNode invokeNode = invoke.asNode(); 676 StructuredGraph invokeGraph = invokeNode.graph(); 677 if (invokeGraph.trackNodeSourcePosition() && invoke.stateAfter() != null) { 678 boolean isSubstitution = isSub || inlineGraph.isSubstitution(); 679 assert !invokeGraph.trackNodeSourcePosition() || inlineGraph.trackNodeSourcePosition() || 680 isSubstitution : String.format("trackNodeSourcePosition mismatch %s %s != %s %s", invokeGraph, invokeGraph.trackNodeSourcePosition(), inlineGraph, 681 inlineGraph.trackNodeSourcePosition()); 682 final NodeSourcePosition invokePos = invoke.asNode().getNodeSourcePosition(); 683 updateSourcePosition(invokeGraph, duplicates, mark, invokePos, isSubstitution); 684 } 685 } 686 687 public static void updateSourcePosition(StructuredGraph invokeGraph, UnmodifiableEconomicMap<Node, Node> duplicates, Mark mark, NodeSourcePosition invokePos, boolean isSubstitution) { 688 /* 689 * Not every duplicate node is newly created, so only update the position of the newly 690 * created nodes. 691 */ 692 EconomicSet<Node> newNodes = EconomicSet.create(Equivalence.DEFAULT); 693 newNodes.addAll(invokeGraph.getNewNodes(mark)); 694 EconomicMap<NodeSourcePosition, NodeSourcePosition> posMap = EconomicMap.create(Equivalence.DEFAULT); 695 UnmodifiableMapCursor<Node, Node> cursor = duplicates.getEntries(); 696 ResolvedJavaMethod inlineeRoot = null; 697 while (cursor.advance()) { 698 Node value = cursor.getValue(); 699 if (!newNodes.contains(value)) { 700 continue; 701 } 702 if (isSubstitution && invokePos == null) { 703 // There's no caller information so the source position for this node will be 704 // invalid, so it should be cleared. 705 value.clearNodeSourcePosition(); 706 } else { 707 NodeSourcePosition pos = cursor.getKey().getNodeSourcePosition(); 708 if (pos != null) { 709 if (inlineeRoot == null) { 710 assert (inlineeRoot = pos.getRootMethod()) != null; 711 } else { 712 assert pos.verifyRootMethod(inlineeRoot); 713 } 714 NodeSourcePosition callerPos = posMap.get(pos); 715 if (callerPos == null) { 716 callerPos = pos.addCaller(invokePos, isSubstitution); 717 posMap.put(pos, callerPos); 718 } 719 value.setNodeSourcePosition(callerPos); 720 721 if (value instanceof DeoptimizingGuard) { 722 ((DeoptimizingGuard) value).addCallerToNoDeoptSuccessorPosition(callerPos.getCaller()); 723 } 724 } else { 725 if (isSubstitution) { 726 /* 727 * If no other position is provided at least attribute the substituted node 728 * to the original invoke. 729 */ 730 value.setNodeSourcePosition(invokePos); 731 } 732 } 733 } 734 } 735 assert invokeGraph.verifySourcePositions(false); 736 } 737 738 public static void processMonitorId(FrameState stateAfter, MonitorIdNode monitorIdNode) { 739 if (stateAfter != null) { 740 int callerLockDepth = stateAfter.nestedLockDepth(); 741 monitorIdNode.setLockDepth(monitorIdNode.getLockDepth() + callerLockDepth); 742 } 743 } 744 745 protected static void processFrameStates(Invoke invoke, StructuredGraph inlineGraph, EconomicMap<Node, Node> duplicates, FrameState stateAtExceptionEdge, 746 boolean alwaysDuplicateStateAfter) { 747 FrameState stateAtReturn = invoke.stateAfter(); 748 FrameState outerFrameState = null; 749 JavaKind invokeReturnKind = invoke.asNode().getStackKind(); 750 EconomicMap<Node, Node> replacements = EconomicMap.create(); 751 for (FrameState original : inlineGraph.getNodes(FrameState.TYPE)) { 752 FrameState frameState = (FrameState) duplicates.get(original); 753 if (frameState != null && frameState.isAlive()) { 754 if (outerFrameState == null) { 755 outerFrameState = stateAtReturn.duplicateModifiedDuringCall(invoke.bci(), invokeReturnKind); 756 } 757 processFrameState(frameState, invoke, replacements, inlineGraph.method(), stateAtExceptionEdge, outerFrameState, alwaysDuplicateStateAfter, invoke.callTarget().targetMethod(), 758 invoke.callTarget().arguments()); 759 } 760 } 761 // If processing the frame states replaced any nodes, update the duplicates map. 762 duplicates.replaceAll((key, value) -> replacements.containsKey(value) ? replacements.get(value) : value); 763 } 764 765 public static FrameState processFrameState(FrameState frameState, Invoke invoke, EconomicMap<Node, Node> replacements, ResolvedJavaMethod inlinedMethod, FrameState stateAtExceptionEdge, 766 FrameState outerFrameState, 767 boolean alwaysDuplicateStateAfter, ResolvedJavaMethod invokeTargetMethod, List<ValueNode> invokeArgsList) { 768 assert outerFrameState == null || !outerFrameState.isDeleted() : outerFrameState; 769 final FrameState stateAtReturn = invoke.stateAfter(); 770 JavaKind invokeReturnKind = invoke.asNode().getStackKind(); 771 772 if (frameState.bci == BytecodeFrame.AFTER_BCI) { 773 return handleAfterBciFrameState(frameState, invoke, alwaysDuplicateStateAfter); 774 } else if (stateAtExceptionEdge != null && isStateAfterException(frameState)) { 775 // pop exception object from invoke's stateAfter and replace with this frameState's 776 // exception object (top of stack) 777 FrameState stateAfterException = stateAtExceptionEdge; 778 if (frameState.stackSize() > 0 && stateAtExceptionEdge.stackAt(0) != frameState.stackAt(0)) { 779 stateAfterException = stateAtExceptionEdge.duplicateModified(JavaKind.Object, JavaKind.Object, frameState.stackAt(0)); 780 } 781 frameState.replaceAndDelete(stateAfterException); 782 return stateAfterException; 783 } else if ((frameState.bci == BytecodeFrame.UNWIND_BCI && frameState.graph().getGuardsStage() == GuardsStage.FLOATING_GUARDS) || frameState.bci == BytecodeFrame.AFTER_EXCEPTION_BCI) { 784 /* 785 * This path converts the frame states relevant for exception unwinding to 786 * deoptimization. This is only allowed in configurations when Graal compiles code for 787 * speculative execution (e.g., JIT compilation in HotSpot) but not when compiled code 788 * must be deoptimization free (e.g., AOT compilation for native image generation). 789 * There is currently no global flag in StructuredGraph to distinguish such modes, but 790 * the GuardsStage during inlining indicates the mode in which Graal operates. 791 */ 792 handleMissingAfterExceptionFrameState(frameState, invoke, replacements, alwaysDuplicateStateAfter); 793 return frameState; 794 } else if (frameState.bci == BytecodeFrame.BEFORE_BCI) { 795 // This is an intrinsic. Deoptimizing within an intrinsic 796 // must re-execute the intrinsified invocation 797 assert frameState.outerFrameState() == null; 798 ValueNode[] invokeArgs = invokeArgsList.isEmpty() ? NO_ARGS : invokeArgsList.toArray(new ValueNode[invokeArgsList.size()]); 799 FrameState stateBeforeCall = stateAtReturn.duplicateModifiedBeforeCall(invoke.bci(), invokeReturnKind, invokeTargetMethod.getSignature().toParameterKinds(!invokeTargetMethod.isStatic()), 800 invokeArgs); 801 frameState.replaceAndDelete(stateBeforeCall); 802 return stateBeforeCall; 803 } else { 804 // only handle the outermost frame states 805 if (frameState.outerFrameState() == null) { 806 assert checkInlineeFrameState(invoke, inlinedMethod, frameState); 807 frameState.setOuterFrameState(outerFrameState); 808 } 809 return frameState; 810 } 811 } 812 813 private static FrameState handleAfterBciFrameState(FrameState frameState, Invoke invoke, boolean alwaysDuplicateStateAfter) { 814 FrameState stateAtReturn = invoke.stateAfter(); 815 JavaKind invokeReturnKind = invoke.asNode().getStackKind(); 816 FrameState stateAfterReturn = stateAtReturn; 817 if (frameState.getCode() == null) { 818 // This is a frame state for a side effect within an intrinsic 819 // that was parsed for post-parse intrinsification 820 for (Node usage : frameState.usages()) { 821 if (usage instanceof ForeignCallNode) { 822 // A foreign call inside an intrinsic needs to have 823 // the BCI of the invoke being intrinsified 824 ForeignCallNode foreign = (ForeignCallNode) usage; 825 foreign.setBci(invoke.bci()); 826 } 827 } 828 } 829 830 // pop return kind from invoke's stateAfter and replace with this frameState's return 831 // value (top of stack) 832 assert !frameState.rethrowException() : frameState; 833 if (frameState.stackSize() > 0 && (alwaysDuplicateStateAfter || stateAfterReturn.stackAt(0) != frameState.stackAt(0))) { 834 // A non-void return value. 835 stateAfterReturn = stateAtReturn.duplicateModified(invokeReturnKind, invokeReturnKind, frameState.stackAt(0)); 836 } else { 837 // A void return value. 838 stateAfterReturn = stateAtReturn.duplicate(); 839 } 840 assert stateAfterReturn.bci != BytecodeFrame.UNKNOWN_BCI; 841 842 // Return value does no longer need to be limited by the monitor exit. 843 for (MonitorExitNode n : frameState.usages().filter(MonitorExitNode.class)) { 844 n.clearEscapedReturnValue(); 845 } 846 847 frameState.replaceAndDelete(stateAfterReturn); 848 return stateAfterReturn; 849 } 850 851 static boolean checkInlineeFrameState(Invoke invoke, ResolvedJavaMethod inlinedMethod, FrameState frameState) { 852 assert frameState.bci != BytecodeFrame.AFTER_EXCEPTION_BCI : frameState; 853 assert frameState.bci != BytecodeFrame.BEFORE_BCI : frameState; 854 assert frameState.bci != BytecodeFrame.UNKNOWN_BCI : frameState; 855 if (frameState.bci != BytecodeFrame.INVALID_FRAMESTATE_BCI) { 856 ResolvedJavaMethod method = frameState.getMethod(); 857 if (method.equals(inlinedMethod)) { 858 // Normal inlining expects all outermost inlinee frame states to 859 // denote the inlinee method 860 } else if (method.equals(invoke.callTarget().targetMethod())) { 861 // This occurs when an intrinsic calls back to the original 862 // method to handle a slow path. During parsing of such a 863 // partial intrinsic, these calls are given frame states 864 // that exclude the outer frame state denoting a position 865 // in the intrinsic code. 866 assert inlinedMethod.getAnnotation( 867 MethodSubstitution.class) != null : "expected an intrinsic when inlinee frame state matches method of call target but does not match the method of the inlinee graph: " + 868 frameState; 869 } else if (method.getName().equals(inlinedMethod.getName())) { 870 // This can happen for method substitutions. 871 } else { 872 throw new AssertionError(String.format("inlinedMethod=%s frameState.method=%s frameState=%s invoke.method=%s", inlinedMethod, method, frameState, 873 invoke.callTarget().targetMethod())); 874 } 875 } 876 return true; 877 } 878 879 private static final ValueNode[] NO_ARGS = {}; 880 881 private static boolean isStateAfterException(FrameState frameState) { 882 return frameState.bci == BytecodeFrame.AFTER_EXCEPTION_BCI || (frameState.bci == BytecodeFrame.UNWIND_BCI && !frameState.getMethod().isSynchronized()); 883 } 884 885 @SuppressWarnings("try") 886 public static FrameState handleMissingAfterExceptionFrameState(FrameState nonReplaceableFrameState, Invoke invoke, EconomicMap<Node, Node> replacements, boolean alwaysDuplicateStateAfter) { 887 StructuredGraph graph = nonReplaceableFrameState.graph(); 888 NodeWorkList workList = graph.createNodeWorkList(); 889 workList.add(nonReplaceableFrameState); 890 for (Node node : workList) { 891 FrameState fs = (FrameState) node; 892 for (Node usage : fs.usages().snapshot()) { 893 if (!usage.isAlive()) { 894 continue; 895 } 896 if (usage instanceof FrameState) { 897 workList.add(usage); 898 } else { 899 StateSplit stateSplit = (StateSplit) usage; 900 FixedNode fixedStateSplit = stateSplit.asNode(); 901 if (fixedStateSplit instanceof AbstractMergeNode) { 902 AbstractMergeNode merge = (AbstractMergeNode) fixedStateSplit; 903 while (merge.isAlive()) { 904 AbstractEndNode end = merge.forwardEnds().first(); 905 try (DebugCloseable position = end.withNodeSourcePosition()) { 906 DeoptimizeNode deoptimizeNode = addDeoptimizeNode(graph, DeoptimizationAction.InvalidateRecompile, DeoptimizationReason.NotCompiledExceptionHandler); 907 end.replaceAtPredecessor(deoptimizeNode); 908 GraphUtil.killCFG(end); 909 } 910 } 911 } else if (fixedStateSplit instanceof ExceptionObjectNode) { 912 // The target invoke does not have an exception edge. This means that the 913 // bytecode parser made the wrong assumption of making an 914 // InvokeWithExceptionNode for the partial intrinsic exit. We therefore 915 // replace the InvokeWithExceptionNode with a normal 916 // InvokeNode -- the deoptimization occurs when the invoke throws. 917 InvokeWithExceptionNode oldInvoke = (InvokeWithExceptionNode) fixedStateSplit.predecessor(); 918 FrameState oldFrameState = oldInvoke.stateAfter(); 919 InvokeNode newInvoke = oldInvoke.replaceWithInvoke(); 920 newInvoke.setStateAfter(oldFrameState.duplicate()); 921 if (replacements != null) { 922 replacements.put(oldInvoke, newInvoke); 923 } 924 handleAfterBciFrameState(newInvoke.stateAfter(), invoke, alwaysDuplicateStateAfter); 925 } else { 926 try (DebugCloseable position = fixedStateSplit.withNodeSourcePosition()) { 927 FixedNode deoptimizeNode = addDeoptimizeNode(graph, DeoptimizationAction.InvalidateRecompile, DeoptimizationReason.NotCompiledExceptionHandler); 928 if (fixedStateSplit instanceof AbstractBeginNode) { 929 deoptimizeNode = BeginNode.begin(deoptimizeNode); 930 } 931 fixedStateSplit.replaceAtPredecessor(deoptimizeNode); 932 GraphUtil.killCFG(fixedStateSplit); 933 } 934 } 935 } 936 } 937 } 938 return nonReplaceableFrameState; 939 } 940 941 private static DeoptimizeNode addDeoptimizeNode(StructuredGraph graph, DeoptimizationAction action, DeoptimizationReason reason) { 942 GraalError.guarantee(graph.getGuardsStage() == GuardsStage.FLOATING_GUARDS, "Cannot introduce speculative deoptimization when Graal is used with fixed guards"); 943 return graph.add(new DeoptimizeNode(action, reason)); 944 } 945 946 /** 947 * Ensure that all states are either {@link BytecodeFrame#INVALID_FRAMESTATE_BCI} or one of 948 * {@link BytecodeFrame#AFTER_BCI} or {@link BytecodeFrame#BEFORE_BCI}. Mixing of before and 949 * after isn't allowed. 950 */ 951 private static boolean checkContainsOnlyInvalidOrAfterFrameState(UnmodifiableEconomicMap<Node, Node> duplicates) { 952 int okBci = BytecodeFrame.INVALID_FRAMESTATE_BCI; 953 for (Node node : duplicates.getValues()) { 954 if (node instanceof FrameState) { 955 FrameState frameState = (FrameState) node; 956 if (frameState.bci == BytecodeFrame.INVALID_FRAMESTATE_BCI) { 957 continue; 958 } 959 if (frameState.bci == BytecodeFrame.AFTER_BCI || frameState.bci == BytecodeFrame.BEFORE_BCI) { 960 if (okBci == BytecodeFrame.INVALID_FRAMESTATE_BCI) { 961 okBci = frameState.bci; 962 } else { 963 assert okBci == frameState.bci : node.toString(Verbosity.Debugger); 964 } 965 } else { 966 assert false : node.toString(Verbosity.Debugger); 967 } 968 } 969 } 970 return true; 971 } 972 973 /** 974 * Gets the receiver for an invoke, adding a guard if necessary to ensure it is non-null, and 975 * ensuring that the resulting type is compatible with the method being invoked. 976 */ 977 @SuppressWarnings("try") 978 public static ValueNode nonNullReceiver(Invoke invoke) { 979 try (DebugCloseable position = invoke.asNode().withNodeSourcePosition()) { 980 MethodCallTargetNode callTarget = (MethodCallTargetNode) invoke.callTarget(); 981 assert !callTarget.isStatic() : callTarget.targetMethod(); 982 StructuredGraph graph = callTarget.graph(); 983 ValueNode oldReceiver = callTarget.arguments().get(0); 984 ValueNode newReceiver = oldReceiver; 985 if (newReceiver.getStackKind() == JavaKind.Object) { 986 987 if (invoke.getInvokeKind() == InvokeKind.Special) { 988 Stamp paramStamp = newReceiver.stamp(NodeView.DEFAULT); 989 Stamp stamp = paramStamp.join(StampFactory.object(TypeReference.create(graph.getAssumptions(), callTarget.targetMethod().getDeclaringClass()))); 990 if (!stamp.equals(paramStamp)) { 991 // The verifier and previous optimizations guarantee unconditionally that 992 // the 993 // receiver is at least of the type of the method holder for a special 994 // invoke. 995 newReceiver = graph.unique(new PiNode(newReceiver, stamp)); 996 } 997 } 998 999 if (!StampTool.isPointerNonNull(newReceiver)) { 1000 LogicNode condition = graph.unique(IsNullNode.create(newReceiver)); 1001 FixedGuardNode fixedGuard = graph.add(new FixedGuardNode(condition, NullCheckException, InvalidateReprofile, true)); 1002 PiNode nonNullReceiver = graph.unique(new PiNode(newReceiver, StampFactory.objectNonNull(), fixedGuard)); 1003 graph.addBeforeFixed(invoke.asNode(), fixedGuard); 1004 newReceiver = nonNullReceiver; 1005 } 1006 } 1007 1008 if (newReceiver != oldReceiver) { 1009 callTarget.replaceFirstInput(oldReceiver, newReceiver); 1010 } 1011 return newReceiver; 1012 } 1013 } 1014 1015 public static boolean canIntrinsify(Replacements replacements, ResolvedJavaMethod target, int invokeBci) { 1016 return replacements.hasSubstitution(target, invokeBci); 1017 } 1018 1019 public static StructuredGraph getIntrinsicGraph(Replacements replacements, ResolvedJavaMethod target, int invokeBci, boolean trackNodeSourcePosition, NodeSourcePosition replaceePosition) { 1020 return replacements.getSubstitution(target, invokeBci, trackNodeSourcePosition, replaceePosition); 1021 } 1022 1023 public static FixedWithNextNode inlineMacroNode(Invoke invoke, ResolvedJavaMethod concrete, Class<? extends FixedWithNextNode> macroNodeClass) throws GraalError { 1024 StructuredGraph graph = invoke.asNode().graph(); 1025 if (!concrete.equals(((MethodCallTargetNode) invoke.callTarget()).targetMethod())) { 1026 assert ((MethodCallTargetNode) invoke.callTarget()).invokeKind().hasReceiver(); 1027 InliningUtil.replaceInvokeCallTarget(invoke, graph, InvokeKind.Special, concrete); 1028 } 1029 1030 FixedWithNextNode macroNode = createMacroNodeInstance(macroNodeClass, invoke); 1031 1032 CallTargetNode callTarget = invoke.callTarget(); 1033 if (invoke instanceof InvokeNode) { 1034 graph.replaceFixedWithFixed((InvokeNode) invoke, graph.add(macroNode)); 1035 } else { 1036 InvokeWithExceptionNode invokeWithException = (InvokeWithExceptionNode) invoke; 1037 invokeWithException.killExceptionEdge(); 1038 graph.replaceSplitWithFixed(invokeWithException, graph.add(macroNode), invokeWithException.next()); 1039 } 1040 GraphUtil.killWithUnusedFloatingInputs(callTarget); 1041 return macroNode; 1042 } 1043 1044 private static FixedWithNextNode createMacroNodeInstance(Class<? extends FixedWithNextNode> macroNodeClass, Invoke invoke) throws GraalError { 1045 try { 1046 Constructor<?> cons = macroNodeClass.getDeclaredConstructor(Invoke.class); 1047 return (FixedWithNextNode) cons.newInstance(invoke); 1048 } catch (ReflectiveOperationException | IllegalArgumentException | SecurityException e) { 1049 throw new GraalGraphError(e).addContext(invoke.asNode()).addContext("macroSubstitution", macroNodeClass); 1050 } 1051 } 1052 1053 /** 1054 * This method exclude InstrumentationNode from inlining heuristics. 1055 */ 1056 public static int getNodeCount(StructuredGraph graph) { 1057 return graph.getNodeCount(); 1058 } 1059 1060 }