1 /* 2 * Copyright (c) 2012, 2015, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 package org.graalvm.compiler.phases.common.inlining; 24 25 import static org.graalvm.compiler.core.common.GraalOptions.UseGraalInstrumentation; 26 import static jdk.vm.ci.meta.DeoptimizationAction.InvalidateReprofile; 27 import static jdk.vm.ci.meta.DeoptimizationReason.NullCheckException; 28 29 import java.lang.reflect.Constructor; 30 import java.util.ArrayList; 31 import java.util.HashMap; 32 import java.util.List; 33 import java.util.Map; 34 import java.util.Map.Entry; 35 import java.util.function.Function; 36 37 import org.graalvm.compiler.api.replacements.MethodSubstitution; 38 import org.graalvm.compiler.core.common.type.Stamp; 39 import org.graalvm.compiler.core.common.type.StampFactory; 40 import org.graalvm.compiler.core.common.type.TypeReference; 41 import org.graalvm.compiler.core.common.util.Util; 42 import org.graalvm.compiler.debug.Debug; 43 import org.graalvm.compiler.debug.Debug.Scope; 44 import org.graalvm.compiler.debug.Fingerprint; 45 import org.graalvm.compiler.debug.GraalError; 46 import org.graalvm.compiler.debug.internal.method.MethodMetricsImpl; 47 import org.graalvm.compiler.debug.internal.method.MethodMetricsInlineeScopeInfo; 48 import org.graalvm.compiler.graph.GraalGraphError; 49 import org.graalvm.compiler.graph.Graph; 50 import org.graalvm.compiler.graph.Graph.DuplicationReplacement; 51 import org.graalvm.compiler.graph.Node; 52 import org.graalvm.compiler.graph.NodeInputList; 53 import org.graalvm.compiler.graph.NodeSourcePosition; 54 import org.graalvm.compiler.graph.NodeWorkList; 55 import org.graalvm.compiler.nodeinfo.Verbosity; 56 import org.graalvm.compiler.nodes.AbstractBeginNode; 57 import org.graalvm.compiler.nodes.AbstractEndNode; 58 import org.graalvm.compiler.nodes.AbstractMergeNode; 59 import org.graalvm.compiler.nodes.BeginNode; 60 import org.graalvm.compiler.nodes.CallTargetNode; 61 import org.graalvm.compiler.nodes.CallTargetNode.InvokeKind; 62 import org.graalvm.compiler.nodes.ControlSinkNode; 63 import org.graalvm.compiler.nodes.DeoptimizeNode; 64 import org.graalvm.compiler.nodes.EndNode; 65 import org.graalvm.compiler.nodes.FixedGuardNode; 66 import org.graalvm.compiler.nodes.FixedNode; 67 import org.graalvm.compiler.nodes.FixedWithNextNode; 68 import org.graalvm.compiler.nodes.FrameState; 69 import org.graalvm.compiler.nodes.GuardedValueNode; 70 import org.graalvm.compiler.nodes.Invoke; 71 import org.graalvm.compiler.nodes.InvokeNode; 72 import org.graalvm.compiler.nodes.InvokeWithExceptionNode; 73 import org.graalvm.compiler.nodes.KillingBeginNode; 74 import org.graalvm.compiler.nodes.LogicNode; 75 import org.graalvm.compiler.nodes.MergeNode; 76 import org.graalvm.compiler.nodes.ParameterNode; 77 import org.graalvm.compiler.nodes.PhiNode; 78 import org.graalvm.compiler.nodes.PiNode; 79 import org.graalvm.compiler.nodes.ReturnNode; 80 import org.graalvm.compiler.nodes.StartNode; 81 import org.graalvm.compiler.nodes.StateSplit; 82 import org.graalvm.compiler.nodes.StructuredGraph; 83 import org.graalvm.compiler.nodes.UnwindNode; 84 import org.graalvm.compiler.nodes.ValueNode; 85 import org.graalvm.compiler.nodes.ValuePhiNode; 86 import org.graalvm.compiler.nodes.calc.IsNullNode; 87 import org.graalvm.compiler.nodes.debug.instrumentation.InstrumentationNode; 88 import org.graalvm.compiler.nodes.extended.ForeignCallNode; 89 import org.graalvm.compiler.nodes.extended.GuardingNode; 90 import org.graalvm.compiler.nodes.java.ExceptionObjectNode; 91 import org.graalvm.compiler.nodes.java.MethodCallTargetNode; 92 import org.graalvm.compiler.nodes.java.MonitorExitNode; 93 import org.graalvm.compiler.nodes.java.MonitorIdNode; 94 import org.graalvm.compiler.nodes.spi.Replacements; 95 import org.graalvm.compiler.nodes.type.StampTool; 96 import org.graalvm.compiler.nodes.util.GraphUtil; 97 import org.graalvm.compiler.phases.common.inlining.info.InlineInfo; 98 99 import jdk.vm.ci.code.BytecodeFrame; 100 import jdk.vm.ci.meta.Assumptions; 101 import jdk.vm.ci.meta.DeoptimizationAction; 102 import jdk.vm.ci.meta.DeoptimizationReason; 103 import jdk.vm.ci.meta.JavaConstant; 104 import jdk.vm.ci.meta.JavaKind; 105 import jdk.vm.ci.meta.ResolvedJavaMethod; 106 import jdk.vm.ci.meta.ResolvedJavaType; 107 108 public class InliningUtil { 109 110 private static final String inliningDecisionsScopeString = "InliningDecisions"; 111 112 /** 113 * Print a HotSpot-style inlining message to the console. 114 */ 115 private static void printInlining(final InlineInfo info, final int inliningDepth, final boolean success, final String msg, final Object... args) { 116 printInlining(info.methodAt(0), info.invoke(), inliningDepth, success, msg, args); 117 } 118 119 private static void printInlining(final ResolvedJavaMethod method, final Invoke invoke, final int inliningDepth, final boolean success, final String msg, final Object... args) { 120 Util.printInlining(method, invoke.bci(), inliningDepth, success, msg, args); 121 } 122 123 public static void logInlinedMethod(InlineInfo info, int inliningDepth, boolean allowLogging, String msg, Object... args) { 124 logInliningDecision(info, inliningDepth, allowLogging, true, msg, args); 125 } 126 127 public static void logNotInlinedMethod(InlineInfo info, int inliningDepth, String msg, Object... args) { 128 logInliningDecision(info, inliningDepth, true, false, msg, args); 129 } 130 131 public static void logInliningDecision(InlineInfo info, int inliningDepth, boolean allowLogging, boolean success, String msg, final Object... args) { 132 if (allowLogging) { 133 printInlining(info, inliningDepth, success, msg, args); 134 if (shouldLogInliningDecision()) { 135 logInliningDecision(methodName(info), success, msg, args); 136 } 137 } 138 } 139 140 @SuppressWarnings("try") 141 public static void logInliningDecision(final String msg, final Object... args) { 142 try (Scope s = Debug.scope(inliningDecisionsScopeString)) { 143 // Can't use log here since we are varargs 144 if (Debug.isLogEnabled()) { 145 Debug.logv(msg, args); 146 } 147 } 148 } 149 150 public static void logNotInlinedMethod(Invoke invoke, String msg) { 151 if (shouldLogInliningDecision()) { 152 String methodString = invoke.toString(); 153 if (invoke.callTarget() == null) { 154 methodString += " callTarget=null"; 155 } else { 156 String targetName = invoke.callTarget().targetName(); 157 if (!methodString.endsWith(targetName)) { 158 methodString += " " + targetName; 159 } 160 } 161 logInliningDecision(methodString, false, msg, new Object[0]); 162 } 163 } 164 165 public static void logNotInlined(Invoke invoke, int inliningDepth, ResolvedJavaMethod method, String msg) { 166 logNotInlinedInvoke(invoke, inliningDepth, method, msg, new Object[0]); 167 } 168 169 public static void logNotInlinedInvoke(Invoke invoke, int inliningDepth, ResolvedJavaMethod method, String msg, Object... args) { 170 printInlining(method, invoke, inliningDepth, false, msg, args); 171 if (shouldLogInliningDecision()) { 172 String methodString = methodName(method, invoke); 173 logInliningDecision(methodString, false, msg, args); 174 } 175 } 176 177 private static void logInliningDecision(final String methodString, final boolean success, final String msg, final Object... args) { 178 String inliningMsg = "inlining " + methodString + ": " + msg; 179 if (!success) { 180 inliningMsg = "not " + inliningMsg; 181 } 182 logInliningDecision(inliningMsg, args); 183 } 184 185 @SuppressWarnings("try") 186 public static boolean shouldLogInliningDecision() { 187 try (Scope s = Debug.scope(inliningDecisionsScopeString)) { 188 return Debug.isLogEnabled(); 189 } 190 } 191 192 private static String methodName(ResolvedJavaMethod method, Invoke invoke) { 193 if (invoke != null && invoke.stateAfter() != null) { 194 return methodName(invoke.stateAfter(), invoke.bci()) + ": " + method.format("%H.%n(%p):%r") + " (" + method.getCodeSize() + " bytes)"; 195 } else { 196 return method.format("%H.%n(%p):%r") + " (" + method.getCodeSize() + " bytes)"; 197 } 198 } 199 200 private static String methodName(InlineInfo info) { 201 if (info == null) { 202 return "null"; 203 } else if (info.invoke() != null && info.invoke().stateAfter() != null) { 204 return methodName(info.invoke().stateAfter(), info.invoke().bci()) + ": " + info.toString(); 205 } else { 206 return info.toString(); 207 } 208 } 209 210 private static String methodName(FrameState frameState, int bci) { 211 StringBuilder sb = new StringBuilder(); 212 if (frameState.outerFrameState() != null) { 213 sb.append(methodName(frameState.outerFrameState(), frameState.outerFrameState().bci)); 214 sb.append("->"); 215 } 216 sb.append(frameState.getMethod().format("%h.%n")); 217 sb.append("@").append(bci); 218 return sb.toString(); 219 } 220 221 public static void replaceInvokeCallTarget(Invoke invoke, StructuredGraph graph, InvokeKind invokeKind, ResolvedJavaMethod targetMethod) { 222 MethodCallTargetNode oldCallTarget = (MethodCallTargetNode) invoke.callTarget(); 223 MethodCallTargetNode newCallTarget = graph.add(new MethodCallTargetNode(invokeKind, targetMethod, oldCallTarget.arguments().toArray(new ValueNode[0]), oldCallTarget.returnStamp(), 224 oldCallTarget.getProfile())); 225 invoke.asNode().replaceFirstInput(oldCallTarget, newCallTarget); 226 } 227 228 public static GuardedValueNode createAnchoredReceiver(StructuredGraph graph, GuardingNode anchor, ResolvedJavaType commonType, ValueNode receiver, boolean exact) { 229 return createAnchoredReceiver(graph, anchor, receiver, 230 exact ? StampFactory.objectNonNull(TypeReference.createExactTrusted(commonType)) : StampFactory.objectNonNull(TypeReference.createTrusted(graph.getAssumptions(), commonType))); 231 } 232 233 private static GuardedValueNode createAnchoredReceiver(StructuredGraph graph, GuardingNode anchor, ValueNode receiver, Stamp stamp) { 234 // to avoid that floating reads on receiver fields float above the type check 235 return graph.unique(new GuardedValueNode(receiver, anchor, stamp)); 236 } 237 238 /** 239 * @return null iff the check succeeds, otherwise a (non-null) descriptive message. 240 */ 241 public static String checkInvokeConditions(Invoke invoke) { 242 if (invoke.predecessor() == null || !invoke.asNode().isAlive()) { 243 return "the invoke is dead code"; 244 } 245 if (!(invoke.callTarget() instanceof MethodCallTargetNode)) { 246 return "the invoke has already been lowered, or has been created as a low-level node"; 247 } 248 MethodCallTargetNode callTarget = (MethodCallTargetNode) invoke.callTarget(); 249 if (callTarget.targetMethod() == null) { 250 return "target method is null"; 251 } 252 assert invoke.stateAfter() != null : invoke; 253 if (!invoke.useForInlining()) { 254 return "the invoke is marked to be not used for inlining"; 255 } 256 ValueNode receiver = callTarget.receiver(); 257 if (receiver != null && receiver.isConstant() && receiver.isNullConstant()) { 258 return "receiver is null"; 259 } 260 return null; 261 } 262 263 /** 264 * Performs an actual inlining, thereby replacing the given invoke with the given inlineGraph. 265 * 266 * @param invoke the invoke that will be replaced 267 * @param inlineGraph the graph that the invoke will be replaced with 268 * @param receiverNullCheck true if a null check needs to be generated for non-static inlinings, 269 * false if no such check is required 270 * @param canonicalizedNodes if non-null then append to this list any nodes which should be 271 * canonicalized after inlining 272 * @param inlineeMethod the actual method being inlined. Maybe be null for snippets. 273 */ 274 @SuppressWarnings("try") 275 public static Map<Node, Node> inline(Invoke invoke, StructuredGraph inlineGraph, boolean receiverNullCheck, List<Node> canonicalizedNodes, ResolvedJavaMethod inlineeMethod) { 276 MethodMetricsInlineeScopeInfo m = MethodMetricsInlineeScopeInfo.create(); 277 try (Debug.Scope s = Debug.methodMetricsScope("InlineEnhancement", m, false)) { 278 FixedNode invokeNode = invoke.asNode(); 279 StructuredGraph graph = invokeNode.graph(); 280 if (Fingerprint.ENABLED) { 281 Fingerprint.submit("inlining %s into %s: %s", formatGraph(inlineGraph), formatGraph(invoke.asNode().graph()), inlineGraph.getNodes().snapshot()); 282 } 283 final NodeInputList<ValueNode> parameters = invoke.callTarget().arguments(); 284 285 assert inlineGraph.getGuardsStage().ordinal() >= graph.getGuardsStage().ordinal(); 286 assert !invokeNode.graph().isAfterFloatingReadPhase() : "inline isn't handled correctly after floating reads phase"; 287 288 if (receiverNullCheck && !((MethodCallTargetNode) invoke.callTarget()).isStatic()) { 289 nonNullReceiver(invoke); 290 } 291 292 ArrayList<Node> nodes = new ArrayList<>(inlineGraph.getNodes().count()); 293 ArrayList<ReturnNode> returnNodes = new ArrayList<>(4); 294 UnwindNode unwindNode = null; 295 final StartNode entryPointNode = inlineGraph.start(); 296 FixedNode firstCFGNode = entryPointNode.next(); 297 if (firstCFGNode == null) { 298 throw new IllegalStateException("Inlined graph is in invalid state: " + inlineGraph); 299 } 300 for (Node node : inlineGraph.getNodes()) { 301 if (node == entryPointNode || (node == entryPointNode.stateAfter() && node.usages().count() == 1) || node instanceof ParameterNode) { 302 // Do nothing. 303 } else { 304 nodes.add(node); 305 if (node instanceof ReturnNode) { 306 returnNodes.add((ReturnNode) node); 307 } else if (node instanceof UnwindNode) { 308 assert unwindNode == null; 309 unwindNode = (UnwindNode) node; 310 } 311 } 312 } 313 314 final AbstractBeginNode prevBegin = AbstractBeginNode.prevBegin(invokeNode); 315 DuplicationReplacement localReplacement = new DuplicationReplacement() { 316 317 @Override 318 public Node replacement(Node node) { 319 if (node instanceof ParameterNode) { 320 return parameters.get(((ParameterNode) node).index()); 321 } else if (node == entryPointNode) { 322 return prevBegin; 323 } 324 return node; 325 } 326 }; 327 328 assert invokeNode.successors().first() != null : invoke; 329 assert invokeNode.predecessor() != null; 330 331 Map<Node, Node> duplicates = graph.addDuplicates(nodes, inlineGraph, inlineGraph.getNodeCount(), localReplacement); 332 333 FrameState stateAfter = invoke.stateAfter(); 334 assert stateAfter == null || stateAfter.isAlive(); 335 336 FrameState stateAtExceptionEdge = null; 337 if (invoke instanceof InvokeWithExceptionNode) { 338 InvokeWithExceptionNode invokeWithException = ((InvokeWithExceptionNode) invoke); 339 if (unwindNode != null) { 340 ExceptionObjectNode obj = (ExceptionObjectNode) invokeWithException.exceptionEdge(); 341 stateAtExceptionEdge = obj.stateAfter(); 342 } 343 } 344 345 updateSourcePositions(invoke, inlineGraph, duplicates); 346 if (stateAfter != null) { 347 processFrameStates(invoke, inlineGraph, duplicates, stateAtExceptionEdge, returnNodes.size() > 1); 348 int callerLockDepth = stateAfter.nestedLockDepth(); 349 if (callerLockDepth != 0) { 350 for (MonitorIdNode original : inlineGraph.getNodes(MonitorIdNode.TYPE)) { 351 MonitorIdNode monitor = (MonitorIdNode) duplicates.get(original); 352 processMonitorId(invoke.stateAfter(), monitor); 353 } 354 } 355 } else { 356 assert checkContainsOnlyInvalidOrAfterFrameState(duplicates); 357 } 358 359 firstCFGNode = (FixedNode) duplicates.get(firstCFGNode); 360 for (int i = 0; i < returnNodes.size(); i++) { 361 returnNodes.set(i, (ReturnNode) duplicates.get(returnNodes.get(i))); 362 } 363 if (unwindNode != null) { 364 unwindNode = (UnwindNode) duplicates.get(unwindNode); 365 } 366 367 if (UseGraalInstrumentation.getValue()) { 368 detachInstrumentation(invoke); 369 } 370 ValueNode returnValue = finishInlining(invoke, graph, firstCFGNode, returnNodes, unwindNode, inlineGraph.getAssumptions(), inlineGraph, canonicalizedNodes); 371 if (canonicalizedNodes != null) { 372 if (returnValue != null) { 373 for (Node usage : returnValue.usages()) { 374 canonicalizedNodes.add(usage); 375 } 376 } 377 for (ParameterNode parameter : inlineGraph.getNodes(ParameterNode.TYPE)) { 378 for (Node usage : parameter.usages()) { 379 Node duplicate = duplicates.get(usage); 380 if (duplicate != null && duplicate.isAlive()) { 381 canonicalizedNodes.add(duplicate); 382 } 383 } 384 } 385 } 386 387 GraphUtil.killCFG(invokeNode); 388 389 if (Debug.isMethodMeterEnabled() && m != null) { 390 MethodMetricsImpl.recordInlinee(m.getRootMethod(), invoke.asNode().graph().method(), inlineeMethod); 391 } 392 return duplicates; 393 } 394 } 395 396 public static ValueNode finishInlining(Invoke invoke, StructuredGraph graph, FixedNode firstNode, List<ReturnNode> returnNodes, UnwindNode unwindNode, Assumptions inlinedAssumptions, 397 StructuredGraph inlineGraph, List<Node> canonicalizedNodes) { 398 FixedNode invokeNode = invoke.asNode(); 399 FrameState stateAfter = invoke.stateAfter(); 400 assert stateAfter == null || stateAfter.isAlive(); 401 402 invokeNode.replaceAtPredecessor(firstNode); 403 404 if (invoke instanceof InvokeWithExceptionNode) { 405 InvokeWithExceptionNode invokeWithException = ((InvokeWithExceptionNode) invoke); 406 if (unwindNode != null && unwindNode.isAlive()) { 407 assert unwindNode.predecessor() != null; 408 assert invokeWithException.exceptionEdge().successors().count() == 1; 409 ExceptionObjectNode obj = (ExceptionObjectNode) invokeWithException.exceptionEdge(); 410 obj.replaceAtUsages(unwindNode.exception()); 411 Node n = obj.next(); 412 obj.setNext(null); 413 unwindNode.replaceAndDelete(n); 414 415 obj.replaceAtPredecessor(null); 416 obj.safeDelete(); 417 } else { 418 invokeWithException.killExceptionEdge(); 419 } 420 421 // get rid of memory kill 422 AbstractBeginNode begin = invokeWithException.next(); 423 if (begin instanceof KillingBeginNode) { 424 AbstractBeginNode newBegin = new BeginNode(); 425 graph.addAfterFixed(begin, graph.add(newBegin)); 426 begin.replaceAtUsages(newBegin); 427 graph.removeFixed(begin); 428 } 429 } else { 430 if (unwindNode != null && unwindNode.isAlive()) { 431 DeoptimizeNode deoptimizeNode = graph.add(new DeoptimizeNode(DeoptimizationAction.InvalidateRecompile, DeoptimizationReason.NotCompiledExceptionHandler)); 432 unwindNode.replaceAndDelete(deoptimizeNode); 433 } 434 } 435 436 ValueNode returnValue; 437 if (!returnNodes.isEmpty()) { 438 FixedNode n = invoke.next(); 439 invoke.setNext(null); 440 if (returnNodes.size() == 1) { 441 ReturnNode returnNode = returnNodes.get(0); 442 returnValue = returnNode.result(); 443 invokeNode.replaceAtUsages(returnValue); 444 returnNode.replaceAndDelete(n); 445 } else { 446 AbstractMergeNode merge = graph.add(new MergeNode()); 447 merge.setStateAfter(stateAfter); 448 returnValue = mergeReturns(merge, returnNodes, canonicalizedNodes); 449 invokeNode.replaceAtUsages(returnValue); 450 merge.setNext(n); 451 } 452 } else { 453 returnValue = null; 454 invokeNode.replaceAtUsages(null); 455 GraphUtil.killCFG(invoke.next()); 456 } 457 458 // Copy assumptions from inlinee to caller 459 Assumptions assumptions = graph.getAssumptions(); 460 if (assumptions != null) { 461 if (inlinedAssumptions != null) { 462 assumptions.record(inlinedAssumptions); 463 } 464 } else { 465 assert inlinedAssumptions == null : "cannot inline graph which makes assumptions into a graph that doesn't"; 466 } 467 468 // Copy inlined methods from inlinee to caller 469 graph.updateMethods(inlineGraph); 470 471 // Update the set of accessed fields 472 graph.updateFields(inlineGraph); 473 474 if (inlineGraph.hasUnsafeAccess()) { 475 graph.markUnsafeAccess(); 476 } 477 assert inlineGraph.getSpeculationLog() == null : "Only the root graph should have a speculation log"; 478 479 return returnValue; 480 } 481 482 private static String formatGraph(StructuredGraph graph) { 483 if (graph.method() == null) { 484 return graph.name; 485 } 486 return graph.method().format("%H.%n(%p)"); 487 } 488 489 @SuppressWarnings("try") 490 private static void updateSourcePositions(Invoke invoke, StructuredGraph inlineGraph, Map<Node, Node> duplicates) { 491 if (inlineGraph.mayHaveNodeSourcePosition() && invoke.stateAfter() != null) { 492 if (invoke.asNode().getNodeSourcePosition() == null) { 493 // Temporarily ignore the assert below. 494 return; 495 } 496 497 JavaConstant constantReceiver = invoke.getInvokeKind().hasReceiver() ? invoke.getReceiver().asJavaConstant() : null; 498 NodeSourcePosition invokePos = invoke.asNode().getNodeSourcePosition(); 499 assert invokePos != null : "missing source information"; 500 501 Map<NodeSourcePosition, NodeSourcePosition> posMap = new HashMap<>(); 502 for (Entry<Node, Node> entry : duplicates.entrySet()) { 503 NodeSourcePosition pos = entry.getKey().getNodeSourcePosition(); 504 if (pos != null) { 505 NodeSourcePosition callerPos = pos.addCaller(constantReceiver, invokePos); 506 posMap.putIfAbsent(callerPos, callerPos); 507 entry.getValue().setNodeSourcePosition(posMap.get(callerPos)); 508 } 509 } 510 } 511 } 512 513 public static void processMonitorId(FrameState stateAfter, MonitorIdNode monitorIdNode) { 514 if (stateAfter != null) { 515 int callerLockDepth = stateAfter.nestedLockDepth(); 516 monitorIdNode.setLockDepth(monitorIdNode.getLockDepth() + callerLockDepth); 517 } 518 } 519 520 protected static void processFrameStates(Invoke invoke, StructuredGraph inlineGraph, Map<Node, Node> duplicates, FrameState stateAtExceptionEdge, boolean alwaysDuplicateStateAfter) { 521 FrameState stateAtReturn = invoke.stateAfter(); 522 FrameState outerFrameState = null; 523 JavaKind invokeReturnKind = invoke.asNode().getStackKind(); 524 for (FrameState original : inlineGraph.getNodes(FrameState.TYPE)) { 525 FrameState frameState = (FrameState) duplicates.get(original); 526 if (frameState != null && frameState.isAlive()) { 527 if (outerFrameState == null) { 528 outerFrameState = stateAtReturn.duplicateModifiedDuringCall(invoke.bci(), invokeReturnKind); 529 } 530 processFrameState(frameState, invoke, inlineGraph.method(), stateAtExceptionEdge, outerFrameState, alwaysDuplicateStateAfter, invoke.callTarget().targetMethod(), 531 invoke.callTarget().arguments()); 532 } 533 } 534 } 535 536 public static FrameState processFrameState(FrameState frameState, Invoke invoke, ResolvedJavaMethod inlinedMethod, FrameState stateAtExceptionEdge, FrameState outerFrameState, 537 boolean alwaysDuplicateStateAfter, ResolvedJavaMethod invokeTargetMethod, List<ValueNode> invokeArgsList) { 538 539 assert outerFrameState == null || !outerFrameState.isDeleted() : outerFrameState; 540 FrameState stateAtReturn = invoke.stateAfter(); 541 JavaKind invokeReturnKind = invoke.asNode().getStackKind(); 542 543 if (frameState.bci == BytecodeFrame.AFTER_BCI) { 544 FrameState stateAfterReturn = stateAtReturn; 545 if (frameState.getCode() == null) { 546 // This is a frame state for a side effect within an intrinsic 547 // that was parsed for post-parse intrinsification 548 for (Node usage : frameState.usages()) { 549 if (usage instanceof ForeignCallNode) { 550 // A foreign call inside an intrinsic needs to have 551 // the BCI of the invoke being intrinsified 552 ForeignCallNode foreign = (ForeignCallNode) usage; 553 foreign.setBci(invoke.bci()); 554 } 555 } 556 } 557 558 // pop return kind from invoke's stateAfter and replace with this frameState's return 559 // value (top of stack) 560 if (frameState.stackSize() > 0 && (alwaysDuplicateStateAfter || stateAfterReturn.stackAt(0) != frameState.stackAt(0))) { 561 stateAfterReturn = stateAtReturn.duplicateModified(invokeReturnKind, invokeReturnKind, frameState.stackAt(0)); 562 } 563 564 // Return value does no longer need to be limited by the monitor exit. 565 for (MonitorExitNode n : frameState.usages().filter(MonitorExitNode.class)) { 566 n.clearEscapedReturnValue(); 567 } 568 569 frameState.replaceAndDelete(stateAfterReturn); 570 return stateAfterReturn; 571 } else if (stateAtExceptionEdge != null && isStateAfterException(frameState)) { 572 // pop exception object from invoke's stateAfter and replace with this frameState's 573 // exception object (top of stack) 574 FrameState stateAfterException = stateAtExceptionEdge; 575 if (frameState.stackSize() > 0 && stateAtExceptionEdge.stackAt(0) != frameState.stackAt(0)) { 576 stateAfterException = stateAtExceptionEdge.duplicateModified(JavaKind.Object, JavaKind.Object, frameState.stackAt(0)); 577 } 578 frameState.replaceAndDelete(stateAfterException); 579 return stateAfterException; 580 } else if (frameState.bci == BytecodeFrame.UNWIND_BCI || frameState.bci == BytecodeFrame.AFTER_EXCEPTION_BCI) { 581 return handleMissingAfterExceptionFrameState(frameState); 582 } else if (frameState.bci == BytecodeFrame.BEFORE_BCI) { 583 // This is an intrinsic. Deoptimizing within an intrinsic 584 // must re-execute the intrinsified invocation 585 assert frameState.outerFrameState() == null; 586 ValueNode[] invokeArgs = invokeArgsList.isEmpty() ? NO_ARGS : invokeArgsList.toArray(new ValueNode[invokeArgsList.size()]); 587 FrameState stateBeforeCall = stateAtReturn.duplicateModifiedBeforeCall(invoke.bci(), invokeReturnKind, invokeTargetMethod.getSignature().toParameterKinds(!invokeTargetMethod.isStatic()), 588 invokeArgs); 589 frameState.replaceAndDelete(stateBeforeCall); 590 return stateBeforeCall; 591 } else { 592 // only handle the outermost frame states 593 if (frameState.outerFrameState() == null) { 594 assert checkInlineeFrameState(invoke, inlinedMethod, frameState); 595 frameState.setOuterFrameState(outerFrameState); 596 } 597 return frameState; 598 } 599 } 600 601 static boolean checkInlineeFrameState(Invoke invoke, ResolvedJavaMethod inlinedMethod, FrameState frameState) { 602 assert frameState.bci != BytecodeFrame.AFTER_EXCEPTION_BCI : frameState; 603 assert frameState.bci != BytecodeFrame.BEFORE_BCI : frameState; 604 assert frameState.bci != BytecodeFrame.UNKNOWN_BCI : frameState; 605 assert frameState.bci != BytecodeFrame.UNWIND_BCI : frameState; 606 if (frameState.bci != BytecodeFrame.INVALID_FRAMESTATE_BCI) { 607 ResolvedJavaMethod method = frameState.getMethod(); 608 if (method.equals(inlinedMethod)) { 609 // Normal inlining expects all outermost inlinee frame states to 610 // denote the inlinee method 611 } else if (method.equals(invoke.callTarget().targetMethod())) { 612 // This occurs when an intrinsic calls back to the original 613 // method to handle a slow path. During parsing of such a 614 // partial intrinsic, these calls are given frame states 615 // that exclude the outer frame state denoting a position 616 // in the intrinsic code. 617 assert inlinedMethod.getAnnotation( 618 MethodSubstitution.class) != null : "expected an intrinsic when inlinee frame state matches method of call target but does not match the method of the inlinee graph: " + 619 frameState; 620 } else if (method.getName().equals(inlinedMethod.getName())) { 621 // This can happen for method substitutions. 622 } else { 623 throw new AssertionError(String.format("inlinedMethod=%s frameState.method=%s frameState=%s invoke.method=%s", inlinedMethod, method, frameState, 624 invoke.callTarget().targetMethod())); 625 } 626 } 627 return true; 628 } 629 630 private static final ValueNode[] NO_ARGS = {}; 631 632 private static boolean isStateAfterException(FrameState frameState) { 633 return frameState.bci == BytecodeFrame.AFTER_EXCEPTION_BCI || (frameState.bci == BytecodeFrame.UNWIND_BCI && !frameState.getMethod().isSynchronized()); 634 } 635 636 public static FrameState handleMissingAfterExceptionFrameState(FrameState nonReplaceableFrameState) { 637 Graph graph = nonReplaceableFrameState.graph(); 638 NodeWorkList workList = graph.createNodeWorkList(); 639 workList.add(nonReplaceableFrameState); 640 for (Node node : workList) { 641 FrameState fs = (FrameState) node; 642 for (Node usage : fs.usages().snapshot()) { 643 if (!usage.isAlive()) { 644 continue; 645 } 646 if (usage instanceof FrameState) { 647 workList.add(usage); 648 } else { 649 StateSplit stateSplit = (StateSplit) usage; 650 FixedNode fixedStateSplit = stateSplit.asNode(); 651 if (fixedStateSplit instanceof AbstractMergeNode) { 652 AbstractMergeNode merge = (AbstractMergeNode) fixedStateSplit; 653 while (merge.isAlive()) { 654 AbstractEndNode end = merge.forwardEnds().first(); 655 DeoptimizeNode deoptimizeNode = graph.add(new DeoptimizeNode(DeoptimizationAction.InvalidateRecompile, DeoptimizationReason.NotCompiledExceptionHandler)); 656 end.replaceAtPredecessor(deoptimizeNode); 657 GraphUtil.killCFG(end); 658 } 659 } else { 660 FixedNode deoptimizeNode = graph.add(new DeoptimizeNode(DeoptimizationAction.InvalidateRecompile, DeoptimizationReason.NotCompiledExceptionHandler)); 661 if (fixedStateSplit instanceof AbstractBeginNode) { 662 deoptimizeNode = BeginNode.begin(deoptimizeNode); 663 } 664 fixedStateSplit.replaceAtPredecessor(deoptimizeNode); 665 GraphUtil.killCFG(fixedStateSplit); 666 } 667 } 668 } 669 } 670 return nonReplaceableFrameState; 671 } 672 673 public static ValueNode mergeReturns(AbstractMergeNode merge, List<? extends ReturnNode> returnNodes, List<Node> canonicalizedNodes) { 674 return mergeValueProducers(merge, returnNodes, canonicalizedNodes, returnNode -> returnNode.result()); 675 } 676 677 public static <T extends ControlSinkNode> ValueNode mergeValueProducers(AbstractMergeNode merge, List<? extends T> valueProducers, List<Node> canonicalizedNodes, 678 Function<T, ValueNode> valueFunction) { 679 ValueNode singleResult = null; 680 PhiNode phiResult = null; 681 for (T valueProducer : valueProducers) { 682 ValueNode result = valueFunction.apply(valueProducer); 683 if (result != null) { 684 if (phiResult == null && (singleResult == null || singleResult == result)) { 685 /* Only one result value, so no need yet for a phi node. */ 686 singleResult = result; 687 } else if (phiResult == null) { 688 /* Found a second result value, so create phi node. */ 689 phiResult = merge.graph().addWithoutUnique(new ValuePhiNode(result.stamp().unrestricted(), merge)); 690 if (canonicalizedNodes != null) { 691 canonicalizedNodes.add(phiResult); 692 } 693 for (int i = 0; i < merge.forwardEndCount(); i++) { 694 phiResult.addInput(singleResult); 695 } 696 phiResult.addInput(result); 697 698 } else { 699 /* Multiple return values, just add to existing phi node. */ 700 phiResult.addInput(result); 701 } 702 } 703 704 // create and wire up a new EndNode 705 EndNode endNode = merge.graph().add(new EndNode()); 706 merge.addForwardEnd(endNode); 707 valueProducer.replaceAndDelete(endNode); 708 } 709 710 if (phiResult != null) { 711 assert phiResult.verify(); 712 phiResult.inferStamp(); 713 return phiResult; 714 } else { 715 return singleResult; 716 } 717 } 718 719 private static boolean checkContainsOnlyInvalidOrAfterFrameState(Map<Node, Node> duplicates) { 720 for (Node node : duplicates.values()) { 721 if (node instanceof FrameState) { 722 FrameState frameState = (FrameState) node; 723 assert frameState.bci == BytecodeFrame.AFTER_BCI || frameState.bci == BytecodeFrame.INVALID_FRAMESTATE_BCI : node.toString(Verbosity.Debugger); 724 } 725 } 726 return true; 727 } 728 729 /** 730 * Gets the receiver for an invoke, adding a guard if necessary to ensure it is non-null, and 731 * ensuring that the resulting type is compatible with the method being invoked. 732 */ 733 public static ValueNode nonNullReceiver(Invoke invoke) { 734 MethodCallTargetNode callTarget = (MethodCallTargetNode) invoke.callTarget(); 735 assert !callTarget.isStatic() : callTarget.targetMethod(); 736 StructuredGraph graph = callTarget.graph(); 737 ValueNode firstParam = callTarget.arguments().get(0); 738 if (firstParam.getStackKind() == JavaKind.Object) { 739 Stamp paramStamp = firstParam.stamp(); 740 Stamp stamp = paramStamp.join(StampFactory.objectNonNull(TypeReference.create(graph.getAssumptions(), callTarget.targetMethod().getDeclaringClass()))); 741 if (!StampTool.isPointerNonNull(firstParam)) { 742 LogicNode condition = graph.unique(IsNullNode.create(firstParam)); 743 FixedGuardNode fixedGuard = graph.add(new FixedGuardNode(condition, NullCheckException, InvalidateReprofile, true)); 744 PiNode nonNullReceiver = graph.unique(new PiNode(firstParam, stamp, fixedGuard)); 745 graph.addBeforeFixed(invoke.asNode(), fixedGuard); 746 callTarget.replaceFirstInput(firstParam, nonNullReceiver); 747 return nonNullReceiver; 748 } 749 if (!stamp.equals(paramStamp)) { 750 PiNode cast = graph.unique(new PiNode(firstParam, stamp)); 751 callTarget.replaceFirstInput(firstParam, cast); 752 return cast; 753 } 754 } 755 return firstParam; 756 } 757 758 public static boolean canIntrinsify(Replacements replacements, ResolvedJavaMethod target, int invokeBci) { 759 return replacements.hasSubstitution(target, invokeBci); 760 } 761 762 public static StructuredGraph getIntrinsicGraph(Replacements replacements, ResolvedJavaMethod target, int invokeBci) { 763 return replacements.getSubstitution(target, invokeBci); 764 } 765 766 public static FixedWithNextNode inlineMacroNode(Invoke invoke, ResolvedJavaMethod concrete, Class<? extends FixedWithNextNode> macroNodeClass) throws GraalError { 767 StructuredGraph graph = invoke.asNode().graph(); 768 if (!concrete.equals(((MethodCallTargetNode) invoke.callTarget()).targetMethod())) { 769 assert ((MethodCallTargetNode) invoke.callTarget()).invokeKind().hasReceiver(); 770 InliningUtil.replaceInvokeCallTarget(invoke, graph, InvokeKind.Special, concrete); 771 } 772 773 FixedWithNextNode macroNode = createMacroNodeInstance(macroNodeClass, invoke); 774 775 CallTargetNode callTarget = invoke.callTarget(); 776 if (invoke instanceof InvokeNode) { 777 graph.replaceFixedWithFixed((InvokeNode) invoke, graph.add(macroNode)); 778 } else { 779 InvokeWithExceptionNode invokeWithException = (InvokeWithExceptionNode) invoke; 780 invokeWithException.killExceptionEdge(); 781 graph.replaceSplitWithFixed(invokeWithException, graph.add(macroNode), invokeWithException.next()); 782 } 783 GraphUtil.killWithUnusedFloatingInputs(callTarget); 784 return macroNode; 785 } 786 787 private static FixedWithNextNode createMacroNodeInstance(Class<? extends FixedWithNextNode> macroNodeClass, Invoke invoke) throws GraalError { 788 try { 789 Constructor<?> cons = macroNodeClass.getDeclaredConstructor(Invoke.class); 790 return (FixedWithNextNode) cons.newInstance(invoke); 791 } catch (ReflectiveOperationException | IllegalArgumentException | SecurityException e) { 792 throw new GraalGraphError(e).addContext(invoke.asNode()).addContext("macroSubstitution", macroNodeClass); 793 } 794 } 795 796 /** 797 * This method exclude InstrumentationNode from inlining heuristics. 798 */ 799 public static int getNodeCount(StructuredGraph graph) { 800 if (UseGraalInstrumentation.getValue()) { 801 return graph.getNodeCount() - graph.getNodes().filter(InstrumentationNode.class).count(); 802 } else { 803 return graph.getNodeCount(); 804 } 805 } 806 807 /** 808 * This method detach the instrumentation attached to the given Invoke. It is called when the 809 * given Invoke is inlined. 810 */ 811 public static void detachInstrumentation(Invoke invoke) { 812 FixedNode invokeNode = invoke.asNode(); 813 for (InstrumentationNode instrumentation : invokeNode.usages().filter(InstrumentationNode.class).snapshot()) { 814 if (instrumentation.getTarget() == invoke) { 815 instrumentation.replaceFirstInput(instrumentation.getTarget(), null); 816 } 817 } 818 } 819 820 }