1 /*
2 * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24
25 package org.graalvm.compiler.hotspot;
26
27 import static jdk.vm.ci.runtime.JVMCI.getRuntime;
28 import static jdk.vm.ci.services.Services.IS_BUILDING_NATIVE_IMAGE;
29 import static jdk.vm.ci.services.Services.IS_IN_NATIVE_IMAGE;
30 import static org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin.InlineInfo.createIntrinsicInlineInfo;
31 import static org.graalvm.compiler.replacements.ReplacementsImpl.Options.UseEncodedSnippets;
32
33 import java.util.ArrayList;
34 import java.util.Arrays;
35 import java.util.Collections;
36 import java.util.HashMap;
37 import java.util.HashSet;
38 import java.util.List;
39 import java.util.Map;
40 import java.util.Objects;
41 import java.util.Set;
42 import java.util.concurrent.ConcurrentHashMap;
43
44 import org.graalvm.compiler.api.replacements.Fold;
45 import org.graalvm.compiler.api.replacements.MethodSubstitution;
46 import org.graalvm.compiler.api.replacements.Snippet;
47 import org.graalvm.compiler.api.replacements.SnippetReflectionProvider;
48 import org.graalvm.compiler.api.runtime.GraalJVMCICompiler;
49 import org.graalvm.compiler.api.runtime.GraalRuntime;
50 import org.graalvm.compiler.bytecode.BytecodeProvider;
51 import org.graalvm.compiler.core.common.spi.ConstantFieldProvider;
52 import org.graalvm.compiler.core.common.type.AbstractObjectStamp;
53 import org.graalvm.compiler.core.common.type.Stamp;
54 import org.graalvm.compiler.core.common.type.StampPair;
55 import org.graalvm.compiler.core.common.type.SymbolicJVMCIReference;
56 import org.graalvm.compiler.debug.DebugContext;
57 import org.graalvm.compiler.debug.GraalError;
58 import org.graalvm.compiler.graph.Node;
59 import org.graalvm.compiler.graph.NodeClass;
60 import org.graalvm.compiler.graph.NodeMap;
61 import org.graalvm.compiler.graph.NodeSourcePosition;
62 import org.graalvm.compiler.hotspot.meta.HotSpotForeignCallsProvider;
63 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
64 import org.graalvm.compiler.java.BytecodeParser;
65 import org.graalvm.compiler.java.GraphBuilderPhase;
66 import org.graalvm.compiler.nodeinfo.Verbosity;
67 import org.graalvm.compiler.nodes.CallTargetNode;
68 import org.graalvm.compiler.nodes.ConstantNode;
69 import org.graalvm.compiler.nodes.EncodedGraph;
70 import org.graalvm.compiler.nodes.FrameState;
71 import org.graalvm.compiler.nodes.FullInfopointNode;
72 import org.graalvm.compiler.nodes.GraphEncoder;
73 import org.graalvm.compiler.nodes.ParameterNode;
74 import org.graalvm.compiler.nodes.ProxyNode;
75 import org.graalvm.compiler.nodes.StructuredGraph;
76 import org.graalvm.compiler.nodes.ValueNode;
77 import org.graalvm.compiler.nodes.cfg.Block;
78 import org.graalvm.compiler.nodes.graphbuilderconf.GeneratedInvocationPlugin;
79 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration;
80 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext;
81 import org.graalvm.compiler.nodes.graphbuilderconf.InlineInvokePlugin;
82 import org.graalvm.compiler.nodes.graphbuilderconf.IntrinsicContext;
83 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin;
84 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins;
85 import org.graalvm.compiler.nodes.graphbuilderconf.MethodSubstitutionPlugin;
86 import org.graalvm.compiler.nodes.graphbuilderconf.NodePlugin;
87 import org.graalvm.compiler.nodes.graphbuilderconf.ParameterPlugin;
88 import org.graalvm.compiler.nodes.java.AccessFieldNode;
89 import org.graalvm.compiler.nodes.java.MethodCallTargetNode;
90 import org.graalvm.compiler.nodes.spi.DelegatingReplacements;
91 import org.graalvm.compiler.nodes.spi.StampProvider;
92 import org.graalvm.compiler.nodes.virtual.VirtualObjectNode;
93 import org.graalvm.compiler.options.OptionValues;
94 import org.graalvm.compiler.phases.OptimisticOptimizations;
95 import org.graalvm.compiler.phases.schedule.SchedulePhase;
96 import org.graalvm.compiler.phases.util.Providers;
97 import org.graalvm.compiler.replacements.ConstantBindingParameterPlugin;
98 import org.graalvm.compiler.replacements.PEGraphDecoder;
99 import org.graalvm.compiler.replacements.ReplacementsImpl;
100 import org.graalvm.compiler.replacements.SnippetCounter;
101 import org.graalvm.compiler.replacements.SnippetIntegerHistogram;
102
103 import jdk.vm.ci.code.Architecture;
104 import jdk.vm.ci.code.TargetDescription;
105 import jdk.vm.ci.hotspot.HotSpotJVMCIRuntime;
106 import jdk.vm.ci.hotspot.HotSpotResolvedJavaField;
107 import jdk.vm.ci.hotspot.HotSpotResolvedJavaMethod;
108 import jdk.vm.ci.hotspot.HotSpotResolvedJavaType;
109 import jdk.vm.ci.meta.Constant;
110 import jdk.vm.ci.meta.ConstantReflectionProvider;
111 import jdk.vm.ci.meta.JavaConstant;
112 import jdk.vm.ci.meta.JavaKind;
113 import jdk.vm.ci.meta.JavaType;
114 import jdk.vm.ci.meta.MemoryAccessProvider;
115 import jdk.vm.ci.meta.MetaAccessProvider;
116 import jdk.vm.ci.meta.MethodHandleAccessProvider;
117 import jdk.vm.ci.meta.ResolvedJavaField;
118 import jdk.vm.ci.meta.ResolvedJavaMethod;
119 import jdk.vm.ci.meta.ResolvedJavaType;
120 import jdk.vm.ci.meta.UnresolvedJavaField;
121 import jdk.vm.ci.meta.UnresolvedJavaMethod;
122 import jdk.vm.ci.meta.UnresolvedJavaType;
123
124 /**
125 * This class performs graph encoding using {@link GraphEncoder} but also converts JVMCI type and
126 * method references into a symbolic form that can be resolved at graph decode time using
127 * {@link SymbolicJVMCIReference}.
128 */
129 public class SymbolicSnippetEncoder extends DelegatingReplacements {
130
131 /**
132 * This is a customized HotSpotReplacementsImpl intended only for parsing snippets and method
133 * substitutions for graph encoding.
134 */
135 private final HotSpotSnippetReplacementsImpl replacements;
136
137 /**
138 * The set of all snippet methods that have been encoded.
139 */
140 private final Set<ResolvedJavaMethod> snippetMethods = Collections.synchronizedSet(new HashSet<>());
141
142 /**
143 * A mapping from the method substitution method to the original method name. The string key and
144 * values are produced using {@link #methodKey(ResolvedJavaMethod)}.
145 */
146 private final Map<String, String> originalMethods = new ConcurrentHashMap<>();
147
148 /**
149 * The current count of graphs encoded. Used to detect when new graphs have been enqueued for
150 * encoding.
151 */
152 int encodedGraphs = 0;
153
154 /**
155 * All the graphs parsed so far.
156 */
157 private Map<String, StructuredGraph> preparedSnippetGraphs = new HashMap<>();
158
159 /**
160 * The invocation plugins which were delayed during graph preparation.
161 */
162 private Set<ResolvedJavaMethod> delayedInvocationPluginMethods = new HashSet<>();
163
164 void addDelayedInvocationPluginMethod(ResolvedJavaMethod method) {
165 delayedInvocationPluginMethods.add(method);
166 }
167
168 Set<ResolvedJavaMethod> getSnippetMethods() {
169 return snippetMethods;
170 }
171
172 protected class SnippetInlineInvokePlugin implements InlineInvokePlugin {
173
174 @Override
175 public InlineInfo shouldInlineInvoke(GraphBuilderContext b, ResolvedJavaMethod method, ValueNode[] args) {
176 if (method.getAnnotation(Fold.class) != null) {
177 delayedInvocationPluginMethods.add(method);
178 return InlineInfo.DO_NOT_INLINE_NO_EXCEPTION;
179 }
180
181 if (getIntrinsifyingPlugin(method) != null) {
182 delayedInvocationPluginMethods.add(method);
183 return InlineInfo.DO_NOT_INLINE_NO_EXCEPTION;
184 }
185
186 // Force inlining when parsing replacements
187 return createIntrinsicInlineInfo(method, null, getDefaultReplacementBytecodeProvider());
188 }
189
190 @Override
191 public void notifyAfterInline(ResolvedJavaMethod methodToInline) {
192 assert methodToInline.getAnnotation(Fold.class) == null : methodToInline;
193 }
194 }
195
196 public static class SnippetInvocationPlugins extends InvocationPlugins {
197
198 SnippetInvocationPlugins(InvocationPlugins invocationPlugins) {
199 super(invocationPlugins);
200 }
201
202 @Override
203 public InvocationPlugin lookupInvocation(ResolvedJavaMethod method) {
204 if (method.getAnnotation(Fold.class) != null) {
205 return null;
206 }
207 return super.lookupInvocation(method);
208 }
209 }
210
211 /**
212 * This plugin disables the snippet counter machinery.
213 */
214 private class SnippetCounterPlugin implements NodePlugin {
215 String snippetCounterName = 'L' + SnippetCounter.class.getName().replace('.', '/') + ';';
216 String snippetIntegerHistogramName = 'L' + SnippetIntegerHistogram.class.getName().replace('.', '/') + ';';
217
218 @Override
219 public boolean handleLoadField(GraphBuilderContext b, ValueNode object, ResolvedJavaField field) {
220 if (field.getName().equals("group") && field.getDeclaringClass().getName().equals(snippetCounterName)) {
221 b.addPush(JavaKind.Object, ConstantNode.forConstant(JavaConstant.NULL_POINTER, b.getMetaAccess()));
222 return true;
223 }
224 if (field.getType().getName().equals(snippetCounterName)) {
225 b.addPush(JavaKind.Object, ConstantNode.forConstant(replacements.snippetReflection.forObject(SnippetCounter.DISABLED_COUNTER), b.getMetaAccess()));
226 return true;
227 }
228
229 if (field.getType().getName().equals(snippetIntegerHistogramName)) {
230 b.addPush(JavaKind.Object, ConstantNode.forConstant(replacements.snippetReflection.forObject(SnippetIntegerHistogram.DISABLED_COUNTER), b.getMetaAccess()));
231 return true;
232 }
233 return false;
234 }
235 }
236
237 /**
238 * Generate a String name for a method including all type information. Used as a symbolic key
239 * for lookup.
240 */
241 private static String methodKey(ResolvedJavaMethod method) {
242 return method.format("%f %H.%n(%P)");
243 }
244
245 SymbolicSnippetEncoder(HotSpotReplacementsImpl replacements) {
246 super(replacements);
247
248 GraphBuilderConfiguration.Plugins plugins = replacements.getGraphBuilderPlugins();
249 SnippetInvocationPlugins invocationPlugins = new SnippetInvocationPlugins(plugins.getInvocationPlugins());
250 GraphBuilderConfiguration.Plugins copy = new GraphBuilderConfiguration.Plugins(plugins, invocationPlugins);
251 copy.clearInlineInvokePlugins();
252 copy.appendInlineInvokePlugin(new SnippetInlineInvokePlugin());
253 copy.appendNodePlugin(new SnippetCounterPlugin());
254 HotSpotProviders providers = (HotSpotProviders) replacements.getProviders().copyWith(new HotSpotSubstrateConstantReflectionProvider(replacements.getProviders().getConstantReflection()));
255 this.replacements = new HotSpotSnippetReplacementsImpl(replacements, providers.copyWith(copy));
256 this.replacements.setGraphBuilderPlugins(copy);
257 }
258
259 @Override
260 public GraphBuilderConfiguration.Plugins getGraphBuilderPlugins() {
261 return replacements.getGraphBuilderPlugins();
262 }
263
264 /**
265 * Compiles the snippet and stores the graph.
266 */
267 public void registerMethodSubstitution(ResolvedJavaMethod method, ResolvedJavaMethod original) {
268 assert method.getAnnotation(MethodSubstitution.class) != null : "MethodSubstitution must be annotated with @" + MethodSubstitution.class.getSimpleName();
269 buildGraph(method, original, null, false, false);
270 snippetMethods.add(method);
271 }
272
273 static class EncodedSnippets {
274 private byte[] snippetEncoding;
275 private Object[] snippetObjects;
276 private NodeClass<?>[] snippetNodeClasses;
277 private Map<String, Integer> snippetStartOffsets;
278 private Map<String, String> originalMethods;
279
280 EncodedSnippets(byte[] snippetEncoding, Object[] snippetObjects, NodeClass<?>[] snippetNodeClasses, Map<String, Integer> snippetStartOffsets, Map<String, String> originalMethods) {
281 this.snippetEncoding = snippetEncoding;
282 this.snippetObjects = snippetObjects;
283 this.snippetNodeClasses = snippetNodeClasses;
284 this.snippetStartOffsets = snippetStartOffsets;
285 this.originalMethods = originalMethods;
286 }
287
288 public StructuredGraph getMethodSubstitutionGraph(MethodSubstitutionPlugin plugin, ResolvedJavaMethod original, ReplacementsImpl replacements) {
289 Integer startOffset = snippetStartOffsets.get(plugin.toString());
290 if (startOffset == null) {
291 throw GraalError.shouldNotReachHere("plugin graph not found: " + plugin);
292 }
293
294 return decodeGraph(original, null, startOffset, replacements);
295 }
296
297 @SuppressWarnings("try")
298 private StructuredGraph decodeGraph(ResolvedJavaMethod method, Object[] args, int startOffset, ReplacementsImpl replacements) {
299 OptionValues options = replacements.getOptions();
300 SnippetReflectionProvider snippetReflection = replacements.snippetReflection;
301 ParameterPlugin parameterPlugin = null;
302 Providers providers = replacements.getProviders();
303 if (args != null) {
304 parameterPlugin = new ConstantBindingParameterPlugin(args, providers.getMetaAccess(), snippetReflection);
305 }
306
307 EncodedGraph encodedGraph = new SymbolicEncodedGraph(snippetEncoding, startOffset, snippetObjects, snippetNodeClasses, method.getDeclaringClass(),
308 originalMethods.get(methodKey(method)));
309 try (DebugContext debug = replacements.openDebugContext("SVMSnippet_", method)) {
310 StructuredGraph result = new StructuredGraph.Builder(options, debug).method(method).setIsSubstitution(true).build();
311 PEGraphDecoder graphDecoder = new PEGraphDecoder(
312 providers.getCodeCache().getTarget().arch,
313 result,
314 providers.getMetaAccess(),
315 providers.getConstantReflection(),
316 providers.getConstantFieldProvider(),
317 providers.getStampProvider(),
318 null, // loopExplosionPlugin
319 replacements.getGraphBuilderPlugins().getInvocationPlugins(),
320 new InlineInvokePlugin[0],
321 parameterPlugin,
322 null, // nodePlugins
323 null, // callInlinedMethod
324 null // sourceLanguagePositionProvider
325 ) {
326 @Override
327 protected EncodedGraph lookupEncodedGraph(ResolvedJavaMethod lookupMethod,
328 ResolvedJavaMethod originalMethod,
329 BytecodeProvider intrinsicBytecodeProvider,
330 boolean isSubstitution,
331 boolean trackNodeSourcePosition) {
332 if (lookupMethod.equals(method)) {
333 return encodedGraph;
334 } else {
335 throw GraalError.shouldNotReachHere(method.format("%H.%n(%p)"));
336 }
337 }
338 };
339
340 graphDecoder.decode(method, result.isSubstitution(), encodedGraph.trackNodeSourcePosition());
341
342 assert result.verify();
343 return result;
344 }
345 }
346
347 StructuredGraph getEncodedSnippet(ResolvedJavaMethod method, ReplacementsImpl replacements, Object[] args) {
348 Integer startOffset = null;
349 if (snippetStartOffsets != null) {
350 startOffset = snippetStartOffsets.get(methodKey(method));
351 }
352 if (startOffset == null) {
353 if (IS_IN_NATIVE_IMAGE) {
354 throw GraalError.shouldNotReachHere("snippet not found: " + method.format("%H.%n(%p)"));
355 } else {
356 return null;
357 }
358 }
359
360 SymbolicEncodedGraph encodedGraph = new SymbolicEncodedGraph(snippetEncoding, startOffset, snippetObjects, snippetNodeClasses, method.getDeclaringClass(),
361 originalMethods.get(methodKey(method)));
362 return decodeSnippetGraph(encodedGraph, method, replacements, args, HotSpotJVMCIRuntime.runtime().getHostJVMCIBackend().getTarget().arch);
363 }
364
365 }
366
367 private StructuredGraph buildGraph(ResolvedJavaMethod method, ResolvedJavaMethod original, Object receiver, boolean requireInlining, boolean trackNodeSourcePosition) {
368 assert method.hasBytecodes() : "Snippet must not be abstract or native";
369 Object[] args = null;
370 if (receiver != null) {
371 args = new Object[method.getSignature().getParameterCount(true)];
372 args[0] = receiver;
373 }
374 try (DebugContext debug = openDebugContext("Snippet_", method)) {
375 StructuredGraph graph = replacements.makeGraph(debug, replacements.getDefaultReplacementBytecodeProvider(), method, args, original, trackNodeSourcePosition, null);
376
377 // Check if all methods which should be inlined are really inlined.
378 for (MethodCallTargetNode callTarget : graph.getNodes(MethodCallTargetNode.TYPE)) {
379 ResolvedJavaMethod callee = callTarget.targetMethod();
380 if (requireInlining && !delayedInvocationPluginMethods.contains(callee) && !Objects.equals(callee, original)) {
381 throw GraalError.shouldNotReachHere("method " + callee.format("%H.%n") + " not inlined in snippet " + method.getName() + " (maybe not final?)");
382 }
383 }
384 assert verifySnippetEncodeDecode(method, original, trackNodeSourcePosition, graph);
385 debug.dump(DebugContext.VERBOSE_LEVEL, graph, "After buildGraph");
386 return graph;
387 }
388 }
389
390 @SuppressWarnings("try")
391 static StructuredGraph decodeSnippetGraph(SymbolicEncodedGraph encodedGraph, ResolvedJavaMethod method, ReplacementsImpl replacements, Object[] args, Architecture architecture) {
392 Providers providers = replacements.getProviders();
393 ParameterPlugin parameterPlugin = null;
394 if (args != null) {
395 parameterPlugin = new ConstantBindingParameterPlugin(args, providers.getMetaAccess(), replacements.snippetReflection);
396 }
397
398 try (DebugContext debug = replacements.openDebugContext("SVMSnippet_", method)) {
399 // @formatter:off
400 StructuredGraph result = new StructuredGraph.Builder(replacements.getOptions(), debug)
401 .method(method)
402 .trackNodeSourcePosition(encodedGraph.trackNodeSourcePosition())
403 .setIsSubstitution(true)
404 .build();
405 // @formatter:on
406 try (DebugContext.Scope scope = debug.scope("DecodeSnippetGraph", result)) {
407 PEGraphDecoder graphDecoder = new PEGraphDecoder(
408 architecture,
409 result,
410 providers.getMetaAccess(),
411 providers.getConstantReflection(),
412 providers.getConstantFieldProvider(),
413 providers.getStampProvider(),
414 null,
415 replacements.getGraphBuilderPlugins().getInvocationPlugins(),
416 new InlineInvokePlugin[0],
417 parameterPlugin,
418 null,
419 null,
420 null) {
421 @Override
422 protected EncodedGraph lookupEncodedGraph(
423 ResolvedJavaMethod lookupMethod,
424 ResolvedJavaMethod originalMethod,
425 BytecodeProvider intrinsicBytecodeProvider,
426 boolean isSubstitution,
427 boolean track) {
428 if (lookupMethod.equals(method)) {
429 assert !track || encodedGraph.trackNodeSourcePosition();
430 return encodedGraph;
431 } else {
432 throw GraalError.shouldNotReachHere(method.format("%H.%n(%p)"));
433 }
434 }
435 };
436
437 graphDecoder.decode(method, result.isSubstitution(), encodedGraph.trackNodeSourcePosition());
438 debug.dump(DebugContext.VERBOSE_LEVEL, result, "After decoding");
439
440 assert result.verify();
441 return result;
442 } catch (Throwable t) {
443 throw debug.handle(t);
444 }
445 }
446 }
447
448 @SuppressWarnings("try")
449 private boolean verifySnippetEncodeDecode(ResolvedJavaMethod method, ResolvedJavaMethod original, boolean trackNodeSourcePosition, StructuredGraph structuredGraph) {
450 // Verify the encoding and decoding process
451 EncodedGraph encodedGraph = GraphEncoder.encodeSingleGraph(structuredGraph, HotSpotJVMCIRuntime.runtime().getHostJVMCIBackend().getTarget().arch);
452
453 Architecture arch = HotSpotJVMCIRuntime.runtime().getHostJVMCIBackend().getTarget().arch;
454
455 try (DebugContext debug = replacements.openDebugContext("VerifySnippetEncodeDecode_", method)) {
456 HotSpotProviders originalProvider = (HotSpotProviders) replacements.getProviders();
457
458 SnippetReflectionProvider snippetReflection = originalProvider.getSnippetReflection();
459 SymbolicSnippetEncoder.HotSpotSubstrateConstantReflectionProvider constantReflection = new SymbolicSnippetEncoder.HotSpotSubstrateConstantReflectionProvider(
460 originalProvider.getConstantReflection());
461 HotSpotProviders newProviders = new HotSpotProviders(originalProvider.getMetaAccess(), originalProvider.getCodeCache(), constantReflection,
462 originalProvider.getConstantFieldProvider(), originalProvider.getForeignCalls(), originalProvider.getLowerer(), null, originalProvider.getSuites(),
463 originalProvider.getRegisters(), snippetReflection, originalProvider.getWordTypes(), originalProvider.getGraphBuilderPlugins());
464 HotSpotSnippetReplacementsImpl filteringReplacements = new HotSpotSnippetReplacementsImpl(getOptions(), newProviders, snippetReflection,
465 originalProvider.getReplacements().getDefaultReplacementBytecodeProvider(),
466 originalProvider.getCodeCache().getTarget());
467 filteringReplacements.setGraphBuilderPlugins(originalProvider.getReplacements().getGraphBuilderPlugins());
468 try (DebugContext.Scope scaope = debug.scope("VerifySnippetEncodeDecode", structuredGraph)) {
469 for (int i = 0; i < encodedGraph.getNumObjects(); i++) {
470 filterSnippetObject(encodedGraph.getObject(i));
471 }
472 StructuredGraph snippet = filteringReplacements.makeGraph(debug, filteringReplacements.getDefaultReplacementBytecodeProvider(), method, null, original,
473 trackNodeSourcePosition, null);
474 SymbolicEncodedGraph symbolicGraph = new SymbolicEncodedGraph(encodedGraph, method.getDeclaringClass(), original != null ? methodKey(original) : null);
475 StructuredGraph decodedSnippet = decodeSnippetGraph(symbolicGraph, method, replacements, null, arch);
476 String snippetString = getCanonicalGraphString(snippet, true, false);
477 String decodedSnippetString = getCanonicalGraphString(decodedSnippet, true, false);
478 if (snippetString.equals(decodedSnippetString)) {
479 debug.log("Snippet decode for %s produces exactly same graph", method);
480 debug.dump(DebugContext.INFO_LEVEL, decodedSnippet, "Decoded snippet graph for %s", method);
481 } else {
482 debug.log("Snippet decode for %s produces different graph", method);
483 debug.log("%s", compareGraphStrings(snippet, snippetString, decodedSnippet, decodedSnippetString));
484 debug.dump(DebugContext.INFO_LEVEL, snippet, "Snippet graph for %s", method);
485 debug.dump(DebugContext.INFO_LEVEL, structuredGraph, "Encoded snippet graph for %s", method);
486 debug.dump(DebugContext.INFO_LEVEL, decodedSnippet, "Decoded snippet graph for %s", method);
487 }
488 } catch (Throwable t) {
489 throw debug.handle(t);
490 }
491 }
492 return true;
493 }
494
495 /**
496 * If there are new graphs waiting to be encoded, reencode all the graphs and return the result.
497 */
498 @SuppressWarnings("try")
499 synchronized EncodedSnippets maybeEncodeSnippets() {
500 Map<String, StructuredGraph> graphs = this.preparedSnippetGraphs;
501 if (encodedGraphs != graphs.size()) {
502 DebugContext debug = openDebugContext("SnippetEncoder", null);
503 try (DebugContext.Scope scope = debug.scope("SnippetSupportEncode")) {
504 encodedGraphs = graphs.size();
505 for (StructuredGraph graph : graphs.values()) {
506 for (Node node : graph.getNodes()) {
507 node.setNodeSourcePosition(null);
508 }
509 }
510 return encodeSnippets(debug);
511 }
512 }
513 return null;
514 }
515
516 @Override
517 public void registerSnippet(ResolvedJavaMethod method, ResolvedJavaMethod original, Object receiver, boolean trackNodeSourcePosition) {
518 if (IS_BUILDING_NATIVE_IMAGE || UseEncodedSnippets.getValue(getOptions())) {
519 assert method.getAnnotation(Snippet.class) != null : "Snippet must be annotated with @" + Snippet.class.getSimpleName();
520 String key = methodKey(method);
521 if (!preparedSnippetGraphs.containsKey(key)) {
522 if (original != null) {
523 originalMethods.put(key, methodKey(original));
524 }
525 StructuredGraph snippet = buildGraph(method, original, receiver, true, trackNodeSourcePosition);
526 snippetMethods.add(method);
527 preparedSnippetGraphs.put(key, snippet);
528 }
529 }
530 }
531
532 EncodedSnippets encodeSnippets(DebugContext debug) {
533 GraphEncoder encoder = new GraphEncoder(HotSpotJVMCIRuntime.runtime().getHostJVMCIBackend().getTarget().arch, debug);
534 for (StructuredGraph graph : preparedSnippetGraphs.values()) {
535 encoder.prepare(graph);
536 }
537 encoder.finishPrepare();
538
539 byte[] snippetEncoding;
540 Object[] snippetObjects;
541 NodeClass<?>[] snippetNodeClasses;
542 Map<String, Integer> snippetStartOffsets;
543
544 snippetStartOffsets = new HashMap<>();
545 for (Map.Entry<String, StructuredGraph> entry : preparedSnippetGraphs.entrySet()) {
546 snippetStartOffsets.put(entry.getKey(), encoder.encode(entry.getValue()));
547 }
548 snippetEncoding = encoder.getEncoding();
549 snippetObjects = encoder.getObjects();
550 snippetNodeClasses = encoder.getNodeClasses();
551 for (int i = 0; i < snippetObjects.length; i++) {
552 Object o = filterSnippetObject(snippetObjects[i]);
553 debug.log("snippetObjects[%d] = %s -> %s", i, o != null ? o.getClass().getSimpleName() : null, o);
554 snippetObjects[i] = o;
555 }
556 debug.log("Encoded %d snippet preparedSnippetGraphs using %d bytes with %d objects", snippetStartOffsets.size(), snippetEncoding.length, snippetObjects.length);
557 return new EncodedSnippets(snippetEncoding, snippetObjects, snippetNodeClasses, snippetStartOffsets, originalMethods);
558 }
559
560 /**
561 * Encode any outstanding graphs and return true if any work was done.
562 */
563 @SuppressWarnings("try")
564 public boolean encode() {
565 EncodedSnippets encodedSnippets = maybeEncodeSnippets();
566 if (encodedSnippets != null) {
567 HotSpotReplacementsImpl.setEncodedSnippets(encodedSnippets);
568 return true;
569 }
570 return false;
571 }
572
573 private DebugContext openDebugContext(String idPrefix, ResolvedJavaMethod method) {
574 return replacements.openDebugContext(idPrefix, method);
575 }
576
577 static class SymbolicEncodedGraph extends EncodedGraph {
578
579 private final ResolvedJavaType accessingClass;
580 private final String originalMethod;
581
582 SymbolicEncodedGraph(byte[] encoding, int startOffset, Object[] objects, NodeClass<?>[] types, ResolvedJavaType accessingClass, String originalMethod) {
583 super(encoding, startOffset, objects, types, null, null, null, false, false);
584 this.accessingClass = accessingClass;
585 this.originalMethod = originalMethod;
586 }
587
588 SymbolicEncodedGraph(EncodedGraph encodedGraph, ResolvedJavaType declaringClass, String originalMethod) {
589 this(encodedGraph.getEncoding(), encodedGraph.getStartOffset(), encodedGraph.getObjects(), encodedGraph.getNodeClasses(), declaringClass, originalMethod);
590 }
591
592 @Override
593 public Object getObject(int i) {
594 Object o = objects[i];
595 if (o instanceof SymbolicJVMCIReference) {
596 objects[i] = o = ((SymbolicJVMCIReference<?>) o).resolve(accessingClass);
597 } else if (o instanceof UnresolvedJavaType) {
598 objects[i] = o = ((UnresolvedJavaType) o).resolve(accessingClass);
599 } else if (o instanceof UnresolvedJavaMethod) {
600 throw new InternalError(o.toString());
601 } else if (o instanceof UnresolvedJavaField) {
602 objects[i] = o = ((UnresolvedJavaField) o).resolve(accessingClass);
603 } else if (o instanceof GraalCapability) {
604 objects[i] = o = ((GraalCapability) o).resolve(((GraalJVMCICompiler) getRuntime().getCompiler()).getGraalRuntime());
605 }
606 return o;
607 }
608
609 @Override
610 public boolean isCallToOriginal(ResolvedJavaMethod callTarget) {
611 if (originalMethod != null && originalMethod.equals(methodKey(callTarget))) {
612 return true;
613 }
614 return super.isCallToOriginal(callTarget);
615 }
616 }
617
618 /**
619 * Symbolic reference to an object which can be retrieved from
620 * {@link GraalRuntime#getCapability(Class)}.
621 */
622 static class GraalCapability {
623 final Class<?> capabilityClass;
624
625 GraalCapability(Class<?> capabilityClass) {
626 this.capabilityClass = capabilityClass;
627 }
628
629 public Object resolve(GraalRuntime runtime) {
630 Object capability = runtime.getCapability(this.capabilityClass);
631 if (capability != null) {
632 assert capability.getClass() == capabilityClass;
633 return capability;
634 }
635 throw new InternalError(this.capabilityClass.getName());
636 }
637 }
638
639 static class SymbolicResolvedJavaMethod implements SymbolicJVMCIReference<ResolvedJavaMethod> {
640 final UnresolvedJavaType type;
641 final String methodName;
642 final String signature;
643
644 SymbolicResolvedJavaMethod(HotSpotResolvedJavaMethod method) {
645 this.type = UnresolvedJavaType.create(method.getDeclaringClass().getName());
646 this.methodName = method.getName();
647 this.signature = method.getSignature().toMethodDescriptor();
648 }
649
650 @Override
651 public String toString() {
652 return "SymbolicResolvedJavaMethod{" +
653 "declaringType='" + type.getName() + '\'' +
654 ", methodName='" + methodName + '\'' +
655 ", signature='" + signature + '\'' +
656 '}';
657 }
658
659 @Override
660 public ResolvedJavaMethod resolve(ResolvedJavaType accessingClass) {
661 ResolvedJavaType resolvedType = type.resolve(accessingClass);
662 for (ResolvedJavaMethod method : methodName.equals("<init>") ? resolvedType.getDeclaredConstructors() : resolvedType.getDeclaredMethods()) {
663 if (method.getName().equals(methodName) && method.getSignature().toMethodDescriptor().equals(signature)) {
664 return method;
665 }
666 }
667 throw new InternalError("Could not resolve " + this + " in context of " + accessingClass.toJavaName());
668 }
669 }
670
671 static class SymbolicResolvedJavaField implements SymbolicJVMCIReference<ResolvedJavaField> {
672 final UnresolvedJavaType declaringType;
673 final String name;
674 final UnresolvedJavaType signature;
675 private final boolean isStatic;
676
677 SymbolicResolvedJavaField(HotSpotResolvedJavaField field) {
678 this.declaringType = UnresolvedJavaType.create(field.getDeclaringClass().getName());
679 this.name = field.getName();
680 this.signature = UnresolvedJavaType.create(field.getType().getName());
681 this.isStatic = field.isStatic();
682 }
683
684 @Override
685 public ResolvedJavaField resolve(ResolvedJavaType accessingClass) {
686 ResolvedJavaType resolvedType = declaringType.resolve(accessingClass);
687 ResolvedJavaType resolvedFieldType = signature.resolve(accessingClass);
688 ResolvedJavaField[] fields = isStatic ? resolvedType.getStaticFields() : resolvedType.getInstanceFields(true);
689 for (ResolvedJavaField field : fields) {
690 if (field.getName().equals(name)) {
691 if (field.getType().equals(resolvedFieldType)) {
692 return field;
693 }
694 }
695 }
696 throw new InternalError("Could not resolve " + this + " in context of " + accessingClass.toJavaName());
697 }
698
699 @Override
700 public String toString() {
701 return "SymbolicResolvedJavaField{" +
702 signature.getName() + ' ' +
703 declaringType.getName() + '.' +
704 name +
705 '}';
706 }
707 }
708
709 static class SymbolicStampPair implements SymbolicJVMCIReference<StampPair> {
710 Object trustedStamp;
711 Object uncheckdStamp;
712
713 SymbolicStampPair(StampPair stamp) {
714 this.trustedStamp = maybeMakeSymbolic(stamp.getTrustedStamp());
715 this.uncheckdStamp = maybeMakeSymbolic(stamp.getUncheckedStamp());
716 }
717
718 @Override
719 public StampPair resolve(ResolvedJavaType accessingClass) {
720 return StampPair.create(resolveStamp(accessingClass, trustedStamp), resolveStamp(accessingClass, uncheckdStamp));
721 }
722 }
723
724 private static Object maybeMakeSymbolic(Stamp trustedStamp) {
725 if (trustedStamp != null) {
726 SymbolicJVMCIReference<?> symbolicJVMCIReference = trustedStamp.makeSymbolic();
727 if (symbolicJVMCIReference != null) {
728 return symbolicJVMCIReference;
729 }
730 }
731 return trustedStamp;
732 }
733
734 private static Stamp resolveStamp(ResolvedJavaType accessingClass, Object stamp) {
735 if (stamp == null) {
736 return null;
737 }
738 if (stamp instanceof Stamp) {
739 return (Stamp) stamp;
740 }
741 return (Stamp) ((SymbolicJVMCIReference<?>) stamp).resolve(accessingClass);
742 }
743
744 public static class HotSpotSubstrateConstantReflectionProvider implements ConstantReflectionProvider {
745
746 private final ConstantReflectionProvider constantReflection;
747
748 HotSpotSubstrateConstantReflectionProvider(ConstantReflectionProvider constantReflection) {
749 this.constantReflection = constantReflection;
750 }
751
752 HashSet<JavaConstant> safeConstants = new HashSet<>();
753
754 @Override
755 public Boolean constantEquals(Constant x, Constant y) {
756 return constantReflection.constantEquals(x, y);
757 }
758
759 @Override
760 public Integer readArrayLength(JavaConstant array) {
761 return constantReflection.readArrayLength(array);
762 }
763
764 @Override
765 public JavaConstant readArrayElement(JavaConstant array, int index) {
766 return constantReflection.readArrayElement(array, index);
767 }
768
769 @Override
770 public JavaConstant readFieldValue(ResolvedJavaField field, JavaConstant receiver) {
771 JavaConstant javaConstant = constantReflection.readFieldValue(field, receiver);
772 if (!safeConstants.contains(receiver) && !field.getDeclaringClass().getName().contains("graalvm") && !field.getDeclaringClass().getName().contains("jdk/vm/ci/") &&
773 !field.getName().equals("TYPE")) {
774 // Only permit constant reflection on compiler classes. This is necessary primarily
775 // because of the boxing snippets which are compiled as snippets but are really just
776 // regular JDK java sources that are being compiled like a snippet. These shouldn't
777 // permit constant folding during graph preparation as that embeds constants from
778 // the runtime into a compiler graph.
779 return null;
780 }
781 if (javaConstant.getJavaKind() == JavaKind.Object) {
782 safeConstants.add(javaConstant);
783 }
784 return javaConstant;
785 }
786
787 @Override
788 public JavaConstant boxPrimitive(JavaConstant source) {
789 return constantReflection.boxPrimitive(source);
790 }
791
792 @Override
793 public JavaConstant unboxPrimitive(JavaConstant source) {
794 return constantReflection.unboxPrimitive(source);
795 }
796
797 @Override
798 public JavaConstant forString(String value) {
799 return constantReflection.forString(value);
800 }
801
802 @Override
803 public ResolvedJavaType asJavaType(Constant constant) {
804 return constantReflection.asJavaType(constant);
805 }
806
807 @Override
808 public MethodHandleAccessProvider getMethodHandleAccess() {
809 return constantReflection.getMethodHandleAccess();
810 }
811
812 @Override
813 public MemoryAccessProvider getMemoryAccessProvider() {
814 return constantReflection.getMemoryAccessProvider();
815 }
816
817 @Override
818 public JavaConstant asJavaClass(ResolvedJavaType type) {
819 return constantReflection.asJavaClass(type);
820 }
821
822 @Override
823 public Constant asObjectHub(ResolvedJavaType type) {
824 return constantReflection.asObjectHub(type);
825 }
826 }
827
828 /**
829 * Objects embedded in encoded graphs might need to converted into a symbolic form so convert
830 * the object or pass it through.
831 */
832 static Object filterSnippetObject(Object o) {
833 if (o instanceof HotSpotResolvedJavaMethod) {
834 return new SymbolicResolvedJavaMethod((HotSpotResolvedJavaMethod) o);
835 } else if (o instanceof HotSpotResolvedJavaField) {
836 return new SymbolicResolvedJavaField((HotSpotResolvedJavaField) o);
837 } else if (o instanceof HotSpotResolvedJavaType) {
838 return UnresolvedJavaType.create(((HotSpotResolvedJavaType) o).getName());
839 } else if (o instanceof NodeSourcePosition) {
840 // Filter these out for now. These can't easily be handled because these positions
841 // description snippet methods which might not be available in the runtime.
842 return null;
843 } else if (o instanceof HotSpotForeignCallsProvider || o instanceof GraalHotSpotVMConfig) {
844 return new GraalCapability(o.getClass());
845 } else if (o instanceof Stamp) {
846 SymbolicJVMCIReference<?> ref = ((Stamp) o).makeSymbolic();
847 if (ref != null) {
848 return ref;
849 }
850 return o;
851 } else if (o instanceof StampPair) {
852 if (((StampPair) o).getTrustedStamp() instanceof AbstractObjectStamp) {
853 return new SymbolicStampPair((StampPair) o);
854 }
855 }
856 return o;
857 }
858
859 static String compareGraphStrings(StructuredGraph expectedGraph, String expectedString, StructuredGraph actualGraph, String actualString) {
860 if (!expectedString.equals(actualString)) {
861 String[] expectedLines = expectedString.split("\n");
862 String[] actualLines = actualString.split("\n");
863 int diffIndex = -1;
864 int limit = Math.min(actualLines.length, expectedLines.length);
865 String marker = " <<<";
866 for (int i = 0; i < limit; i++) {
867 if (!expectedLines[i].equals(actualLines[i])) {
868 diffIndex = i;
869 break;
870 }
871 }
872 if (diffIndex == -1) {
873 // Prefix is the same so add some space after the prefix
874 diffIndex = limit;
875 if (actualLines.length == limit) {
876 actualLines = Arrays.copyOf(actualLines, limit + 1);
877 actualLines[diffIndex] = "";
878 } else {
879 assert expectedLines.length == limit;
880 expectedLines = Arrays.copyOf(expectedLines, limit + 1);
881 expectedLines[diffIndex] = "";
882 }
883 }
884 // Place a marker next to the first line that differs
885 expectedLines[diffIndex] = expectedLines[diffIndex] + marker;
886 actualLines[diffIndex] = actualLines[diffIndex] + marker;
887 String ediff = String.join("\n", expectedLines);
888 String adiff = String.join("\n", actualLines);
889 return "mismatch in preparedSnippetGraphs:\n========= expected (" + expectedGraph + ") =========\n" + ediff + "\n\n========= actual (" + actualGraph + ") =========\n" + adiff;
890 } else {
891 return "mismatch in preparedSnippetGraphs";
892 }
893 }
894
895 static String getCanonicalGraphString(StructuredGraph graph, boolean excludeVirtual, boolean checkConstants) {
896 SchedulePhase schedule = new SchedulePhase(SchedulePhase.SchedulingStrategy.EARLIEST);
897 schedule.apply(graph);
898 StructuredGraph.ScheduleResult scheduleResult = graph.getLastSchedule();
899
900 NodeMap<Integer> canonicalId = graph.createNodeMap();
901 int nextId = 0;
902
903 List<String> constantsLines = new ArrayList<>();
904
905 StringBuilder result = new StringBuilder();
906 for (Block block : scheduleResult.getCFG().getBlocks()) {
907 result.append("Block ").append(block).append(' ');
908 if (block == scheduleResult.getCFG().getStartBlock()) {
909 result.append("* ");
910 }
911 result.append("-> ");
912 for (Block succ : block.getSuccessors()) {
913 result.append(succ).append(' ');
914 }
915 result.append('\n');
916 for (Node node : scheduleResult.getBlockToNodesMap().get(block)) {
917 if (node instanceof ValueNode && node.isAlive()) {
918 if (!excludeVirtual || !(node instanceof VirtualObjectNode || node instanceof ProxyNode || node instanceof FullInfopointNode || node instanceof ParameterNode)) {
919 if (node instanceof ConstantNode) {
920 if (checkConstants) {
921 String name = node.toString(Verbosity.Name);
922 if (excludeVirtual) {
923 constantsLines.add(name);
924 } else {
925 constantsLines.add(name + " (" + filteredUsageCount(node) + ")");
926 }
927 }
928 } else {
929 int id;
930 if (canonicalId.get(node) != null) {
931 id = canonicalId.get(node);
932 } else {
933 id = nextId++;
934 canonicalId.set(node, id);
935 }
936 String name = node.getClass().getSimpleName();
937 result.append(" ").append(id).append('|').append(name);
938 if (node instanceof AccessFieldNode) {
939 result.append('#');
940 result.append(((AccessFieldNode) node).field());
941 }
942 if (!excludeVirtual) {
943 result.append(" (");
944 result.append(filteredUsageCount(node));
945 result.append(')');
946 }
947 result.append('\n');
948 }
949 }
950 }
951 }
952 }
953
954 StringBuilder constantsLinesResult = new StringBuilder();
955 if (checkConstants) {
956 constantsLinesResult.append(constantsLines.size()).append(" constants:\n");
957 }
958 Collections.sort(constantsLines);
959 for (String s : constantsLines) {
960 constantsLinesResult.append(s);
961 constantsLinesResult.append('\n');
962 }
963
964 return constantsLinesResult.toString() + result.toString();
965 }
966
967 private static int filteredUsageCount(Node node) {
968 return node.usages().filter(n -> !(n instanceof FrameState)).count();
969 }
970
971 /**
972 * This horror show of classes exists solely get {@link HotSpotSnippetBytecodeParser} to be used
973 * as the parser for these snippets.
974 */
975 static class HotSpotSnippetReplacementsImpl extends HotSpotReplacementsImpl {
976 HotSpotSnippetReplacementsImpl(HotSpotReplacementsImpl replacements, Providers providers) {
977 super(replacements, providers);
978 }
979
980 HotSpotSnippetReplacementsImpl(OptionValues options, Providers providers, SnippetReflectionProvider snippetReflection, BytecodeProvider bytecodeProvider, TargetDescription target) {
981 super(options, providers, snippetReflection, bytecodeProvider, target);
982 }
983
984 @Override
985 protected GraphMaker createGraphMaker(ResolvedJavaMethod substitute, ResolvedJavaMethod original) {
986 return new SnippetGraphMaker(this, substitute, original);
987 }
988 }
989
990 static class SnippetGraphMaker extends ReplacementsImpl.GraphMaker {
991 SnippetGraphMaker(ReplacementsImpl replacements, ResolvedJavaMethod substitute, ResolvedJavaMethod substitutedMethod) {
992 super(replacements, substitute, substitutedMethod);
993 }
994
995 @Override
996 protected GraphBuilderPhase.Instance createGraphBuilder(MetaAccessProvider metaAccess, StampProvider stampProvider, ConstantReflectionProvider constantReflection,
997 ConstantFieldProvider constantFieldProvider, GraphBuilderConfiguration graphBuilderConfig, OptimisticOptimizations optimisticOpts, IntrinsicContext initialIntrinsicContext) {
998 return new HotSpotSnippetGraphBuilderPhase(metaAccess, stampProvider, constantReflection, constantFieldProvider, graphBuilderConfig, optimisticOpts,
999 initialIntrinsicContext);
1000 }
1001 }
1002
1003 static class HotSpotSnippetGraphBuilderPhase extends GraphBuilderPhase.Instance {
1004 HotSpotSnippetGraphBuilderPhase(MetaAccessProvider metaAccess, StampProvider stampProvider, ConstantReflectionProvider constantReflection, ConstantFieldProvider constantFieldProvider,
1005 GraphBuilderConfiguration graphBuilderConfig, OptimisticOptimizations optimisticOpts, IntrinsicContext initialIntrinsicContext) {
1006 super(metaAccess, stampProvider, constantReflection, constantFieldProvider, graphBuilderConfig, optimisticOpts, initialIntrinsicContext);
1007 }
1008
1009 @Override
1010 protected BytecodeParser createBytecodeParser(StructuredGraph graph, BytecodeParser parent, ResolvedJavaMethod method, int entryBCI, IntrinsicContext intrinsicContext) {
1011 return new HotSpotSnippetBytecodeParser(this, graph, parent, method, entryBCI, intrinsicContext);
1012 }
1013 }
1014
1015 static class HotSpotSnippetBytecodeParser extends BytecodeParser {
1016 HotSpotSnippetBytecodeParser(GraphBuilderPhase.Instance graphBuilderInstance, StructuredGraph graph, BytecodeParser parent, ResolvedJavaMethod method, int entryBCI,
1017 IntrinsicContext intrinsicContext) {
1018 super(graphBuilderInstance, graph, parent, method, entryBCI, intrinsicContext);
1019 }
1020
1021 @Override
1022 public boolean canDeferPlugin(GeneratedInvocationPlugin plugin) {
1023 return plugin.getSource().equals(Fold.class) || plugin.getSource().equals(Node.NodeIntrinsic.class);
1024 }
1025
1026 @Override
1027 protected boolean tryInvocationPlugin(CallTargetNode.InvokeKind invokeKind, ValueNode[] args, ResolvedJavaMethod targetMethod, JavaKind resultType, JavaType returnType) {
1028 if (intrinsicContext != null && intrinsicContext.isCallToOriginal(targetMethod)) {
1029 return false;
1030 }
1031 return super.tryInvocationPlugin(invokeKind, args, targetMethod, resultType, returnType);
1032 }
1033 }
1034 }