1 /*
2 * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23 package org.graalvm.compiler.hotspot.replacements.profiling;
24
25 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG;
26 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.config;
27 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
28 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;
29 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER;
30
31 import org.graalvm.compiler.api.replacements.Snippet;
32 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter;
33 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor;
34 import org.graalvm.compiler.debug.DebugHandlersFactory;
35 import org.graalvm.compiler.debug.GraalError;
36 import org.graalvm.compiler.graph.Node.ConstantNodeParameter;
37 import org.graalvm.compiler.graph.Node.NodeIntrinsic;
38 import org.graalvm.compiler.hotspot.HotSpotBackend;
39 import org.graalvm.compiler.hotspot.meta.HotSpotProviders;
40 import org.graalvm.compiler.hotspot.nodes.aot.LoadMethodCountersNode;
41 import org.graalvm.compiler.hotspot.nodes.profiling.ProfileBranchNode;
42 import org.graalvm.compiler.hotspot.nodes.profiling.ProfileInvokeNode;
43 import org.graalvm.compiler.hotspot.nodes.profiling.ProfileNode;
44 import org.graalvm.compiler.hotspot.word.MethodCountersPointer;
45 import org.graalvm.compiler.nodes.ConstantNode;
46 import org.graalvm.compiler.nodes.StructuredGraph;
47 import org.graalvm.compiler.nodes.extended.ForeignCallNode;
48 import org.graalvm.compiler.nodes.spi.LoweringTool;
49 import org.graalvm.compiler.nodes.util.GraphUtil;
50 import org.graalvm.compiler.options.OptionValues;
51 import org.graalvm.compiler.replacements.SnippetTemplate;
52 import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates;
53 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
54 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
55 import org.graalvm.compiler.replacements.Snippets;
56
57 import jdk.vm.ci.code.CodeUtil;
58 import jdk.vm.ci.code.TargetDescription;
59
60 public class ProbabilisticProfileSnippets implements Snippets {
61 @Snippet
62 public static boolean shouldProfile(@ConstantParameter int probLog, int random) {
63 int probabilityMask = (1 << probLog) - 1;
64 return (random & probabilityMask) == 0;
65 }
66
67 @Snippet
68 public static int notificationMask(int freqLog, int probLog, int stepLog) {
69 int frequencyMask = (1 << freqLog) - 1;
70 int stepMask = (1 << (stepLog + probLog)) - 1;
71 return frequencyMask & ~stepMask;
72 }
73
74 @NodeIntrinsic(ForeignCallNode.class)
75 public static native void methodInvocationEvent(@ConstantNodeParameter ForeignCallDescriptor descriptor, MethodCountersPointer counters);
76
77 @Snippet
78 public static void profileMethodEntryWithProbability(MethodCountersPointer counters, int random, int step, int stepLog, @ConstantParameter int freqLog, @ConstantParameter int probLog) {
79 if (probability(1.0 / (1 << probLog), shouldProfile(probLog, random))) {
80 int counterValue = counters.readInt(config(INJECTED_VMCONFIG).invocationCounterOffset) + ((config(INJECTED_VMCONFIG).invocationCounterIncrement * step) << probLog);
81 counters.writeInt(config(INJECTED_VMCONFIG).invocationCounterOffset, counterValue);
82 if (freqLog >= 0) {
83 int mask = notificationMask(freqLog, probLog, stepLog);
84 if (probability(SLOW_PATH_PROBABILITY, (counterValue & (mask << config(INJECTED_VMCONFIG).invocationCounterShift)) == 0)) {
85 methodInvocationEvent(HotSpotBackend.INVOCATION_EVENT, counters);
86 }
87 }
88 }
89 }
90
91 @NodeIntrinsic(ForeignCallNode.class)
92 public static native void methodBackedgeEvent(@ConstantNodeParameter ForeignCallDescriptor descriptor, MethodCountersPointer counters, int bci, int targetBci);
93
94 @Snippet
95 public static void profileBackedgeWithProbability(MethodCountersPointer counters, int random, int step, int stepLog, @ConstantParameter int freqLog, @ConstantParameter int probLog, int bci,
96 int targetBci) {
97 if (probability(1.0 / (1 << probLog), shouldProfile(probLog, random))) {
98 int counterValue = counters.readInt(config(INJECTED_VMCONFIG).backedgeCounterOffset) + ((config(INJECTED_VMCONFIG).invocationCounterIncrement * step) << probLog);
99 counters.writeInt(config(INJECTED_VMCONFIG).backedgeCounterOffset, counterValue);
100 int mask = notificationMask(freqLog, probLog, stepLog);
101 if (probability(SLOW_PATH_PROBABILITY, (counterValue & (mask << config(INJECTED_VMCONFIG).invocationCounterShift)) == 0)) {
102 methodBackedgeEvent(HotSpotBackend.BACKEDGE_EVENT, counters, bci, targetBci);
103 }
104 }
105 }
106
107 @Snippet
108 public static void profileConditionalBackedgeWithProbability(MethodCountersPointer counters, int random, int step, int stepLog, @ConstantParameter int freqLog,
109 @ConstantParameter int probLog, boolean branchCondition,
110 int bci, int targetBci) {
111 if (branchCondition) {
112 profileBackedgeWithProbability(counters, random, step, stepLog, freqLog, probLog, bci, targetBci);
113 }
114 }
115
116 public static class Templates extends AbstractTemplates {
117 private final SnippetInfo profileMethodEntryWithProbability = snippet(ProbabilisticProfileSnippets.class, "profileMethodEntryWithProbability");
118 private final SnippetInfo profileBackedgeWithProbability = snippet(ProbabilisticProfileSnippets.class, "profileBackedgeWithProbability");
119 private final SnippetInfo profileConditionalBackedgeWithProbability = snippet(ProbabilisticProfileSnippets.class, "profileConditionalBackedgeWithProbability");
120
121 public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, HotSpotProviders providers, TargetDescription target) {
122 super(options, factories, providers, providers.getSnippetReflection(), target);
123 }
124
125 public void lower(ProfileNode profileNode, LoweringTool tool) {
126 assert profileNode.getRandom() != null;
127
128 StructuredGraph graph = profileNode.graph();
129 LoadMethodCountersNode counters = graph.unique(new LoadMethodCountersNode(profileNode.getProfiledMethod()));
130 ConstantNode step = ConstantNode.forInt(profileNode.getStep(), graph);
131 ConstantNode stepLog = ConstantNode.forInt(CodeUtil.log2(profileNode.getStep()), graph);
132
133 if (profileNode instanceof ProfileBranchNode) {
134 // Backedge event
135 ProfileBranchNode profileBranchNode = (ProfileBranchNode) profileNode;
136 SnippetInfo snippet = profileBranchNode.hasCondition() ? profileConditionalBackedgeWithProbability : profileBackedgeWithProbability;
137 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage());
138 ConstantNode bci = ConstantNode.forInt(profileBranchNode.bci(), graph);
139 ConstantNode targetBci = ConstantNode.forInt(profileBranchNode.targetBci(), graph);
140
141 args.add("counters", counters);
142 args.add("random", profileBranchNode.getRandom());
143 args.add("step", step);
144 args.add("stepLog", stepLog);
145 args.addConst("freqLog", profileBranchNode.getNotificationFreqLog());
146 args.addConst("probLog", profileBranchNode.getProbabilityLog());
147 if (profileBranchNode.hasCondition()) {
148 args.add("branchCondition", profileBranchNode.branchCondition());
149 }
150 args.add("bci", bci);
151 args.add("targetBci", targetBci);
152
153 SnippetTemplate template = template(graph.getDebug(), args);
154 template.instantiate(providers.getMetaAccess(), profileNode, DEFAULT_REPLACER, args);
155 } else if (profileNode instanceof ProfileInvokeNode) {
156 ProfileInvokeNode profileInvokeNode = (ProfileInvokeNode) profileNode;
157 // Method invocation event
158 Arguments args = new Arguments(profileMethodEntryWithProbability, graph.getGuardsStage(), tool.getLoweringStage());
159
160 args.add("counters", counters);
161 args.add("random", profileInvokeNode.getRandom());
162 args.add("step", step);
163 args.add("stepLog", stepLog);
164 args.addConst("freqLog", profileInvokeNode.getNotificationFreqLog());
165 args.addConst("probLog", profileInvokeNode.getProbabilityLog());
166 SnippetTemplate template = template(graph.getDebug(), args);
167 template.instantiate(providers.getMetaAccess(), profileNode, DEFAULT_REPLACER, args);
168 } else {
169 throw new GraalError("Unsupported profile node type: " + profileNode);
170 }
171
172 assert profileNode.hasNoUsages();
173 if (!profileNode.isDeleted()) {
174 GraphUtil.killWithUnusedFloatingInputs(profileNode);
175 }
176 }
177 }
178 }