1 /* 2 * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 25 package org.graalvm.compiler.hotspot.replacements.profiling; 26 27 import static org.graalvm.compiler.hotspot.GraalHotSpotVMConfig.INJECTED_VMCONFIG; 28 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.backedgeCounterOffset; 29 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.invocationCounterIncrement; 30 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.invocationCounterOffset; 31 import static org.graalvm.compiler.hotspot.replacements.HotSpotReplacementsUtil.invocationCounterShift; 32 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY; 33 import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability; 34 import static org.graalvm.compiler.replacements.SnippetTemplate.DEFAULT_REPLACER; 35 36 import org.graalvm.compiler.api.replacements.Snippet; 37 import org.graalvm.compiler.api.replacements.Snippet.ConstantParameter; 38 import org.graalvm.compiler.core.common.spi.ForeignCallDescriptor; 39 import org.graalvm.compiler.debug.DebugHandlersFactory; 40 import org.graalvm.compiler.debug.GraalError; 41 import org.graalvm.compiler.graph.Node.ConstantNodeParameter; 42 import org.graalvm.compiler.graph.Node.NodeIntrinsic; 43 import org.graalvm.compiler.hotspot.HotSpotBackend; 44 import org.graalvm.compiler.hotspot.meta.HotSpotProviders; 45 import org.graalvm.compiler.hotspot.nodes.aot.LoadMethodCountersNode; 46 import org.graalvm.compiler.hotspot.nodes.profiling.ProfileBranchNode; 47 import org.graalvm.compiler.hotspot.nodes.profiling.ProfileInvokeNode; 48 import org.graalvm.compiler.hotspot.nodes.profiling.ProfileNode; 49 import org.graalvm.compiler.hotspot.word.MethodCountersPointer; 50 import org.graalvm.compiler.nodes.ConstantNode; 51 import org.graalvm.compiler.nodes.StructuredGraph; 52 import org.graalvm.compiler.nodes.extended.ForeignCallNode; 53 import org.graalvm.compiler.nodes.spi.LoweringTool; 54 import org.graalvm.compiler.nodes.util.GraphUtil; 55 import org.graalvm.compiler.options.OptionValues; 56 import org.graalvm.compiler.replacements.SnippetTemplate; 57 import org.graalvm.compiler.replacements.SnippetTemplate.AbstractTemplates; 58 import org.graalvm.compiler.replacements.SnippetTemplate.Arguments; 59 import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo; 60 import org.graalvm.compiler.replacements.Snippets; 61 62 import jdk.vm.ci.code.CodeUtil; 63 import jdk.vm.ci.code.TargetDescription; 64 65 public class ProbabilisticProfileSnippets implements Snippets { 66 @Snippet 67 public static boolean shouldProfile(@ConstantParameter int probLog, int random) { 68 int probabilityMask = (1 << probLog) - 1; 69 return (random & probabilityMask) == 0; 70 } 71 72 @Snippet 73 public static int notificationMask(int freqLog, int probLog, int stepLog) { 74 int frequencyMask = (1 << freqLog) - 1; 75 int stepMask = (1 << (stepLog + probLog)) - 1; 76 return frequencyMask & ~stepMask; 77 } 78 79 @NodeIntrinsic(ForeignCallNode.class) 80 public static native void methodInvocationEvent(@ConstantNodeParameter ForeignCallDescriptor descriptor, MethodCountersPointer counters); 81 82 @Snippet 83 public static void profileMethodEntryWithProbability(MethodCountersPointer counters, int random, int step, int stepLog, @ConstantParameter int freqLog, @ConstantParameter int probLog) { 84 if (probability(1.0 / (1 << probLog), shouldProfile(probLog, random))) { 85 int counterValue = counters.readInt(invocationCounterOffset(INJECTED_VMCONFIG)) + ((invocationCounterIncrement(INJECTED_VMCONFIG) * step) << probLog); 86 counters.writeInt(invocationCounterOffset(INJECTED_VMCONFIG), counterValue); 87 if (freqLog >= 0) { 88 int mask = notificationMask(freqLog, probLog, stepLog); 89 if (probability(SLOW_PATH_PROBABILITY, (counterValue & (mask << invocationCounterShift(INJECTED_VMCONFIG))) == 0)) { 90 methodInvocationEvent(HotSpotBackend.INVOCATION_EVENT, counters); 91 } 92 } 93 } 94 } 95 96 @NodeIntrinsic(ForeignCallNode.class) 97 public static native void methodBackedgeEvent(@ConstantNodeParameter ForeignCallDescriptor descriptor, MethodCountersPointer counters, int bci, int targetBci); 98 99 @Snippet 100 public static void profileBackedgeWithProbability(MethodCountersPointer counters, int random, int step, int stepLog, @ConstantParameter int freqLog, @ConstantParameter int probLog, int bci, 101 int targetBci) { 102 if (probability(1.0 / (1 << probLog), shouldProfile(probLog, random))) { 103 int counterValue = counters.readInt(backedgeCounterOffset(INJECTED_VMCONFIG)) + ((invocationCounterIncrement(INJECTED_VMCONFIG) * step) << probLog); 104 counters.writeInt(backedgeCounterOffset(INJECTED_VMCONFIG), counterValue); 105 int mask = notificationMask(freqLog, probLog, stepLog); 106 if (probability(SLOW_PATH_PROBABILITY, (counterValue & (mask << invocationCounterShift(INJECTED_VMCONFIG))) == 0)) { 107 methodBackedgeEvent(HotSpotBackend.BACKEDGE_EVENT, counters, bci, targetBci); 108 } 109 } 110 } 111 112 @Snippet 113 public static void profileConditionalBackedgeWithProbability(MethodCountersPointer counters, int random, int step, int stepLog, @ConstantParameter int freqLog, 114 @ConstantParameter int probLog, boolean branchCondition, 115 int bci, int targetBci) { 116 if (branchCondition) { 117 profileBackedgeWithProbability(counters, random, step, stepLog, freqLog, probLog, bci, targetBci); 118 } 119 } 120 121 public static class Templates extends AbstractTemplates { 122 private final SnippetInfo profileMethodEntryWithProbability = snippet(ProbabilisticProfileSnippets.class, "profileMethodEntryWithProbability"); 123 private final SnippetInfo profileBackedgeWithProbability = snippet(ProbabilisticProfileSnippets.class, "profileBackedgeWithProbability"); 124 private final SnippetInfo profileConditionalBackedgeWithProbability = snippet(ProbabilisticProfileSnippets.class, "profileConditionalBackedgeWithProbability"); 125 126 public Templates(OptionValues options, Iterable<DebugHandlersFactory> factories, HotSpotProviders providers, TargetDescription target) { 127 super(options, factories, providers, providers.getSnippetReflection(), target); 128 } 129 130 public void lower(ProfileNode profileNode, LoweringTool tool) { 131 assert profileNode.getRandom() != null; 132 133 StructuredGraph graph = profileNode.graph(); 134 LoadMethodCountersNode counters = graph.unique(new LoadMethodCountersNode(profileNode.getProfiledMethod())); 135 ConstantNode step = ConstantNode.forInt(profileNode.getStep(), graph); 136 ConstantNode stepLog = ConstantNode.forInt(CodeUtil.log2(profileNode.getStep()), graph); 137 138 if (profileNode instanceof ProfileBranchNode) { 139 // Backedge event 140 ProfileBranchNode profileBranchNode = (ProfileBranchNode) profileNode; 141 SnippetInfo snippet = profileBranchNode.hasCondition() ? profileConditionalBackedgeWithProbability : profileBackedgeWithProbability; 142 Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage()); 143 ConstantNode bci = ConstantNode.forInt(profileBranchNode.bci(), graph); 144 ConstantNode targetBci = ConstantNode.forInt(profileBranchNode.targetBci(), graph); 145 146 args.add("counters", counters); 147 args.add("random", profileBranchNode.getRandom()); 148 args.add("step", step); 149 args.add("stepLog", stepLog); 150 args.addConst("freqLog", profileBranchNode.getNotificationFreqLog()); 151 args.addConst("probLog", profileBranchNode.getProbabilityLog()); 152 if (profileBranchNode.hasCondition()) { 153 args.add("branchCondition", profileBranchNode.branchCondition()); 154 } 155 args.add("bci", bci); 156 args.add("targetBci", targetBci); 157 158 SnippetTemplate template = template(profileNode, args); 159 template.instantiate(providers.getMetaAccess(), profileNode, DEFAULT_REPLACER, args); 160 } else if (profileNode instanceof ProfileInvokeNode) { 161 ProfileInvokeNode profileInvokeNode = (ProfileInvokeNode) profileNode; 162 // Method invocation event 163 Arguments args = new Arguments(profileMethodEntryWithProbability, graph.getGuardsStage(), tool.getLoweringStage()); 164 165 args.add("counters", counters); 166 args.add("random", profileInvokeNode.getRandom()); 167 args.add("step", step); 168 args.add("stepLog", stepLog); 169 args.addConst("freqLog", profileInvokeNode.getNotificationFreqLog()); 170 args.addConst("probLog", profileInvokeNode.getProbabilityLog()); 171 SnippetTemplate template = template(profileNode, args); 172 template.instantiate(providers.getMetaAccess(), profileNode, DEFAULT_REPLACER, args); 173 } else { 174 throw new GraalError("Unsupported profile node type: " + profileNode); 175 } 176 177 assert profileNode.hasNoUsages(); 178 if (!profileNode.isDeleted()) { 179 GraphUtil.killWithUnusedFloatingInputs(profileNode); 180 } 181 } 182 } 183 }