1 /*
2 * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24
25 package org.graalvm.compiler.replacements.amd64;
26
27 import static org.graalvm.compiler.replacements.StandardGraphBuilderPlugins.registerPlatformSpecificUnsafePlugins;
28 import static org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode.BinaryOperation.POW;
29 import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.COS;
30 import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.EXP;
31 import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.LOG;
32 import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.LOG10;
33 import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.SIN;
34 import static org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation.TAN;
35
36 import java.util.Arrays;
37
38 import org.graalvm.compiler.bytecode.BytecodeProvider;
39 import org.graalvm.compiler.lir.amd64.AMD64ArithmeticLIRGeneratorTool.RoundingMode;
40 import org.graalvm.compiler.nodes.PauseNode;
41 import org.graalvm.compiler.nodes.ValueNode;
42 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderConfiguration.Plugins;
43 import org.graalvm.compiler.nodes.graphbuilderconf.GraphBuilderContext;
44 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin;
45 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugin.Receiver;
46 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins;
47 import org.graalvm.compiler.nodes.graphbuilderconf.InvocationPlugins.Registration;
48 import org.graalvm.compiler.nodes.java.AtomicReadAndAddNode;
49 import org.graalvm.compiler.nodes.java.AtomicReadAndWriteNode;
50 import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
51 import org.graalvm.compiler.replacements.ArraysSubstitutions;
52 import org.graalvm.compiler.replacements.IntegerSubstitutions;
53 import org.graalvm.compiler.replacements.LongSubstitutions;
54 import org.graalvm.compiler.replacements.StandardGraphBuilderPlugins.UnsafeAccessPlugin;
55 import org.graalvm.compiler.replacements.StandardGraphBuilderPlugins.UnsafeGetPlugin;
56 import org.graalvm.compiler.replacements.StandardGraphBuilderPlugins.UnsafePutPlugin;
57 import org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode;
58 import org.graalvm.compiler.replacements.nodes.BinaryMathIntrinsicNode.BinaryOperation;
59 import org.graalvm.compiler.replacements.nodes.BitCountNode;
60 import org.graalvm.compiler.replacements.nodes.FusedMultiplyAddNode;
61 import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode;
62 import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode.UnaryOperation;
63 import org.graalvm.compiler.serviceprovider.JavaVersionUtil;
64
65 import jdk.vm.ci.amd64.AMD64;
66 import jdk.vm.ci.amd64.AMD64.CPUFeature;
67 import jdk.vm.ci.meta.JavaKind;
68 import jdk.vm.ci.meta.ResolvedJavaMethod;
69 import sun.misc.Unsafe;
70
71 public class AMD64GraphBuilderPlugins {
72
73 public static void register(Plugins plugins, BytecodeProvider replacementsBytecodeProvider, AMD64 arch, boolean explicitUnsafeNullChecks, boolean emitJDK9StringSubstitutions,
74 boolean useFMAIntrinsics) {
75 InvocationPlugins invocationPlugins = plugins.getInvocationPlugins();
76 invocationPlugins.defer(new Runnable() {
77 @Override
78 public void run() {
79 registerThreadPlugins(invocationPlugins, arch);
80 registerIntegerLongPlugins(invocationPlugins, IntegerSubstitutions.class, JavaKind.Int, arch, replacementsBytecodeProvider);
81 registerIntegerLongPlugins(invocationPlugins, LongSubstitutions.class, JavaKind.Long, arch, replacementsBytecodeProvider);
82 registerPlatformSpecificUnsafePlugins(invocationPlugins, replacementsBytecodeProvider, explicitUnsafeNullChecks,
83 new JavaKind[]{JavaKind.Int, JavaKind.Long, JavaKind.Object, JavaKind.Boolean, JavaKind.Byte, JavaKind.Short, JavaKind.Char, JavaKind.Float, JavaKind.Double});
84 registerUnsafePlugins(invocationPlugins, replacementsBytecodeProvider, explicitUnsafeNullChecks);
85 registerStringPlugins(invocationPlugins, replacementsBytecodeProvider);
86 if (emitJDK9StringSubstitutions) {
87 registerStringLatin1Plugins(invocationPlugins, replacementsBytecodeProvider);
88 registerStringUTF16Plugins(invocationPlugins, replacementsBytecodeProvider);
89 }
90 registerMathPlugins(invocationPlugins, useFMAIntrinsics, arch, replacementsBytecodeProvider);
91 registerArraysEqualsPlugins(invocationPlugins, replacementsBytecodeProvider);
92 }
93 });
94 }
95
96 private static void registerThreadPlugins(InvocationPlugins plugins, AMD64 arch) {
97 if (JavaVersionUtil.JAVA_SPEC > 8) {
98 // Pause instruction introduced with SSE2
99 if (arch.getFeatures().contains(AMD64.CPUFeature.SSE2)) {
100 Registration r = new Registration(plugins, Thread.class);
101 r.register0("onSpinWait", new InvocationPlugin() {
102 @Override
103 public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver) {
104 b.append(new PauseNode());
105 return true;
106 }
107 });
108 }
109 }
110 }
111
112 private static void registerIntegerLongPlugins(InvocationPlugins plugins, Class<?> substituteDeclaringClass, JavaKind kind, AMD64 arch, BytecodeProvider bytecodeProvider) {
113 Class<?> declaringClass = kind.toBoxedJavaClass();
114 Class<?> type = kind.toJavaClass();
115 Registration r = new Registration(plugins, declaringClass, bytecodeProvider);
116 r.registerMethodSubstitution(substituteDeclaringClass, "numberOfLeadingZeros", type);
117 if (arch.getFeatures().contains(AMD64.CPUFeature.LZCNT) && arch.getFlags().contains(AMD64.Flag.UseCountLeadingZerosInstruction)) {
118 r.setAllowOverwrite(true);
119 r.register1("numberOfLeadingZeros", type, new InvocationPlugin() {
120 @Override
121 public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) {
122 ValueNode folded = AMD64CountLeadingZerosNode.tryFold(value);
123 if (folded != null) {
124 b.addPush(JavaKind.Int, folded);
125 } else {
126 b.addPush(JavaKind.Int, new AMD64CountLeadingZerosNode(value));
127 }
128 return true;
129 }
130 });
131 }
132
133 r.registerMethodSubstitution(substituteDeclaringClass, "numberOfTrailingZeros", type);
134 if (arch.getFeatures().contains(AMD64.CPUFeature.BMI1) && arch.getFlags().contains(AMD64.Flag.UseCountTrailingZerosInstruction)) {
135 r.setAllowOverwrite(true);
136 r.register1("numberOfTrailingZeros", type, new InvocationPlugin() {
137
138 @Override
139 public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) {
140 ValueNode folded = AMD64CountTrailingZerosNode.tryFold(value);
141 if (folded != null) {
142 b.addPush(JavaKind.Int, folded);
143 } else {
144 b.addPush(JavaKind.Int, new AMD64CountTrailingZerosNode(value));
145 }
146 return true;
147 }
148 });
149 }
150
151 if (arch.getFeatures().contains(AMD64.CPUFeature.POPCNT)) {
152 r.register1("bitCount", type, new InvocationPlugin() {
153 @Override
154 public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) {
155 b.push(JavaKind.Int, b.append(new BitCountNode(value).canonical(null)));
156 return true;
157 }
158 });
159 }
160
161 }
162
163 private static void registerMathPlugins(InvocationPlugins plugins, boolean useFMAIntrinsics, AMD64 arch, BytecodeProvider bytecodeProvider) {
164 Registration r = new Registration(plugins, Math.class, bytecodeProvider);
165 registerUnaryMath(r, "log", LOG);
166 registerUnaryMath(r, "log10", LOG10);
167 registerUnaryMath(r, "exp", EXP);
168 registerBinaryMath(r, "pow", POW);
169 registerUnaryMath(r, "sin", SIN);
170 registerUnaryMath(r, "cos", COS);
171 registerUnaryMath(r, "tan", TAN);
172
173 if (arch.getFeatures().contains(CPUFeature.SSE4_1)) {
174 registerRound(r, "rint", RoundingMode.NEAREST);
175 registerRound(r, "ceil", RoundingMode.UP);
176 registerRound(r, "floor", RoundingMode.DOWN);
177 }
178
179 if (useFMAIntrinsics && JavaVersionUtil.JAVA_SPEC > 8 && arch.getFeatures().contains(CPUFeature.FMA)) {
180 registerFMA(r);
181 }
182 }
183
184 private static void registerFMA(Registration r) {
185 r.register3("fma",
186 Double.TYPE,
187 Double.TYPE,
188 Double.TYPE,
189 new InvocationPlugin() {
190 @Override
191 public boolean apply(GraphBuilderContext b,
192 ResolvedJavaMethod targetMethod,
193 Receiver receiver,
194 ValueNode na,
195 ValueNode nb,
196 ValueNode nc) {
197 b.push(JavaKind.Double, b.append(new FusedMultiplyAddNode(na, nb, nc)));
198 return true;
199 }
200 });
201 r.register3("fma",
202 Float.TYPE,
203 Float.TYPE,
204 Float.TYPE,
205 new InvocationPlugin() {
206 @Override
207 public boolean apply(GraphBuilderContext b,
208 ResolvedJavaMethod targetMethod,
209 Receiver receiver,
210 ValueNode na,
211 ValueNode nb,
212 ValueNode nc) {
213 b.push(JavaKind.Float, b.append(new FusedMultiplyAddNode(na, nb, nc)));
214 return true;
215 }
216 });
217 }
218
219 private static void registerUnaryMath(Registration r, String name, UnaryOperation operation) {
220 r.register1(name, Double.TYPE, new InvocationPlugin() {
221 @Override
222 public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode value) {
223 b.push(JavaKind.Double, b.append(UnaryMathIntrinsicNode.create(value, operation)));
224 return true;
225 }
226 });
227 }
228
229 private static void registerBinaryMath(Registration r, String name, BinaryOperation operation) {
230 r.register2(name, Double.TYPE, Double.TYPE, new InvocationPlugin() {
231 @Override
232 public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode x, ValueNode y) {
233 b.push(JavaKind.Double, b.append(BinaryMathIntrinsicNode.create(x, y, operation)));
234 return true;
235 }
236 });
237 }
238
239 private static void registerRound(Registration r, String name, RoundingMode mode) {
240 r.register1(name, Double.TYPE, new InvocationPlugin() {
241 @Override
242 public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode arg) {
243 b.push(JavaKind.Double, b.append(new AMD64RoundNode(arg, mode)));
244 return true;
245 }
246 });
247 }
248
249 private static void registerStringPlugins(InvocationPlugins plugins, BytecodeProvider replacementsBytecodeProvider) {
250 if (JavaVersionUtil.JAVA_SPEC <= 8) {
251 Registration r;
252 r = new Registration(plugins, String.class, replacementsBytecodeProvider);
253 r.setAllowOverwrite(true);
254 r.registerMethodSubstitution(AMD64StringSubstitutions.class, "indexOf", char[].class, int.class,
255 int.class, char[].class, int.class, int.class, int.class);
256 r.registerMethodSubstitution(AMD64StringSubstitutions.class, "indexOf", Receiver.class, int.class, int.class);
257 r.registerMethodSubstitution(AMD64StringSubstitutions.class, "compareTo", Receiver.class, String.class);
258 }
259 }
260
261 private static void registerStringLatin1Plugins(InvocationPlugins plugins, BytecodeProvider replacementsBytecodeProvider) {
262 Registration r = new Registration(plugins, "java.lang.StringLatin1", replacementsBytecodeProvider);
263 r.setAllowOverwrite(true);
264 r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "compareTo", byte[].class, byte[].class);
265 r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "compareToUTF16", byte[].class, byte[].class);
266 r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "inflate", byte[].class, int.class, char[].class, int.class, int.class);
267 r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "inflate", byte[].class, int.class, byte[].class, int.class, int.class);
268 r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "indexOf", byte[].class, int.class, int.class);
269 r.registerMethodSubstitution(AMD64StringLatin1Substitutions.class, "indexOf", byte[].class, int.class, byte[].class, int.class, int.class);
270 }
271
272 private static void registerStringUTF16Plugins(InvocationPlugins plugins, BytecodeProvider replacementsBytecodeProvider) {
273 Registration r = new Registration(plugins, "java.lang.StringUTF16", replacementsBytecodeProvider);
274 r.setAllowOverwrite(true);
275 r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "compareTo", byte[].class, byte[].class);
276 r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "compareToLatin1", byte[].class, byte[].class);
277 r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "compress", char[].class, int.class, byte[].class, int.class, int.class);
278 r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "compress", byte[].class, int.class, byte[].class, int.class, int.class);
279 r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "indexOfCharUnsafe", byte[].class, int.class, int.class, int.class);
280 r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "indexOfUnsafe", byte[].class, int.class, byte[].class, int.class, int.class);
281 r.registerMethodSubstitution(AMD64StringUTF16Substitutions.class, "indexOfLatin1Unsafe", byte[].class, int.class, byte[].class, int.class, int.class);
282 }
283
284 private static void registerUnsafePlugins(InvocationPlugins plugins, BytecodeProvider replacementsBytecodeProvider, boolean explicitUnsafeNullChecks) {
285 registerUnsafePlugins(new Registration(plugins, Unsafe.class), explicitUnsafeNullChecks, new JavaKind[]{JavaKind.Int, JavaKind.Long, JavaKind.Object}, true);
286 if (JavaVersionUtil.JAVA_SPEC > 8) {
287 registerUnsafePlugins(new Registration(plugins, "jdk.internal.misc.Unsafe", replacementsBytecodeProvider), explicitUnsafeNullChecks,
288 new JavaKind[]{JavaKind.Boolean, JavaKind.Byte, JavaKind.Char, JavaKind.Short, JavaKind.Int, JavaKind.Long, JavaKind.Object},
289 JavaVersionUtil.JAVA_SPEC <= 11);
290 }
291 }
292
293 private static void registerUnsafePlugins(Registration r, boolean explicitUnsafeNullChecks, JavaKind[] unsafeJavaKinds, boolean java11OrEarlier) {
294 for (JavaKind kind : unsafeJavaKinds) {
295 Class<?> javaClass = kind == JavaKind.Object ? Object.class : kind.toJavaClass();
296 String kindName = (kind == JavaKind.Object && !java11OrEarlier) ? "Reference" : kind.name();
297 r.register4("getAndSet" + kindName, Receiver.class, Object.class, long.class, javaClass, new UnsafeAccessPlugin(kind, explicitUnsafeNullChecks) {
298 @Override
299 public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver unsafe, ValueNode object, ValueNode offset, ValueNode value) {
300 // Emits a null-check for the otherwise unused receiver
301 unsafe.get();
302 createUnsafeAccess(object, b, (obj, loc) -> new AtomicReadAndWriteNode(obj, offset, value, kind, loc));
303 return true;
304 }
305 });
306 if (kind != JavaKind.Boolean && kind.isNumericInteger()) {
307 r.register4("getAndAdd" + kindName, Receiver.class, Object.class, long.class, javaClass, new UnsafeAccessPlugin(kind, explicitUnsafeNullChecks) {
308 @Override
309 public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver unsafe, ValueNode object, ValueNode offset, ValueNode delta) {
310 // Emits a null-check for the otherwise unused receiver
311 unsafe.get();
312 createUnsafeAccess(object, b, (obj, loc) -> new AtomicReadAndAddNode(b.add(new OffsetAddressNode(obj, offset)), delta, kind, loc));
313 return true;
314 }
315 });
316 }
317 }
318
319 for (JavaKind kind : new JavaKind[]{JavaKind.Char, JavaKind.Short, JavaKind.Int, JavaKind.Long}) {
320 Class<?> javaClass = kind.toJavaClass();
321 r.registerOptional3("get" + kind.name() + "Unaligned", Receiver.class, Object.class, long.class, new UnsafeGetPlugin(kind, explicitUnsafeNullChecks));
322 r.registerOptional4("put" + kind.name() + "Unaligned", Receiver.class, Object.class, long.class, javaClass, new UnsafePutPlugin(kind, explicitUnsafeNullChecks));
323 }
324 }
325
326 private static void registerArraysEqualsPlugins(InvocationPlugins plugins, BytecodeProvider bytecodeProvider) {
327 Registration r = new Registration(plugins, Arrays.class, bytecodeProvider);
328 r.registerMethodSubstitution(ArraysSubstitutions.class, "equals", float[].class, float[].class);
329 r.registerMethodSubstitution(ArraysSubstitutions.class, "equals", double[].class, double[].class);
330 }
331 }