1 /*
2 * Copyright (c) 2013, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24
25 package org.graalvm.compiler.hotspot.test;
26
27 import static java.util.Collections.singletonList;
28 import static org.graalvm.compiler.core.CompilationWrapper.ExceptionAction.Print;
29 import static org.graalvm.compiler.core.GraalCompilerOptions.CompilationBailoutAsFailure;
30 import static org.graalvm.compiler.core.GraalCompilerOptions.CompilationFailureAction;
31 import static org.graalvm.compiler.core.test.ReflectionOptionDescriptors.extractEntries;
32 import static org.graalvm.compiler.debug.MemUseTrackerKey.getCurrentThreadAllocatedBytes;
33 import static org.graalvm.compiler.hotspot.CompilationTask.CompilationTime;
34 import static org.graalvm.compiler.hotspot.CompilationTask.CompiledAndInstalledBytecodes;
35 import static org.graalvm.compiler.hotspot.test.CompileTheWorld.Options.DESCRIPTORS;
36 import static org.graalvm.compiler.hotspot.test.CompileTheWorld.Options.InvalidateInstalledCode;
37 import static sun.misc.Unsafe.ARRAY_BYTE_BASE_OFFSET;
38
39 import java.io.ByteArrayOutputStream;
40 import java.io.Closeable;
41 import java.io.File;
42 import java.io.IOException;
43 import java.lang.annotation.Annotation;
44 import java.lang.reflect.Constructor;
45 import java.lang.reflect.Method;
46 import java.lang.reflect.Modifier;
47 import java.net.URI;
48 import java.net.URL;
49 import java.net.URLClassLoader;
50 import java.nio.file.FileSystem;
51 import java.nio.file.FileSystems;
52 import java.nio.file.FileVisitResult;
53 import java.nio.file.Files;
54 import java.nio.file.Path;
55 import java.nio.file.SimpleFileVisitor;
56 import java.nio.file.attribute.BasicFileAttributes;
57 import java.util.ArrayList;
58 import java.util.Arrays;
59 import java.util.Collections;
60 import java.util.Enumeration;
61 import java.util.HashMap;
62 import java.util.HashSet;
63 import java.util.List;
64 import java.util.Map;
65 import java.util.ServiceLoader;
66 import java.util.Set;
67 import java.util.concurrent.ExecutionException;
68 import java.util.concurrent.Future;
69 import java.util.concurrent.LinkedBlockingQueue;
70 import java.util.concurrent.ThreadPoolExecutor;
71 import java.util.concurrent.TimeUnit;
72 import java.util.concurrent.atomic.AtomicLong;
73 import java.util.jar.JarEntry;
74 import java.util.jar.JarFile;
75 import java.util.regex.Matcher;
76 import java.util.regex.Pattern;
77 import java.util.stream.Collectors;
78
79 import jdk.internal.vm.compiler.collections.EconomicMap;
80 import jdk.internal.vm.compiler.collections.UnmodifiableMapCursor;
81 import org.graalvm.compiler.api.replacements.Snippet;
82 import org.graalvm.compiler.bytecode.Bytecodes;
83 import org.graalvm.compiler.core.CompilerThreadFactory;
84 import org.graalvm.compiler.core.phases.HighTier;
85 import org.graalvm.compiler.core.test.ReflectionOptionDescriptors;
86 import org.graalvm.compiler.debug.DebugOptions;
87 import org.graalvm.compiler.debug.GlobalMetrics;
88 import org.graalvm.compiler.debug.GraalError;
89 import org.graalvm.compiler.debug.MethodFilter;
90 import org.graalvm.compiler.debug.MetricKey;
91 import org.graalvm.compiler.debug.TTY;
92 import org.graalvm.compiler.hotspot.CompilationTask;
93 import org.graalvm.compiler.hotspot.GraalHotSpotVMConfig;
94 import org.graalvm.compiler.hotspot.HotSpotGraalCompiler;
95 import org.graalvm.compiler.hotspot.HotSpotGraalRuntime;
96 import org.graalvm.compiler.hotspot.HotSpotGraalRuntimeProvider;
97 import org.graalvm.compiler.hotspot.test.CompileTheWorld.LibGraalParams.StackTraceBuffer;
98 import org.graalvm.compiler.options.OptionDescriptors;
99 import org.graalvm.compiler.options.OptionKey;
100 import org.graalvm.compiler.options.OptionValues;
101 import org.graalvm.compiler.options.OptionsParser;
102 import org.graalvm.compiler.serviceprovider.GraalUnsafeAccess;
103 import org.graalvm.compiler.serviceprovider.JavaVersionUtil;
104 import org.graalvm.compiler.test.ModuleSupport;
105 import jdk.internal.vm.compiler.libgraal.LibGraal;
106 import jdk.internal.vm.compiler.libgraal.LibGraalScope;
107 import jdk.internal.vm.compiler.libgraal.OptionsEncoder;
108
109 import jdk.vm.ci.hotspot.HotSpotCodeCacheProvider;
110 import jdk.vm.ci.hotspot.HotSpotCompilationRequest;
111 import jdk.vm.ci.hotspot.HotSpotInstalledCode;
112 import jdk.vm.ci.hotspot.HotSpotJVMCIRuntime;
113 import jdk.vm.ci.hotspot.HotSpotResolvedJavaMethod;
114 import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
115 import jdk.vm.ci.meta.ConstantPool;
116 import jdk.vm.ci.meta.MetaAccessProvider;
117 import jdk.vm.ci.runtime.JVMCI;
118 import jdk.vm.ci.runtime.JVMCICompiler;
119 import sun.misc.Unsafe;
120
121 /**
122 * This class implements compile-the-world functionality with JVMCI.
123 */
124 public final class CompileTheWorld {
125
126 static {
127 ModuleSupport.exportAndOpenAllPackagesToUnnamed("jdk.internal.vm.compiler");
128 }
129
130 /**
131 * Magic token to denote that JDK classes are to be compiled. For JDK 8, the classes in
132 * {@code rt.jar} are compiled. Otherwise the classes in the Java runtime image are compiled.
133 */
134 public static final String SUN_BOOT_CLASS_PATH = "sun.boot.class.path";
135
136 /**
137 * Magic token to denote the classes in the Java runtime image (i.e. in the {@code jrt:/} file
138 * system).
139 */
140 public static final String JRT_CLASS_PATH_ENTRY = "<jrt>";
141
142 /**
143 * @param options a space separated set of option value settings with each option setting in a
144 * {@code -Dgraal.<name>=<value>} format but without the leading {@code -Dgraal.}.
145 * Ignored if null.
146 */
147 public static EconomicMap<OptionKey<?>, Object> parseOptions(String options) {
148 EconomicMap<OptionKey<?>, Object> values = OptionValues.newOptionMap();
149 if (options != null) {
150 EconomicMap<String, String> optionSettings = EconomicMap.create();
151 for (String optionSetting : options.split("\\s+|#")) {
152 OptionsParser.parseOptionSettingTo(optionSetting, optionSettings);
153 }
154 ServiceLoader<OptionDescriptors> loader = ServiceLoader.load(OptionDescriptors.class, OptionDescriptors.class.getClassLoader());
155 OptionsParser.parseOptions(optionSettings, values, loader);
156 }
157 if (!values.containsKey(HighTier.Options.Inline)) {
158 values.put(HighTier.Options.Inline, false);
159 }
160 return values;
161 }
162
163 private final HotSpotJVMCIRuntime jvmciRuntime;
164
165 private final HotSpotGraalCompiler compiler;
166
167 /**
168 * Class path denoting classes to compile.
169 *
170 * @see Options#Classpath
171 */
172 private final String inputClassPath;
173
174 /**
175 * Class index to start compilation at.
176 *
177 * @see Options#StartAt
178 */
179 private final int startAt;
180
181 /**
182 * Class index to stop compilation at.
183 *
184 * @see Options#StopAt
185 */
186 private final int stopAt;
187
188 /**
189 * Max classes to compile.
190 *
191 * @see Options#MaxClasses
192 */
193 private final int maxClasses;
194
195 /** Only compile methods matching one of the filters in this array if the array is non-null. */
196 private final MethodFilter[] methodFilters;
197
198 /** Exclude methods matching one of the filters in this array if the array is non-null. */
199 private final MethodFilter[] excludeMethodFilters;
200
201 // Counters
202 private int classFileCounter = 0;
203 private AtomicLong compiledMethodsCounter = new AtomicLong();
204 private AtomicLong compileTime = new AtomicLong();
205 private AtomicLong memoryUsed = new AtomicLong();
206
207 private boolean verbose;
208
209 /**
210 * Signal that the threads should start compiling in multithreaded mode.
211 */
212 private boolean running;
213
214 private ThreadPoolExecutor threadPool;
215
216 /**
217 * Values for {@link CompileTheWorld.Options}.
218 */
219 private final OptionValues harnessOptions;
220
221 /**
222 * Option values used during compilation.
223 */
224 private final OptionValues compilerOptions;
225
226 /**
227 * Manages native memory buffers for passing arguments into libgraal and receiving return
228 * values. The native memory buffers are freed when this object is {@linkplain #close() closed}.
229 */
230 static class LibGraalParams implements AutoCloseable {
231
232 static {
233 LibGraal.registerNativeMethods(HotSpotJVMCIRuntime.runtime(), CompileTheWorld.class);
234 }
235
236 /**
237 * Native memory containing {@linkplain OptionsEncoder encoded} {@link OptionValues}.
238 */
239 static class OptionsBuffer {
240 private long address;
241 final int size;
242 final int hash;
243
244 OptionsBuffer(OptionValues options) {
245 Map<String, Object> map = new HashMap<>();
246 UnmodifiableMapCursor<OptionKey<?>, Object> cursor = options.getMap().getEntries();
247 while (cursor.advance()) {
248 final OptionKey<?> key = cursor.getKey();
249 Object value = cursor.getValue();
250 map.put(key.getName(), value);
251 }
252
253 byte[] encoded = OptionsEncoder.encode(map);
254 size = encoded.length;
255 hash = Arrays.hashCode(encoded);
256 address = UNSAFE.allocateMemory(encoded.length);
257 UNSAFE.copyMemory(encoded, ARRAY_BYTE_BASE_OFFSET, null, address, size);
258 }
259
260 long getAddress() {
261 if (address == 0) {
262 throw new IllegalStateException();
263 }
264 return address;
265 }
266
267 void free() {
268 if (address != 0) {
269 UNSAFE.freeMemory(address);
270 address = 0;
271 }
272 }
273 }
274
275 /**
276 * Manages native memory for receiving a {@linkplain Throwable#printStackTrace() stack
277 * trace} from libgraal serialized via {@link ByteArrayOutputStream} to a byte array.
278 */
279 static class StackTraceBuffer {
280 final int size;
281 private long address;
282
283 StackTraceBuffer(int size) {
284 this.size = size;
285 address = UNSAFE.allocateMemory(size);
286 }
287
288 void free() {
289 if (address != 0L) {
290 UNSAFE.freeMemory(address);
291 address = 0L;
292 }
293 }
294
295 long getAddress() {
296 if (address == 0) {
297 throw new IllegalStateException();
298 }
299 return address;
300 }
301 }
302
303 final OptionsBuffer options;
304
305 private final List<StackTraceBuffer> stackTraceBuffers = new ArrayList<>();
306
307 /**
308 * Gets a stack trace buffer for the current thread.
309 */
310 StackTraceBuffer getStackTraceBuffer() {
311 return stackTraceBuffer.get();
312 }
313
314 private final ThreadLocal<StackTraceBuffer> stackTraceBuffer = new ThreadLocal<StackTraceBuffer>() {
315 @Override
316 protected StackTraceBuffer initialValue() {
317 StackTraceBuffer buffer = new StackTraceBuffer(10_000);
318 synchronized (stackTraceBuffers) {
319 stackTraceBuffers.add(buffer);
320 }
321 return buffer;
322 }
323 };
324
325 LibGraalParams(OptionValues options) {
326 this.options = new OptionsBuffer(options);
327 }
328
329 @Override
330 public void close() {
331 options.free();
332 synchronized (stackTraceBuffers) {
333 for (StackTraceBuffer buffer : stackTraceBuffers) {
334 buffer.free();
335 }
336 stackTraceBuffers.clear();
337 }
338 }
339 }
340
341 /**
342 * Creates a compile-the-world instance.
343 *
344 * @param files {@link File#pathSeparator} separated list of Zip/Jar files to compile
345 * @param startAt index of the class file to start compilation at
346 * @param stopAt index of the class file to stop compilation at
347 * @param maxClasses maximum number of classes to process
348 * @param methodFilters
349 * @param excludeMethodFilters
350 * @param harnessOptions values for {@link CompileTheWorld.Options}
351 * @param compilerOptions option values used by the compiler
352 */
353 public CompileTheWorld(HotSpotJVMCIRuntime jvmciRuntime,
354 HotSpotGraalCompiler compiler,
355 String files,
356 int startAt,
357 int stopAt,
358 int maxClasses,
359 String methodFilters,
360 String excludeMethodFilters,
361 boolean verbose,
362 OptionValues harnessOptions,
363 OptionValues compilerOptions) {
364 this.jvmciRuntime = jvmciRuntime;
365 this.compiler = compiler;
366 this.inputClassPath = files;
367 this.startAt = Math.max(startAt, 1);
368 this.stopAt = Math.max(stopAt, 1);
369 this.maxClasses = Math.max(maxClasses, 1);
370 this.methodFilters = methodFilters == null || methodFilters.isEmpty() ? null : MethodFilter.parse(methodFilters);
371 this.excludeMethodFilters = excludeMethodFilters == null || excludeMethodFilters.isEmpty() ? null : MethodFilter.parse(excludeMethodFilters);
372 this.verbose = verbose;
373 this.harnessOptions = harnessOptions;
374
375 // Copy the initial options and add in any extra options
376 EconomicMap<OptionKey<?>, Object> compilerOptionsMap = EconomicMap.create(compilerOptions.getMap());
377
378 // We want to see stack traces when a method fails to compile
379 CompilationBailoutAsFailure.putIfAbsent(compilerOptionsMap, true);
380 CompilationFailureAction.putIfAbsent(compilerOptionsMap, Print);
381
382 // By default only report statistics for the CTW threads themselves
383 DebugOptions.MetricsThreadFilter.putIfAbsent(compilerOptionsMap, "^CompileTheWorld");
384 this.compilerOptions = new OptionValues(compilerOptionsMap);
385 }
386
387 public CompileTheWorld(HotSpotJVMCIRuntime jvmciRuntime,
388 HotSpotGraalCompiler compiler,
389 OptionValues harnessOptions,
390 OptionValues compilerOptions) {
391 this(jvmciRuntime, compiler, Options.Classpath.getValue(harnessOptions),
392 Options.StartAt.getValue(harnessOptions),
393 Options.StopAt.getValue(harnessOptions),
394 Options.MaxClasses.getValue(harnessOptions),
395 Options.MethodFilter.getValue(harnessOptions),
396 Options.ExcludeMethodFilter.getValue(harnessOptions),
397 Options.Verbose.hasBeenSet(harnessOptions) ? Options.Verbose.getValue(harnessOptions) : !Options.MultiThreaded.getValue(harnessOptions),
398 harnessOptions,
399 new OptionValues(compilerOptions, parseOptions(Options.Config.getValue(harnessOptions))));
400 }
401
402 /**
403 * Compiles all methods in all classes in {@link #inputClassPath}. If {@link #inputClassPath}
404 * equals {@link #SUN_BOOT_CLASS_PATH} the boot classes are used.
405 */
406 @SuppressWarnings("try")
407 public void compile() throws Throwable {
408 try (LibGraalParams libgraal = LibGraal.isAvailable() ? new LibGraalParams(compilerOptions) : null) {
409 if (SUN_BOOT_CLASS_PATH.equals(inputClassPath)) {
410 String bcpEntry = null;
411 if (JavaVersionUtil.JAVA_SPEC <= 8) {
412 final String[] entries = System.getProperty(SUN_BOOT_CLASS_PATH).split(File.pathSeparator);
413 for (int i = 0; i < entries.length && bcpEntry == null; i++) {
414 String entry = entries[i];
415 File entryFile = new File(entry);
416 if (entryFile.getName().endsWith("rt.jar") && entryFile.isFile()) {
417 bcpEntry = entry;
418 }
419 }
420 if (bcpEntry == null) {
421 throw new GraalError("Could not find rt.jar on boot class path %s", System.getProperty(SUN_BOOT_CLASS_PATH));
422 }
423 } else {
424 bcpEntry = JRT_CLASS_PATH_ENTRY;
425 }
426 compile(bcpEntry, libgraal);
427 } else {
428 compile(inputClassPath, libgraal);
429 }
430 }
431 }
432
433 public void println() {
434 println("");
435 }
436
437 public void println(String format, Object... args) {
438 println(String.format(format, args));
439 }
440
441 public void println(String s) {
442 println(verbose, s);
443 }
444
445 public static void println(boolean cond, String s) {
446 if (cond) {
447 TTY.println(s);
448 }
449 }
450
451 public void printStackTrace(Throwable t) {
452 if (verbose) {
453 t.printStackTrace(TTY.out);
454 }
455 }
456
457 @SuppressWarnings("unused")
458 private static void dummy() {
459 }
460
461 /**
462 * Abstraction over different types of class path entries.
463 */
464 abstract static class ClassPathEntry implements Closeable {
465 final String name;
466
467 ClassPathEntry(String name) {
468 this.name = name;
469 }
470
471 /**
472 * Creates a {@link ClassLoader} for loading classes from this entry.
473 */
474 public abstract ClassLoader createClassLoader() throws IOException;
475
476 /**
477 * Gets the list of classes available under this entry.
478 */
479 public abstract List<String> getClassNames() throws IOException;
480
481 @Override
482 public String toString() {
483 return name;
484 }
485
486 @Override
487 public void close() throws IOException {
488 }
489 }
490
491 /**
492 * A class path entry that is a normal file system directory.
493 */
494 static class DirClassPathEntry extends ClassPathEntry {
495
496 private final File dir;
497
498 DirClassPathEntry(String name) {
499 super(name);
500 dir = new File(name);
501 assert dir.isDirectory();
502 }
503
504 @Override
505 public ClassLoader createClassLoader() throws IOException {
506 URL url = dir.toURI().toURL();
507 return new URLClassLoader(new URL[]{url});
508 }
509
510 @Override
511 public List<String> getClassNames() throws IOException {
512 List<String> classNames = new ArrayList<>();
513 String root = dir.getPath();
514 SimpleFileVisitor<Path> visitor = new SimpleFileVisitor<Path>() {
515 @Override
516 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
517 if (attrs.isRegularFile()) {
518 File path = file.toFile();
519 if (path.getName().endsWith(".class")) {
520 String pathString = path.getPath();
521 assert pathString.startsWith(root);
522 String classFile = pathString.substring(root.length() + 1);
523 String className = classFile.replace(File.separatorChar, '.');
524 classNames.add(className.replace('/', '.').substring(0, className.length() - ".class".length()));
525 }
526 }
527 return super.visitFile(file, attrs);
528 }
529 };
530 Files.walkFileTree(dir.toPath(), visitor);
531 return classNames;
532 }
533 }
534
535 /**
536 * A class path entry that is a jar or zip file.
537 */
538 static class JarClassPathEntry extends ClassPathEntry {
539
540 private final JarFile jarFile;
541
542 JarClassPathEntry(String name) throws IOException {
543 super(name);
544 jarFile = new JarFile(name);
545 }
546
547 @Override
548 public ClassLoader createClassLoader() throws IOException {
549 URL url = new URL("jar", "", "file:" + name + "!/");
550 return new URLClassLoader(new URL[]{url});
551 }
552
553 /**
554 * @see "https://docs.oracle.com/javase/9/docs/specs/jar/jar.html#Multi-release"
555 */
556 static Pattern MultiReleaseJarVersionedClassRE = Pattern.compile("META-INF/versions/[1-9][0-9]*/(.+)");
557
558 @Override
559 public List<String> getClassNames() throws IOException {
560 Enumeration<JarEntry> e = jarFile.entries();
561 List<String> classNames = new ArrayList<>(jarFile.size());
562 while (e.hasMoreElements()) {
563 JarEntry je = e.nextElement();
564 if (je.isDirectory() || !je.getName().endsWith(".class")) {
565 continue;
566 }
567 String className = je.getName().substring(0, je.getName().length() - ".class".length());
568 if (className.equals("module-info")) {
569 continue;
570 }
571 if (className.startsWith("META-INF/versions/")) {
572 Matcher m = MultiReleaseJarVersionedClassRE.matcher(className);
573 if (m.matches()) {
574 className = m.group(1);
575 } else {
576 continue;
577 }
578 }
579 classNames.add(className.replace('/', '.'));
580 }
581 return classNames;
582 }
583
584 @Override
585 public void close() throws IOException {
586 jarFile.close();
587 }
588 }
589
590 /**
591 * A class path entry representing the {@code jrt:/} file system.
592 */
593 static class JRTClassPathEntry extends ClassPathEntry {
594
595 private final String limitModules;
596
597 JRTClassPathEntry(String name, String limitModules) {
598 super(name);
599 this.limitModules = limitModules;
600 }
601
602 @Override
603 public ClassLoader createClassLoader() throws IOException {
604 URL url = URI.create("jrt:/").toURL();
605 return new URLClassLoader(new URL[]{url});
606 }
607
608 @Override
609 public List<String> getClassNames() throws IOException {
610 Set<String> negative = new HashSet<>();
611 Set<String> positive = new HashSet<>();
612 if (limitModules != null && !limitModules.isEmpty()) {
613 for (String s : limitModules.split(",")) {
614 if (s.startsWith("~")) {
615 negative.add(s.substring(1));
616 } else {
617 positive.add(s);
618 }
619 }
620 }
621 List<String> classNames = new ArrayList<>();
622 FileSystem fs = FileSystems.newFileSystem(URI.create("jrt:/"), Collections.emptyMap());
623 Path top = fs.getPath("/modules/");
624 Files.find(top, Integer.MAX_VALUE,
625 (path, attrs) -> attrs.isRegularFile()).forEach(p -> {
626 int nameCount = p.getNameCount();
627 if (nameCount > 2) {
628 String base = p.getName(nameCount - 1).toString();
629 if (base.endsWith(".class") && !base.equals("module-info.class")) {
630 String module = p.getName(1).toString();
631 if (positive.isEmpty() || positive.contains(module)) {
632 if (negative.isEmpty() || !negative.contains(module)) {
633 // Strip module prefix and convert to dotted form
634 String className = p.subpath(2, nameCount).toString().replace('/', '.');
635 // Strip ".class" suffix
636 className = className.replace('/', '.').substring(0, className.length() - ".class".length());
637 classNames.add(className);
638 }
639 }
640 }
641 }
642 });
643 return classNames;
644 }
645 }
646
647 private boolean isClassIncluded(String className) {
648 if (methodFilters != null && !MethodFilter.matchesClassName(methodFilters, className)) {
649 return false;
650 }
651 if (excludeMethodFilters != null && MethodFilter.matchesClassName(excludeMethodFilters, className)) {
652 return false;
653 }
654 return true;
655 }
656
657 private ClassPathEntry openClassPathEntry(String entry) throws IOException {
658 if (entry.endsWith(".zip") || entry.endsWith(".jar")) {
659 return new JarClassPathEntry(entry);
660 } else if (entry.equals(JRT_CLASS_PATH_ENTRY)) {
661 return new JRTClassPathEntry(entry, Options.LimitModules.getValue(harnessOptions));
662 } else {
663 if (!new File(entry).isDirectory()) {
664 return null;
665 }
666 return new DirClassPathEntry(entry);
667 }
668 }
669
670 /**
671 * Compiles all methods in all classes in a given class path.
672 *
673 * @param classPath class path denoting classes to compile
674 * @throws IOException
675 */
676 @SuppressWarnings("try")
677 private void compile(String classPath, LibGraalParams libgraal) throws IOException {
678 final String[] entries = classPath.split(File.pathSeparator);
679 long start = System.nanoTime();
680 Map<Thread, StackTraceElement[]> initialThreads = Thread.getAllStackTraces();
681
682 if (libgraal == null) {
683 try {
684 // compile dummy method to get compiler initialized outside of the
685 // config debug override.
686 HotSpotResolvedJavaMethod dummyMethod = (HotSpotResolvedJavaMethod) JVMCI.getRuntime().getHostJVMCIBackend().getMetaAccess().lookupJavaMethod(
687 CompileTheWorld.class.getDeclaredMethod("dummy"));
688 int entryBCI = JVMCICompiler.INVOCATION_ENTRY_BCI;
689 boolean useProfilingInfo = false;
690 boolean installAsDefault = false;
691 CompilationTask task = new CompilationTask(jvmciRuntime, compiler, new HotSpotCompilationRequest(dummyMethod, entryBCI, 0L), useProfilingInfo, installAsDefault);
692 task.runCompilation(compilerOptions);
693 } catch (NoSuchMethodException | SecurityException e1) {
694 printStackTrace(e1);
695 }
696 }
697
698 /*
699 * Always use a thread pool, even for single threaded mode since it simplifies the use of
700 * DebugValueThreadFilter to filter on the thread names.
701 */
702 int threadCount = 1;
703 if (Options.MultiThreaded.getValue(harnessOptions)) {
704 threadCount = Options.Threads.getValue(harnessOptions);
705 if (threadCount == 0) {
706 threadCount = Runtime.getRuntime().availableProcessors();
707 }
708 } else {
709 running = true;
710 }
711
712 threadPool = new ThreadPoolExecutor(threadCount, threadCount, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(), new CompilerThreadFactory("CompileTheWorld"));
713
714 int compileStartAt = startAt;
715 int compileStopAt = stopAt;
716 int compileStep = 1;
717 if (maxClasses != Integer.MAX_VALUE) {
718 int totalClassFileCount = 0;
719 for (String entry : entries) {
720 try (ClassPathEntry cpe = openClassPathEntry(entry)) {
721 if (cpe != null) {
722 totalClassFileCount += cpe.getClassNames().size();
723 }
724 }
725 }
726
727 int lastClassFile = totalClassFileCount - 1;
728 compileStartAt = Math.min(startAt, lastClassFile);
729 compileStopAt = Math.min(stopAt, lastClassFile);
730 int range = compileStopAt - compileStartAt + 1;
731 if (maxClasses < range) {
732 compileStep = range / maxClasses;
733 }
734 }
735
736 for (int i = 0; i < entries.length; i++) {
737 final String entry = entries[i];
738 try (ClassPathEntry cpe = openClassPathEntry(entry)) {
739 if (cpe == null) {
740 println("CompileTheWorld : Skipped classes in " + entry);
741 println();
742 continue;
743 }
744
745 if (methodFilters == null || methodFilters.length == 0) {
746 println("CompileTheWorld : Compiling all classes in " + entry);
747 } else {
748 String include = Arrays.asList(methodFilters).stream().map(MethodFilter::toString).collect(Collectors.joining(", "));
749 println("CompileTheWorld : Compiling all methods in " + entry + " matching one of the following filters: " + include);
750 }
751 if (excludeMethodFilters != null && excludeMethodFilters.length > 0) {
752 String exclude = Arrays.asList(excludeMethodFilters).stream().map(MethodFilter::toString).collect(Collectors.joining(", "));
753 println("CompileTheWorld : Excluding all methods matching one of the following filters: " + exclude);
754 }
755 println();
756
757 ClassLoader loader = cpe.createClassLoader();
758
759 for (String className : cpe.getClassNames()) {
760
761 // Are we done?
762 if (classFileCounter >= compileStopAt) {
763 break;
764 }
765
766 classFileCounter++;
767
768 if (compileStep > 1 && ((classFileCounter - compileStartAt) % compileStep) != 0) {
769 continue;
770 }
771
772 if (className.startsWith("jdk.management.") ||
773 className.startsWith("jdk.internal.cmm.*") ||
774 // GR-5881: The class initializer for
775 // sun.tools.jconsole.OutputViewer
776 // spawns non-daemon threads for redirecting sysout and syserr.
777 // These threads tend to cause deadlock at VM exit
778 className.startsWith("sun.tools.jconsole.")) {
779 continue;
780 }
781
782 if (!isClassIncluded(className)) {
783 continue;
784 }
785
786 try {
787 // Load and initialize class
788 Class<?> javaClass = Class.forName(className, true, loader);
789 MetaAccessProvider metaAccess = JVMCI.getRuntime().getHostJVMCIBackend().getMetaAccess();
790
791 // Pre-load all classes in the constant pool.
792 try {
793 HotSpotResolvedObjectType objectType = (HotSpotResolvedObjectType) metaAccess.lookupJavaType(javaClass);
794 ConstantPool constantPool = objectType.getConstantPool();
795 for (int cpi = 1; cpi < constantPool.length(); cpi++) {
796 constantPool.loadReferencedType(cpi, Bytecodes.LDC);
797 }
798 } catch (Throwable t) {
799 // If something went wrong during pre-loading we just ignore it.
800 if (isClassIncluded(className)) {
801 println("Preloading failed for (%d) %s: %s", classFileCounter, className, t);
802 }
803 continue;
804 }
805
806 // Are we compiling this class?
807 if (classFileCounter >= compileStartAt) {
808
809 long start0 = System.nanoTime();
810 // Compile each constructor/method in the class.
811 for (Constructor<?> constructor : javaClass.getDeclaredConstructors()) {
812 HotSpotResolvedJavaMethod javaMethod = (HotSpotResolvedJavaMethod) metaAccess.lookupJavaMethod(constructor);
813 if (canBeCompiled(javaMethod, constructor.getModifiers())) {
814 compileMethod(javaMethod, libgraal);
815 }
816 }
817 for (Method method : javaClass.getDeclaredMethods()) {
818 HotSpotResolvedJavaMethod javaMethod = (HotSpotResolvedJavaMethod) metaAccess.lookupJavaMethod(method);
819 if (canBeCompiled(javaMethod, method.getModifiers())) {
820 compileMethod(javaMethod, libgraal);
821 }
822 }
823
824 // Also compile the class initializer if it exists
825 HotSpotResolvedJavaMethod clinit = (HotSpotResolvedJavaMethod) metaAccess.lookupJavaType(javaClass).getClassInitializer();
826 if (clinit != null && canBeCompiled(clinit, clinit.getModifiers())) {
827 compileMethod(clinit, libgraal);
828 }
829 println("CompileTheWorld (%d) : %s (%d us)", classFileCounter, className, (System.nanoTime() - start0) / 1000);
830 }
831 } catch (Throwable t) {
832 if (isClassIncluded(className)) {
833 println("CompileTheWorld (%d) : Skipping %s %s", classFileCounter, className, t.toString());
834 printStackTrace(t);
835 }
836 }
837 }
838 }
839 }
840
841 if (!running) {
842 startThreads();
843 }
844 int wakeups = 0;
845 long lastCompletedTaskCount = 0;
846 for (long completedTaskCount = threadPool.getCompletedTaskCount(); completedTaskCount != threadPool.getTaskCount(); completedTaskCount = threadPool.getCompletedTaskCount()) {
847 if (wakeups % 15 == 0) {
848 TTY.printf("CompileTheWorld : Waiting for %d compiles, just completed %d compiles%n", threadPool.getTaskCount() - completedTaskCount, completedTaskCount - lastCompletedTaskCount);
849 lastCompletedTaskCount = completedTaskCount;
850 }
851 try {
852 threadPool.awaitTermination(1, TimeUnit.SECONDS);
853 wakeups++;
854 } catch (InterruptedException e) {
855 }
856 }
857 threadPool.shutdown();
858 threadPool = null;
859
860 long elapsedTime = System.nanoTime() - start;
861
862 println();
863 int compiledClasses = classFileCounter > compileStartAt ? classFileCounter - compileStartAt : 0;
864 if (Options.MultiThreaded.getValue(harnessOptions)) {
865 TTY.println("CompileTheWorld : Done (%d classes, %d methods, %d ms elapsed, %d ms compile time, %d bytes of memory used)", compiledClasses, compiledMethodsCounter.get(), elapsedTime,
866 compileTime.get() / 1000000, memoryUsed.get());
867 } else {
868 TTY.println("CompileTheWorld : Done (%d classes, %d methods, %d ms, %d bytes of memory used)", compiledClasses, compiledMethodsCounter.get(), compileTime.get(), memoryUsed.get());
869 }
870
871 GlobalMetrics metricValues = ((HotSpotGraalRuntime) compiler.getGraalRuntime()).getMetricValues();
872 EconomicMap<MetricKey, Long> map = metricValues.asKeyValueMap();
873 Long compiledAndInstalledBytecodes = map.get(CompiledAndInstalledBytecodes);
874 Long compilationTime = map.get(CompilationTime);
875 if (compiledAndInstalledBytecodes != null && compilationTime != null) {
876 TTY.println("CompileTheWorld : Aggregate compile speed %d bytecodes per second (%d / %d)", (int) (compiledAndInstalledBytecodes / (compilationTime / 1000000000.0)),
877 compiledAndInstalledBytecodes, compilationTime);
878 }
879
880 metricValues.print(compilerOptions);
881 metricValues.clear();
882
883 // Apart from the main thread, there should be only be daemon threads
884 // alive now. If not, then a class initializer has probably started
885 // a thread that could cause a deadlock while trying to exit the VM.
886 // One known example of this is sun.tools.jconsole.OutputViewer which
887 // spawns threads to redirect sysout and syserr. To help debug such
888 // scenarios, the stacks of potentially problematic threads are dumped.
889 Map<Thread, StackTraceElement[]> suspiciousThreads = new HashMap<>();
890 for (Map.Entry<Thread, StackTraceElement[]> e : Thread.getAllStackTraces().entrySet()) {
891 Thread thread = e.getKey();
892 if (thread != Thread.currentThread() && !initialThreads.containsKey(thread) && !thread.isDaemon() && thread.isAlive()) {
893 suspiciousThreads.put(thread, e.getValue());
894 }
895 }
896 if (!suspiciousThreads.isEmpty()) {
897 TTY.println("--- Non-daemon threads started during CTW ---");
898 for (Map.Entry<Thread, StackTraceElement[]> e : suspiciousThreads.entrySet()) {
899 Thread thread = e.getKey();
900 if (thread.isAlive()) {
901 TTY.println(thread.toString() + " " + thread.getState());
902 for (StackTraceElement ste : e.getValue()) {
903 TTY.println("\tat " + ste);
904 }
905 }
906 }
907 TTY.println("---------------------------------------------");
908 }
909 }
910
911 private synchronized void startThreads() {
912 running = true;
913 // Wake up any waiting threads
914 notifyAll();
915 }
916
917 private synchronized void waitToRun() {
918 while (!running) {
919 try {
920 wait();
921 } catch (InterruptedException e) {
922 }
923 }
924 }
925
926 @SuppressWarnings("try")
927 private void compileMethod(HotSpotResolvedJavaMethod method, LibGraalParams libgraal) throws InterruptedException, ExecutionException {
928 if (methodFilters != null && !MethodFilter.matches(methodFilters, method)) {
929 return;
930 }
931 if (excludeMethodFilters != null && MethodFilter.matches(excludeMethodFilters, method)) {
932 return;
933 }
934 Future<?> task = threadPool.submit(new Runnable() {
935 @Override
936 public void run() {
937 waitToRun();
938 compileMethod(method, classFileCounter, libgraal);
939 }
940 });
941 if (threadPool.getCorePoolSize() == 1) {
942 task.get();
943 }
944 }
945
946 private static final Unsafe UNSAFE = GraalUnsafeAccess.getUnsafe();
947
948 /**
949 * Implemented by
950 * {@code com.oracle.svm.graal.hotspot.libgraal.LibGraalEntryPoints.compileMethod}.
951 */
952 static native long compileMethodInLibgraal(long isolateThread,
953 long methodHandle,
954 boolean useProfilingInfo,
955 boolean installAsDefault,
956 long optionsAddress,
957 int optionsSize,
958 int optionsHash,
959 long encodedThrowableBufferAddress,
960 int encodedThrowableBufferSize);
961
962 /**
963 * Compiles a method and gathers some statistics.
964 */
965 @SuppressWarnings("try")
966 private void compileMethod(HotSpotResolvedJavaMethod method, int counter, LibGraalParams libgraal) {
967 try {
968 long start = System.nanoTime();
969 long allocatedAtStart = getCurrentThreadAllocatedBytes();
970 // For more stable CTW execution, disable use of profiling information
971 boolean useProfilingInfo = false;
972 boolean installAsDefault = false;
973 HotSpotInstalledCode installedCode;
974 if (libgraal != null) {
975 HotSpotJVMCIRuntime runtime = HotSpotJVMCIRuntime.runtime();
976 try (LibGraalScope scope = new LibGraalScope(runtime)) {
977 long methodHandle = LibGraal.translate(runtime, method);
978 long isolateThread = LibGraalScope.getIsolateThread();
979
980 StackTraceBuffer stackTraceBuffer = libgraal.getStackTraceBuffer();
981
982 long stackTraceBufferAddress = stackTraceBuffer.getAddress();
983 long installedCodeHandle = compileMethodInLibgraal(isolateThread,
984 methodHandle,
985 useProfilingInfo,
986 installAsDefault,
987 libgraal.options.getAddress(),
988 libgraal.options.size,
989 libgraal.options.hash,
990 stackTraceBufferAddress,
991 stackTraceBuffer.size);
992
993 installedCode = LibGraal.unhand(runtime, HotSpotInstalledCode.class, installedCodeHandle);
994 if (installedCode == null) {
995 int length = UNSAFE.getInt(stackTraceBufferAddress);
996 byte[] data = new byte[length];
997 UNSAFE.copyMemory(null, stackTraceBufferAddress + Integer.BYTES, data, ARRAY_BYTE_BASE_OFFSET, length);
998 String stackTrace = new String(data).trim();
999 println(true, String.format("CompileTheWorld (%d) : Error compiling method: %s", counter, method.format("%H.%n(%p):%r")));
1000 println(true, stackTrace);
1001 }
1002 }
1003 } else {
1004 int entryBCI = JVMCICompiler.INVOCATION_ENTRY_BCI;
1005 HotSpotCompilationRequest request = new HotSpotCompilationRequest(method, entryBCI, 0L);
1006 CompilationTask task = new CompilationTask(jvmciRuntime, compiler, request, useProfilingInfo, installAsDefault);
1007 task.runCompilation(compilerOptions);
1008 installedCode = task.getInstalledCode();
1009 }
1010
1011 // Invalidate the generated code so the code cache doesn't fill up
1012 if (installedCode != null && InvalidateInstalledCode.getValue(compilerOptions)) {
1013 installedCode.invalidate();
1014 }
1015
1016 memoryUsed.getAndAdd(getCurrentThreadAllocatedBytes() - allocatedAtStart);
1017 compileTime.getAndAdd(System.nanoTime() - start);
1018 compiledMethodsCounter.incrementAndGet();
1019 } catch (Throwable t) {
1020 // Catch everything and print a message
1021 println("CompileTheWorld (%d) : Error compiling method: %s", counter, method.format("%H.%n(%p):%r"));
1022 printStackTrace(t);
1023 }
1024 }
1025
1026 /**
1027 * Determines if a method should be compiled (Cf. CompilationPolicy::can_be_compiled).
1028 *
1029 * @return true if it can be compiled, false otherwise
1030 */
1031 private boolean canBeCompiled(HotSpotResolvedJavaMethod javaMethod, int modifiers) {
1032 if (Modifier.isAbstract(modifiers) || Modifier.isNative(modifiers)) {
1033 return false;
1034 }
1035 GraalHotSpotVMConfig c = compiler.getGraalRuntime().getVMConfig();
1036 if (c.dontCompileHugeMethods && javaMethod.getCodeSize() > c.hugeMethodLimit) {
1037 println(verbose || methodFilters != null,
1038 String.format("CompileTheWorld (%d) : Skipping huge method %s (use -XX:-DontCompileHugeMethods or -XX:HugeMethodLimit=%d to include it)", classFileCounter,
1039 javaMethod.format("%H.%n(%p):%r"),
1040 javaMethod.getCodeSize()));
1041 return false;
1042 }
1043 // Allow use of -XX:CompileCommand=dontinline to exclude problematic methods
1044 if (!javaMethod.canBeInlined()) {
1045 return false;
1046 }
1047 // Skip @Snippets for now
1048 for (Annotation annotation : javaMethod.getAnnotations()) {
1049 if (annotation.annotationType().equals(Snippet.class)) {
1050 return false;
1051 }
1052 }
1053 return true;
1054 }
1055
1056 static class Options {
1057 public static final OptionKey<Boolean> Help = new OptionKey<>(false);
1058 public static final OptionKey<String> Classpath = new OptionKey<>(CompileTheWorld.SUN_BOOT_CLASS_PATH);
1059 public static final OptionKey<Boolean> Verbose = new OptionKey<>(true);
1060 /**
1061 * Ignore Graal classes by default to avoid problems associated with compiling snippets and
1062 * method substitutions.
1063 */
1064 public static final OptionKey<String> LimitModules = new OptionKey<>("~jdk.internal.vm.compiler");
1065 public static final OptionKey<Integer> Iterations = new OptionKey<>(1);
1066 public static final OptionKey<String> MethodFilter = new OptionKey<>(null);
1067 public static final OptionKey<String> ExcludeMethodFilter = new OptionKey<>(null);
1068 public static final OptionKey<Integer> StartAt = new OptionKey<>(1);
1069 public static final OptionKey<Integer> StopAt = new OptionKey<>(Integer.MAX_VALUE);
1070 public static final OptionKey<Integer> MaxClasses = new OptionKey<>(Integer.MAX_VALUE);
1071 public static final OptionKey<String> Config = new OptionKey<>(null);
1072 public static final OptionKey<Boolean> MultiThreaded = new OptionKey<>(false);
1073 public static final OptionKey<Integer> Threads = new OptionKey<>(0);
1074 public static final OptionKey<Boolean> InvalidateInstalledCode = new OptionKey<>(false);
1075
1076 // @formatter:off
1077 static final ReflectionOptionDescriptors DESCRIPTORS = new ReflectionOptionDescriptors(Options.class,
1078 "Help", "List options and their help messages and then exit.",
1079 "Classpath", "Class path denoting methods to compile. Default is to compile boot classes.",
1080 "Verbose", "Verbose operation. Default is !MultiThreaded.",
1081 "LimitModules", "Comma separated list of module names to which compilation should be limited. " +
1082 "Module names can be prefixed with \"~\" to exclude the named module.",
1083 "Iterations", "The number of iterations to perform.",
1084 "MethodFilter", "Only compile methods matching this filter.",
1085 "ExcludeMethodFilter", "Exclude methods matching this filter from compilation.",
1086 "StartAt", "First class to consider for compilation (default = 1).",
1087 "StopAt", "Last class to consider for compilation (default = <number of classes>).",
1088 "MaxClasses", "Maximum number of classes to process (default = <number of classes>). " +
1089 "Ignored if less than (StopAt - StartAt + 1).",
1090 "Config", "Option values to use during compile the world compilations. For example, " +
1091 "to disable partial escape analysis and print compilations specify " +
1092 "'PartialEscapeAnalysis=false PrintCompilation=true'. " +
1093 "Unless explicitly enabled with 'Inline=true' here, inlining is disabled.",
1094 "MultiThreaded", "Run using multiple threads for compilation.",
1095 "Threads", "Number of threads to use for multithreaded execution. Defaults to Runtime.getRuntime().availableProcessors().");
1096 // @formatter:on
1097 }
1098
1099 public static OptionValues loadHarnessOptions() {
1100 EconomicMap<OptionKey<?>, Object> values = OptionValues.newOptionMap();
1101 List<OptionDescriptors> loader = singletonList(DESCRIPTORS);
1102 OptionsParser.parseOptions(extractEntries(System.getProperties(), "CompileTheWorld.", true), values, loader);
1103 OptionValues options = new OptionValues(values);
1104 if (Options.Help.getValue(options)) {
1105 options.printHelp(loader, System.out, "CompileTheWorld.");
1106 System.exit(0);
1107 }
1108 return options;
1109 }
1110
1111 public static void main(String[] args) throws Throwable {
1112 HotSpotJVMCIRuntime jvmciRuntime = HotSpotJVMCIRuntime.runtime();
1113 HotSpotGraalCompiler compiler = (HotSpotGraalCompiler) jvmciRuntime.getCompiler();
1114 HotSpotGraalRuntimeProvider graalRuntime = compiler.getGraalRuntime();
1115 HotSpotCodeCacheProvider codeCache = graalRuntime.getHostProviders().getCodeCache();
1116 OptionValues harnessOptions = loadHarnessOptions();
1117
1118 int iterations = Options.Iterations.getValue(harnessOptions);
1119 for (int i = 0; i < iterations; i++) {
1120 codeCache.resetCompilationStatistics();
1121 TTY.println("CompileTheWorld : iteration " + i);
1122
1123 CompileTheWorld ctw = new CompileTheWorld(jvmciRuntime, compiler, harnessOptions, graalRuntime.getOptions());
1124 ctw.compile();
1125 if (iterations > 1) {
1126 // Force a GC to encourage reclamation of nmethods when their InstalledCode
1127 // reference has been dropped.
1128 System.gc();
1129 }
1130 }
1131 // This is required as non-daemon threads can be started by class initializers
1132 System.exit(0);
1133 }
1134 }