1 /*
2 * Copyright (c) 2010, 2016, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25
26 package jdk.nashorn.internal.test.framework;
27
28 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_FAILED_LIST_FILE;
29 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_ENABLE_STRICT_MODE;
30 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_EXCLUDES_FILE;
31 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_EXCLUDE_LIST;
32 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_FRAMEWORK;
33 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_ROOTS;
34 import java.io.BufferedReader;
35 import java.io.ByteArrayOutputStream;
36 import java.io.File;
37 import java.io.FileInputStream;
38 import java.io.FileOutputStream;
39 import java.io.FileReader;
40 import java.io.FileWriter;
41 import java.io.IOException;
42 import java.io.InputStreamReader;
43 import java.io.OutputStream;
44 import java.io.PrintStream;
45 import java.io.PrintWriter;
46 import java.io.StringReader;
47 import java.nio.file.FileSystems;
48 import java.nio.file.Files;
49 import java.nio.file.StandardCopyOption;
50 import java.util.ArrayList;
51 import java.util.Collections;
52 import java.util.Comparator;
53 import java.util.List;
54 import java.util.Locale;
55 import java.util.Map;
56 import java.util.Properties;
57 import java.util.Set;
58 import java.util.TreeSet;
59 import java.util.concurrent.Callable;
60 import java.util.concurrent.CancellationException;
61 import java.util.concurrent.CountDownLatch;
62 import java.util.concurrent.ExecutionException;
63 import java.util.concurrent.ExecutorService;
64 import java.util.concurrent.Executors;
65 import java.util.concurrent.Future;
66 import java.util.concurrent.TimeUnit;
67 import java.util.regex.Matcher;
68 import java.util.regex.Pattern;
69 import jdk.nashorn.internal.test.framework.TestFinder.TestFactory;
70
71 /**
72 * Parallel test runner runs tests in multiple threads - but avoids any dependency
73 * on third-party test framework library such as TestNG.
74 */
75 @SuppressWarnings("javadoc")
76 public class ParallelTestRunner {
77
78 // ParallelTestRunner-specific
79 private static final String TEST_JS_THREADS = "test.js.threads";
80 private static final String TEST_JS_REPORT_FILE = "test.js.report.file";
81 // test262 does a lot of eval's and the JVM hates multithreaded class definition, so lower thread count is usually faster.
82 private static final int THREADS = Integer.getInteger(TEST_JS_THREADS, Runtime.getRuntime().availableProcessors() > 4 ? 4 : 2);
83
84 private final List<ScriptRunnable> tests = new ArrayList<>();
85 private final Set<String> orphans = new TreeSet<>();
86 private final ExecutorService executor = Executors.newFixedThreadPool(THREADS);
87
88 // Ctrl-C handling
89 private final CountDownLatch finishedLatch = new CountDownLatch(1);
90 private final Thread shutdownHook = new Thread() {
91 @Override
92 public void run() {
93 if (!executor.isTerminated()) {
94 executor.shutdownNow();
95 try {
96 executor.awaitTermination(25, TimeUnit.SECONDS);
97 finishedLatch.await(5, TimeUnit.SECONDS);
98 } catch (final InterruptedException e) {
99 // empty
100 }
101 }
102 }
103 };
104
105 public ParallelTestRunner() throws Exception {
106 suite();
107 }
108
109 private static PrintStream outputStream() {
110 final String reportFile = System.getProperty(TEST_JS_REPORT_FILE, "");
111 PrintStream output = System.out;
112
113 if (!reportFile.isEmpty()) {
114 try {
115 output = new PrintStream(new OutputStreamDelegator(System.out, new FileOutputStream(reportFile)));
116 } catch (final IOException e) {
117 System.err.println(e);
118 }
119 }
120
121 return output;
122 }
123
124 public static final class ScriptRunnable extends AbstractScriptRunnable implements Callable<ScriptRunnable.Result> {
125 private final Result result = new Result();
126
127 public class Result {
128 private boolean passed = true;
129 public String expected;
130 public String out;
131 public String err;
132 public Throwable exception;
133
134 public ScriptRunnable getTest() {
135 return ScriptRunnable.this;
136 }
137
138 public boolean passed() {
139 return passed;
140 }
141
142 @Override
143 public String toString() {
144 return getTest().toString();
145 }
146 }
147
148 public ScriptRunnable(final String framework, final File testFile, final List<String> engineOptions, final Map<String, String> testOptions, final List<String> scriptArguments) {
149 super(framework, testFile, engineOptions, testOptions, scriptArguments);
150 }
151
152 @Override
153 protected void log(final String msg) {
154 System.err.println(msg);
155 }
156
157 @Override
158 protected void fail(final String message) {
159 throw new TestFailedError(message);
160 }
161
162 @Override
163 protected void compile() throws IOException {
164 final ByteArrayOutputStream out = new ByteArrayOutputStream();
165 final ByteArrayOutputStream err = new ByteArrayOutputStream();
166 final List<String> args = getCompilerArgs();
167 int errors;
168 try {
169 errors = evaluateScript(out, err, args.toArray(new String[0]));
170 } catch (final AssertionError e) {
171 final PrintWriter writer = new PrintWriter(err);
172 e.printStackTrace(writer);
173 writer.flush();
174 errors = 1;
175 }
176 if (errors != 0 || checkCompilerMsg) {
177 result.err = err.toString();
178 if (expectCompileFailure || checkCompilerMsg) {
179 final PrintStream outputDest = new PrintStream(new FileOutputStream(getErrorFileName()));
180 TestHelper.dumpFile(outputDest, new StringReader(new String(err.toByteArray())));
181 outputDest.println("--");
182 }
183 if (errors != 0 && !expectCompileFailure) {
184 fail(String.format("%d errors compiling %s", errors, testFile));
185 }
186 if (checkCompilerMsg) {
187 compare(getErrorFileName(), expectedFileName, true);
188 }
189 }
190 if (expectCompileFailure && errors == 0) {
191 fail(String.format("No errors encountered compiling negative test %s", testFile));
192 }
193 }
194
195 @Override
196 protected void execute() {
197 final List<String> args = getRuntimeArgs();
198 final ByteArrayOutputStream out = new ByteArrayOutputStream();
199 final ByteArrayOutputStream err = new ByteArrayOutputStream();
200
201 try {
202 final int errors = evaluateScript(out, err, args.toArray(new String[0]));
203
204 if (errors != 0 || err.size() > 0) {
205 if (expectRunFailure) {
206 return;
207 }
208 if (!ignoreStdError) {
209
210 try (OutputStream outputFile = new FileOutputStream(getOutputFileName()); OutputStream errorFile = new FileOutputStream(getErrorFileName())) {
211 outputFile.write(out.toByteArray());
212 errorFile.write(err.toByteArray());
213 }
214
215 result.out = out.toString();
216 result.err = err.toString();
217 fail(err.toString());
218 }
219 }
220
221 if (compare) {
222 final File expectedFile = new File(expectedFileName);
223 try {
224 BufferedReader expected;
225 if (expectedFile.exists()) {
226 expected = new BufferedReader(new FileReader(expectedFile));
227 } else {
228 expected = new BufferedReader(new StringReader(""));
229 }
230 compare(new BufferedReader(new StringReader(out.toString())), expected, false);
231 } catch (final Throwable ex) {
232 if (expectedFile.exists()) {
233 copyExpectedFile();
234 }
235 try (OutputStream outputFile = new FileOutputStream(getOutputFileName()); OutputStream errorFile = new FileOutputStream(getErrorFileName())) {
236 outputFile.write(out.toByteArray());
237 errorFile.write(err.toByteArray());
238 }
239 ex.printStackTrace();
240 throw ex;
241 }
242 }
243 } catch (final IOException e) {
244 if (!expectRunFailure) {
245 fail("Failure running test " + testFile + ": " + e.getMessage());
246 } // else success
247 }
248 }
249
250 private void compare(final String fileName, final String expected, final boolean compareCompilerMsg) throws IOException {
251 final File expectedFile = new File(expected);
252
253 BufferedReader expectedReader;
254 if (expectedFile.exists()) {
255 expectedReader = new BufferedReader(new InputStreamReader(new FileInputStream(expectedFileName)));
256 } else {
257 expectedReader = new BufferedReader(new StringReader(""));
258 }
259
260 final BufferedReader actual = new BufferedReader(new InputStreamReader(new FileInputStream(fileName)));
261
262 compare(actual, expectedReader, compareCompilerMsg);
263 }
264
265 private void copyExpectedFile() {
266 if (!new File(expectedFileName).exists()) {
267 return;
268 }
269 // copy expected file overwriting existing file and preserving last
270 // modified time of source
271 try {
272 Files.copy(FileSystems.getDefault().getPath(expectedFileName), FileSystems.getDefault().getPath(getCopyExpectedFileName()), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES);
273 } catch (final IOException ex) {
274 fail("failed to copy expected " + expectedFileName + " to " + getCopyExpectedFileName() + ": " + ex.getMessage());
275 }
276 }
277
278 @Override
279 public Result call() {
280 try {
281 runTest();
282 } catch (final Throwable ex) {
283 result.exception = ex;
284 result.passed = false;
285 ex.printStackTrace();
286 }
287 return result;
288 }
289
290 private String getOutputFileName() {
291 buildDir.mkdirs();
292 return outputFileName;
293 }
294
295 private String getErrorFileName() {
296 buildDir.mkdirs();
297 return errorFileName;
298 }
299
300 private String getCopyExpectedFileName() {
301 buildDir.mkdirs();
302 return copyExpectedFileName;
303 }
304 }
305
306 private void suite() throws Exception {
307 Locale.setDefault(new Locale(""));
308 System.setOut(outputStream());
309
310 final TestFactory<ScriptRunnable> testFactory = new TestFactory<ScriptRunnable>() {
311 @Override
312 public ScriptRunnable createTest(final String framework, final File testFile, final List<String> engineOptions, final Map<String, String> testOptions, final List<String> arguments) {
313 return new ScriptRunnable(framework, testFile, engineOptions, testOptions, arguments);
314 }
315
316 @Override
317 public void log(final String msg) {
318 System.err.println(msg);
319 }
320 };
321
322 TestFinder.findAllTests(tests, orphans, testFactory);
323
324 Collections.sort(tests, new Comparator<ScriptRunnable>() {
325 @Override
326 public int compare(final ScriptRunnable o1, final ScriptRunnable o2) {
327 return o1.testFile.compareTo(o2.testFile);
328 }
329 });
330 }
331
332 @SuppressWarnings("resource")
333 public boolean run() throws IOException {
334 final int testCount = tests.size();
335 int passCount = 0;
336 int doneCount = 0;
337 System.out.printf("Found %d tests.\n", testCount);
338 final long startTime = System.nanoTime();
339
340 Runtime.getRuntime().addShutdownHook(shutdownHook);
341
342 final List<Future<ScriptRunnable.Result>> futures = new ArrayList<>();
343 for (final ScriptRunnable test : tests) {
344 futures.add(executor.submit(test));
345 }
346
347 executor.shutdown();
348 try {
349 executor.awaitTermination(60, TimeUnit.MINUTES);
350 } catch (final InterruptedException ex) {
351 // empty
352 }
353
354 final List<ScriptRunnable.Result> results = new ArrayList<>();
355 for (final Future<ScriptRunnable.Result> future : futures) {
356 if (future.isDone()) {
357 try {
358 final ScriptRunnable.Result result = future.get();
359 results.add(result);
360 doneCount++;
361 if (result.passed()) {
362 passCount++;
363 }
364 } catch (CancellationException | ExecutionException ex) {
365 ex.printStackTrace();
366 } catch (final InterruptedException ex) {
367 assert false : "should not reach here";
368 }
369 }
370 }
371
372 Collections.sort(results, new Comparator<ScriptRunnable.Result>() {
373 @Override
374 public int compare(final ScriptRunnable.Result o1, final ScriptRunnable.Result o2) {
375 return o1.getTest().testFile.compareTo(o2.getTest().testFile);
376 }
377 });
378
379 boolean hasFailed = false;
380 final String failedList = System.getProperty(TEST_FAILED_LIST_FILE);
381 final boolean hasFailedList = failedList != null;
382 final boolean hadPreviouslyFailingTests = hasFailedList && new File(failedList).length() > 0;
383 final FileWriter failedFileWriter = hasFailedList ? new FileWriter(failedList) : null;
384 try {
385 final PrintWriter failedListWriter = failedFileWriter == null ? null : new PrintWriter(failedFileWriter);
386 for (final ScriptRunnable.Result result : results) {
387 if (!result.passed()) {
388 if (hasFailed == false) {
389 hasFailed = true;
390 System.out.println();
391 System.out.println("FAILED TESTS");
392 }
393
394 System.out.println(result.getTest());
395 if(failedFileWriter != null) {
396 failedListWriter.println(result.getTest().testFile.getPath());
397 }
398 if (result.exception != null) {
399 final String exceptionString = result.exception instanceof TestFailedError ? result.exception.getMessage() : result.exception.toString();
400 System.out.print(exceptionString.endsWith("\n") ? exceptionString : exceptionString + "\n");
401 System.out.print(result.out != null ? result.out : "");
402 }
403 }
404 }
405 } finally {
406 if(failedFileWriter != null) {
407 failedFileWriter.close();
408 }
409 }
410 final double timeElapsed = (System.nanoTime() - startTime) / 1e9; // [s]
411 System.out.printf("Tests run: %d/%d tests, passed: %d (%.2f%%), failed: %d. Time elapsed: %.0fmin %.0fs.\n", doneCount, testCount, passCount, 100d * passCount / doneCount, doneCount - passCount, timeElapsed / 60, timeElapsed % 60);
412 System.out.flush();
413
414 finishedLatch.countDown();
415
416 if (hasFailed) {
417 throw new AssertionError("TEST FAILED");
418 }
419
420 if(hasFailedList) {
421 new File(failedList).delete();
422 }
423
424 if(hadPreviouslyFailingTests) {
425 System.out.println();
426 System.out.println("Good job on getting all your previously failing tests pass!");
427 System.out.println("NOW re-running all tests to make sure you haven't caused any NEW test failures.");
428 System.out.println();
429 }
430
431 return hadPreviouslyFailingTests;
432 }
433
434 public static void main(final String[] args) throws Exception {
435 parseArgs(args);
436
437 while (new ParallelTestRunner().run()) {
438 //empty
439 }
440 }
441
442 private static void parseArgs(final String[] args) {
443 if (args.length > 0) {
444 String roots = "";
445 String reportFile = "";
446 for (int i = 0; i < args.length; i++) {
447 if (args[i].equals("--roots") && i != args.length - 1) {
448 roots += args[++i] + " ";
449 } else if (args[i].equals("--report-file") && i != args.length - 1) {
450 reportFile = args[++i];
451 } else if (args[i].equals("--test262")) {
452 try {
453 setTest262Properties();
454 } catch (final IOException ex) {
455 System.err.println(ex);
456 }
457 }
458 }
459 if (!roots.isEmpty()) {
460 System.setProperty(TEST_JS_ROOTS, roots.trim());
461 }
462 if (!reportFile.isEmpty()) {
463 System.setProperty(TEST_JS_REPORT_FILE, reportFile);
464 }
465 }
466 }
467
468 private static void setTest262Properties() throws IOException {
469 System.setProperty(TEST_JS_ROOTS, "test/nashorn/script/external/test262/test/suite/");
470 System.setProperty(TEST_JS_FRAMEWORK, "test/nashorn/script/test262.js test/nashorn/script/external/test262/test/harness/framework.js test/nashorn/script/external/test262/test/harness/sta.js");
471 System.setProperty(TEST_JS_EXCLUDES_FILE, "test/nashorn/script/external/test262/test/config/excludelist.xml");
472 System.setProperty(TEST_JS_ENABLE_STRICT_MODE, "true");
473
474 final Properties projectProperties = new Properties();
475 projectProperties.load(new FileInputStream("project.properties"));
476 String excludeList = projectProperties.getProperty("test262-test-sys-prop.test.js.exclude.list", "");
477 final Pattern pattern = Pattern.compile("\\$\\{([^}]+)}");
478 for (;;) {
479 final Matcher matcher = pattern.matcher(excludeList);
480 if (!matcher.find()) {
481 break;
482 }
483 final String propertyValue = projectProperties.getProperty(matcher.group(1), "");
484 excludeList = excludeList.substring(0, matcher.start()) + propertyValue + excludeList.substring(matcher.end());
485 }
486 System.setProperty(TEST_JS_EXCLUDE_LIST, excludeList);
487 }
488
489 public static final class OutputStreamDelegator extends OutputStream {
490 private final OutputStream[] streams;
491
492 public OutputStreamDelegator(final OutputStream... streams) {
493 this.streams = streams;
494 }
495
496 @Override
497 public void write(final int b) throws IOException {
498 for (final OutputStream stream : streams) {
499 stream.write(b);
500 }
501 }
502
503 @Override
504 public void flush() throws IOException {
505 for (final OutputStream stream : streams) {
506 stream.flush();
507 }
508 }
509 }
510 }
511
512 final class TestFailedError extends Error {
513 private static final long serialVersionUID = 1L;
514
515 public TestFailedError(final String message) {
516 super(message);
517 }
518
519 public TestFailedError(final String message, final Throwable cause) {
520 super(message, cause);
521 }
522
523 public TestFailedError(final Throwable cause) {
524 super(cause);
525 }
526 }