1 /*
   2  * Copyright (c) 2010, 2016, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 package jdk.nashorn.internal.test.framework;
  26 
  27 import static jdk.nashorn.internal.test.framework.TestConfig.OPTIONS_CHECK_COMPILE_MSG;
  28 import static jdk.nashorn.internal.test.framework.TestConfig.OPTIONS_COMPARE;
  29 import static jdk.nashorn.internal.test.framework.TestConfig.OPTIONS_EXPECT_COMPILE_FAIL;
  30 import static jdk.nashorn.internal.test.framework.TestConfig.OPTIONS_EXPECT_RUN_FAIL;
  31 import static jdk.nashorn.internal.test.framework.TestConfig.OPTIONS_FORK;
  32 import static jdk.nashorn.internal.test.framework.TestConfig.OPTIONS_IGNORE_STD_ERROR;
  33 import static jdk.nashorn.internal.test.framework.TestConfig.OPTIONS_RUN;
  34 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_FAILED_LIST_FILE;
  35 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_ENABLE_STRICT_MODE;
  36 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_EXCLUDES_FILE;
  37 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_EXCLUDE_DIR;
  38 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_EXCLUDE_LIST;
  39 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_FRAMEWORK;
  40 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_INCLUDES;
  41 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_LIST;
  42 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_ROOTS;
  43 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_UNCHECKED_DIR;
  44 import java.io.BufferedReader;
  45 import java.io.File;
  46 import java.io.FileReader;
  47 import java.io.IOException;
  48 import java.nio.ByteOrder;
  49 import java.nio.file.FileSystem;
  50 import java.nio.file.FileSystems;
  51 import java.nio.file.FileVisitOption;
  52 import java.nio.file.FileVisitResult;
  53 import java.nio.file.Files;
  54 import java.nio.file.Path;
  55 import java.nio.file.SimpleFileVisitor;
  56 import java.nio.file.attribute.BasicFileAttributes;
  57 import java.util.ArrayList;
  58 import java.util.Arrays;
  59 import java.util.Collections;
  60 import java.util.EnumSet;
  61 import java.util.HashMap;
  62 import java.util.HashSet;
  63 import java.util.Iterator;
  64 import java.util.List;
  65 import java.util.Map;
  66 import java.util.Set;
  67 import javax.xml.xpath.XPath;
  68 import javax.xml.xpath.XPathConstants;
  69 import javax.xml.xpath.XPathExpressionException;
  70 import javax.xml.xpath.XPathFactory;
  71 import jdk.nashorn.tools.Shell;
  72 import org.w3c.dom.NodeList;
  73 import org.xml.sax.InputSource;
  74 
  75 /**
  76  * Utility class to find/parse script test files and to create 'test' instances.
  77  * Actual 'test' object type is decided by clients of this class.
  78  */
  79 @SuppressWarnings("javadoc")
  80 public final class TestFinder {
  81 
  82     private TestFinder() {
  83     }
  84 
  85     interface TestFactory<T> {
  86 
  87         // 'test' instance type is decided by the client.
  88 
  89         T createTest(final String framework, final File testFile, final List<String> engineOptions, final Map<String, String> testOptions, final List<String> arguments);
  90 
  91         // place to log messages from TestFinder
  92 
  93         void log(String mg);
  94     }
  95 
  96     // finds all tests from configuration and calls TestFactory to create 'test' instance for each script test found
  97     static <T> void findAllTests(final List<T> tests, final Set<String> orphans, final TestFactory<T> testFactory) throws Exception {
  98         final String framework = System.getProperty(TEST_JS_FRAMEWORK);
  99         final String testList = System.getProperty(TEST_JS_LIST);
 100         final String failedTestFileName = System.getProperty(TEST_FAILED_LIST_FILE);
 101         if (failedTestFileName != null) {
 102             final File failedTestFile = new File(failedTestFileName);
 103             if (failedTestFile.exists() && failedTestFile.length() > 0L) {
 104                 try (final BufferedReader r = new BufferedReader(new FileReader(failedTestFile))) {
 105                     for (;;) {
 106                         final String testFileName = r.readLine();
 107                         if (testFileName == null) {
 108                             break;
 109                         }
 110                         handleOneTest(framework, new File(testFileName).toPath(), tests, orphans, testFactory);
 111                     }
 112                 }
 113                 return;
 114             }
 115         }
 116         if (testList == null || testList.length() == 0) {
 117             // Run the tests under the test roots dir, selected by the
 118             // TEST_JS_INCLUDES patterns
 119             final String testRootsString = System.getProperty(TEST_JS_ROOTS, "test/nashorn/script");
 120             if (testRootsString == null || testRootsString.length() == 0) {
 121                 throw new Exception("Error: " + TEST_JS_ROOTS + " must be set");
 122             }
 123             final String testRoots[] = testRootsString.split(" ");
 124             final FileSystem fileSystem = FileSystems.getDefault();
 125             final Set<String> testExcludeSet = getExcludeSet();
 126             final Path[] excludePaths = getExcludeDirs();
 127             for (final String root : testRoots) {
 128                 final Path dir = fileSystem.getPath(root);
 129                 findTests(framework, dir, tests, orphans, excludePaths, testExcludeSet, testFactory);
 130             }
 131         } else {
 132             // TEST_JS_LIST contains a blank speparated list of test file names.
 133             final String strArray[] = testList.split(" ");
 134             for (final String ss : strArray) {
 135                 handleOneTest(framework, new File(ss).toPath(), tests, orphans, testFactory);
 136             }
 137         }
 138     }
 139 
 140     private static boolean inExcludePath(final Path file, final Path[] excludePaths) {
 141         if (excludePaths == null) {
 142             return false;
 143         }
 144 
 145         for (final Path excludePath : excludePaths) {
 146             if (file.startsWith(excludePath)) {
 147                 return true;
 148             }
 149         }
 150         return false;
 151     }
 152 
 153     private static <T> void findTests(final String framework, final Path dir, final List<T> tests, final Set<String> orphanFiles, final Path[] excludePaths, final Set<String> excludedTests, final TestFactory<T> factory) throws Exception {
 154         final String pattern = System.getProperty(TEST_JS_INCLUDES);
 155         final String extension = pattern == null ? "js" : pattern;
 156         final Exception[] exceptions = new Exception[1];
 157         final List<String> excludedActualTests = new ArrayList<>();
 158 
 159         if (!dir.toFile().isDirectory()) {
 160             factory.log("WARNING: " + dir + " not found or not a directory");
 161         }
 162 
 163 
 164         Files.walkFileTree(dir, EnumSet.of(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, new SimpleFileVisitor<Path>() {
 165             @Override
 166             public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException {
 167                 final String fileName = file.getName(file.getNameCount() - 1).toString();
 168                 if (fileName.endsWith(extension)) {
 169                     final String namex = file.toString().replace('\\', '/');
 170                     if (!inExcludePath(file, excludePaths) && !excludedTests.contains(file.getFileName().toString())) {
 171                         try {
 172                             handleOneTest(framework, file, tests, orphanFiles, factory);
 173                         } catch (final Exception ex) {
 174                             exceptions[0] = ex;
 175                             return FileVisitResult.TERMINATE;
 176                         }
 177                     } else {
 178                         excludedActualTests.add(namex);
 179                     }
 180                 }
 181                 return FileVisitResult.CONTINUE;
 182             }
 183         });
 184         Collections.sort(excludedActualTests);
 185 
 186         for (final String excluded : excludedActualTests) {
 187             factory.log("Excluding " + excluded);
 188         }
 189 
 190         if (exceptions[0] != null) {
 191             throw exceptions[0];
 192         }
 193     }
 194 
 195     private static final String uncheckedDirs[] = System.getProperty(TEST_JS_UNCHECKED_DIR, "test/nashorn/script/external/test262/").split(" ");
 196 
 197     private static boolean isUnchecked(final Path testFile) {
 198         for (final String uncheckedDir : uncheckedDirs) {
 199             if (testFile.startsWith(uncheckedDir)) {
 200                 return true;
 201             }
 202         }
 203         return false;
 204     }
 205 
 206     private static <T> void handleOneTest(final String framework, final Path testFile, final List<T> tests, final Set<String> orphans, final TestFactory<T> factory) throws Exception {
 207         final String name = testFile.getFileName().toString();
 208 
 209         assert name.lastIndexOf(".js") > 0 : "not a JavaScript: " + name;
 210 
 211         // defaults: testFile is a test and should be run
 212         boolean isTest = isUnchecked(testFile);
 213         boolean isNotTest = false;
 214         boolean shouldRun = true;
 215         boolean compileFailure = false;
 216         boolean runFailure = false;
 217         boolean checkCompilerMsg = false;
 218         boolean noCompare = false;
 219         boolean ignoreStdError = false;
 220         boolean fork = false;
 221 
 222         final List<String> engineOptions = new ArrayList<>();
 223         final List<String> scriptArguments = new ArrayList<>();
 224         boolean inComment = false;
 225 
 226         boolean explicitOptimistic = false;
 227 
 228         final String allContent = new String(Files.readAllBytes(testFile));
 229         final Iterator<String> scanner = Shell.tokenizeString(allContent).iterator();
 230         while (scanner.hasNext()) {
 231             // TODO: Scan for /ref=file qualifiers, etc, to determine run
 232             // behavior
 233             String token = scanner.next();
 234             if (token.startsWith("/*")) {
 235                 inComment = true;
 236             } else if (token.endsWith(("*/"))) {
 237                 inComment = false;
 238             } else if (!inComment) {
 239                 continue;
 240             }
 241 
 242             // remove whitespace and trailing semicolons, if any
 243             // (trailing semicolons are found in some sputnik tests)
 244             token = token.trim();
 245             final int semicolon = token.indexOf(';');
 246             if (semicolon > 0) {
 247                 token = token.substring(0, semicolon);
 248             }
 249             switch (token) {
 250                 case "@test":
 251                     isTest = true;
 252                     break;
 253                 case "@test/fail":
 254                     isTest = true;
 255                     compileFailure = true;
 256                     break;
 257                 case "@test/compile-error":
 258                     isTest = true;
 259                     compileFailure = true;
 260                     checkCompilerMsg = true;
 261                     shouldRun = false;
 262                     break;
 263                 case "@test/warning":
 264                     isTest = true;
 265                     checkCompilerMsg = true;
 266                     break;
 267                 case "@test/nocompare":
 268                     isTest = true;
 269                     noCompare = true;
 270                     break;
 271                 case "@subtest":
 272                     isTest = false;
 273                     isNotTest = true;
 274                     break;
 275                 case "@bigendian":
 276                     shouldRun = ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN;
 277                     break;
 278                 case "@littleendian":
 279                     shouldRun = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN;
 280                     break;
 281                 case "@runif": {
 282                     final String prop = scanner.next();
 283                     if (System.getProperty(prop) != null) {
 284                         shouldRun = true;
 285                     } else {
 286                         factory.log("WARNING: (" + prop + ") skipping " + testFile);
 287                         isTest = false;
 288                         isNotTest = true;
 289                     }
 290                     break;
 291                 }
 292                 case "@run":
 293                     shouldRun = true;
 294                     break;
 295                 case "@run/fail":
 296                     shouldRun = true;
 297                     runFailure = true;
 298                     break;
 299                 case "@run/ignore-std-error":
 300                     shouldRun = true;
 301                     ignoreStdError = true;
 302                     break;
 303                 case "@argument":
 304                     scriptArguments.add(scanner.next());
 305                     break;
 306                 case "@option":
 307                     final String next = scanner.next();
 308                     engineOptions.add(next);
 309                     if (next.startsWith("--optimistic-types")) {
 310                         explicitOptimistic = true;
 311                     }
 312                     break;
 313                 case "@fork":
 314                     fork = true;
 315                     break;
 316                 default:
 317                     break;
 318             }
 319 
 320             // negative tests are expected to fail at runtime only
 321             // for those tests that are expected to fail at compile time,
 322             // add @test/compile-error
 323             if (token.equals("@negative") || token.equals("@strict_mode_negative")) {
 324                 shouldRun = true;
 325                 runFailure = true;
 326             }
 327 
 328             if (token.equals("@strict_mode") || token.equals("@strict_mode_negative") || token.equals("@onlyStrict") || token.equals("@noStrict")) {
 329                 if (!strictModeEnabled()) {
 330                     return;
 331                 }
 332             }
 333         }
 334 
 335         if (isTest) {
 336             final Map<String, String> testOptions = new HashMap<>();
 337             if (compileFailure) {
 338                 testOptions.put(OPTIONS_EXPECT_COMPILE_FAIL, "true");
 339             }
 340             if (shouldRun) {
 341                 testOptions.put(OPTIONS_RUN, "true");
 342             }
 343             if (runFailure) {
 344                 testOptions.put(OPTIONS_EXPECT_RUN_FAIL, "true");
 345             }
 346             if (checkCompilerMsg) {
 347                 testOptions.put(OPTIONS_CHECK_COMPILE_MSG, "true");
 348             }
 349             if (!noCompare) {
 350                 testOptions.put(OPTIONS_COMPARE, "true");
 351             }
 352             if (ignoreStdError) {
 353                 testOptions.put(OPTIONS_IGNORE_STD_ERROR, "true");
 354             }
 355             if (fork) {
 356                 testOptions.put(OPTIONS_FORK, "true");
 357             }
 358 
 359             //if there are explicit optimistic type settings, use those - do not override
 360             //the test might only work with optimistic types on or off.
 361             if (!explicitOptimistic) {
 362                 addExplicitOptimisticTypes(engineOptions);
 363             }
 364 
 365             tests.add(factory.createTest(framework, testFile.toFile(), engineOptions, testOptions, scriptArguments));
 366         } else if (!isNotTest) {
 367             orphans.add(name);
 368         }
 369     }
 370 
 371     //the reverse of the default setting for optimistic types, if enabled, false, otherwise true
 372     //thus, true for 8u40, false for 9
 373     private static final boolean OPTIMISTIC_OVERRIDE = false;
 374 
 375     /**
 376      * Check if there is an optimistic override, that disables the default false
 377      * optimistic types and sets them to true, for testing purposes
 378      *
 379      * @return true if optimistic type override has been set by test suite
 380      */
 381     public static boolean hasOptimisticOverride() {
 382         return Boolean.toString(OPTIMISTIC_OVERRIDE).equals(System.getProperty("optimistic.override"));
 383     }
 384 
 385     /**
 386      * Add an optimistic-types=true option to an argument list if this is set to
 387      * override the default false. Add an optimistic-types=true options to an
 388      * argument list if this is set to override the default true
 389      *
 390      * @args new argument list array
 391      */
 392     public static String[] addExplicitOptimisticTypes(final String[] args) {
 393         if (hasOptimisticOverride()) {
 394             final List<String> newList = new ArrayList<>(Arrays.asList(args));
 395             newList.add("--optimistic-types=" + OPTIMISTIC_OVERRIDE);
 396             return newList.toArray(new String[0]);
 397         }
 398         return args;
 399     }
 400 
 401     /**
 402      * Add an optimistic-types=true option to an argument list if this is set to
 403      * override the default false
 404      *
 405      * @args argument list
 406      */
 407     public static void addExplicitOptimisticTypes(final List<String> args) {
 408         if (hasOptimisticOverride()) {
 409             args.add("--optimistic-types=" + OPTIMISTIC_OVERRIDE);
 410         }
 411     }
 412 
 413     private static boolean strictModeEnabled() {
 414         return Boolean.getBoolean(TEST_JS_ENABLE_STRICT_MODE);
 415     }
 416 
 417     private static Set<String> getExcludeSet() throws XPathExpressionException {
 418         final String testExcludeList = System.getProperty(TEST_JS_EXCLUDE_LIST);
 419 
 420         String[] testExcludeArray = {};
 421         if (testExcludeList != null) {
 422             testExcludeArray = testExcludeList.split(" ");
 423         }
 424         final Set<String> testExcludeSet = new HashSet<>(testExcludeArray.length);
 425         for (final String test : testExcludeArray) {
 426             testExcludeSet.add(test);
 427         }
 428 
 429         final String testExcludesFile = System.getProperty(TEST_JS_EXCLUDES_FILE);
 430         if (testExcludesFile != null && !testExcludesFile.isEmpty()) {
 431             try {
 432                 loadExcludesFile(testExcludesFile, testExcludeSet);
 433             } catch (final XPathExpressionException e) {
 434                 System.err.println("Error: unable to load test excludes from " + testExcludesFile);
 435                 e.printStackTrace();
 436                 throw e;
 437             }
 438         }
 439         return testExcludeSet;
 440     }
 441 
 442     private static void loadExcludesFile(final String testExcludesFile, final Set<String> testExcludeSet) throws XPathExpressionException {
 443         final XPath xpath = XPathFactory.newInstance().newXPath();
 444         final NodeList testIds = (NodeList) xpath.evaluate("/excludeList/test/@id", new InputSource(testExcludesFile), XPathConstants.NODESET);
 445         for (int i = testIds.getLength() - 1; i >= 0; i--) {
 446             testExcludeSet.add(testIds.item(i).getNodeValue());
 447         }
 448     }
 449 
 450     private static Path[] getExcludeDirs() {
 451         final String excludeDirs[] = System.getProperty(TEST_JS_EXCLUDE_DIR, "test/nashorn/script/currently-failing").split(" ");
 452         final Path[] excludePaths = new Path[excludeDirs.length];
 453         final FileSystem fileSystem = FileSystems.getDefault();
 454         int i = 0;
 455         for (final String excludeDir : excludeDirs) {
 456             excludePaths[i++] = fileSystem.getPath(excludeDir);
 457         }
 458         return excludePaths;
 459     }
 460 }