1 /*
   2  * Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 
  26 package jdk.nashorn.internal.test.framework;
  27 
  28 import static jdk.nashorn.internal.test.framework.TestConfig.OPTIONS_CHECK_COMPILE_MSG;
  29 import static jdk.nashorn.internal.test.framework.TestConfig.OPTIONS_COMPARE;
  30 import static jdk.nashorn.internal.test.framework.TestConfig.OPTIONS_EXPECT_COMPILE_FAIL;
  31 import static jdk.nashorn.internal.test.framework.TestConfig.OPTIONS_EXPECT_RUN_FAIL;
  32 import static jdk.nashorn.internal.test.framework.TestConfig.OPTIONS_FORK;
  33 import static jdk.nashorn.internal.test.framework.TestConfig.OPTIONS_IGNORE_STD_ERROR;
  34 import static jdk.nashorn.internal.test.framework.TestConfig.OPTIONS_RUN;
  35 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_FAIL_LIST;
  36 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_SHARED_CONTEXT;
  37 import java.io.BufferedReader;
  38 import java.io.File;
  39 import java.io.IOException;
  40 import java.io.OutputStream;
  41 import java.util.ArrayList;
  42 import java.util.Arrays;
  43 import java.util.HashSet;
  44 import java.util.List;
  45 import java.util.Map;
  46 import java.util.Set;
  47 import java.util.regex.Matcher;
  48 
  49 /**
  50  * Abstract class to compile and run one .js script file.
  51  */
  52 @SuppressWarnings("javadoc")
  53 public abstract class AbstractScriptRunnable {
  54     // some test scripts need a "framework" script - whose features are used
  55     // in the test script. This optional framework script can be null.
  56 
  57     protected final String framework;
  58     // Script file that is being tested
  59     protected final File testFile;
  60     // build directory where test output, stderr etc are redirected
  61     protected final File buildDir;
  62     // should run the test or just compile?
  63     protected final boolean shouldRun;
  64     // is compiler error expected?
  65     protected final boolean expectCompileFailure;
  66     // is runtime error expected?
  67     protected final boolean expectRunFailure;
  68     // is compiler error captured and checked against known error strings?
  69     protected final boolean checkCompilerMsg;
  70     // .EXPECTED file compared for this or test?
  71     protected final boolean compare;
  72     // should test run in a separate process?
  73     protected final boolean fork;
  74     // ignore stderr output?
  75     protected final boolean ignoreStdError;
  76     // Foo.js.OUTPUT file where test stdout messages go
  77     protected final String outputFileName;
  78     // Foo.js.ERROR where test's stderr messages go.
  79     protected final String errorFileName;
  80     // copy of Foo.js.EXPECTED file
  81     protected final String copyExpectedFileName;
  82     // Foo.js.EXPECTED - output expected by running Foo.js
  83     protected final String expectedFileName;
  84     // options passed to Nashorn engine
  85     protected final List<String> engineOptions;
  86     // arguments passed to script - these are visible as "arguments" array to script
  87     protected final List<String> scriptArguments;
  88     // Tests that are forced to fail always
  89     protected final Set<String> failList = new HashSet<>();
  90 
  91     public AbstractScriptRunnable(final String framework, final File testFile, final List<String> engineOptions, final Map<String, String> testOptions, final List<String> scriptArguments) {
  92         this.framework = framework;
  93         this.testFile = testFile;
  94         this.buildDir = TestHelper.makeBuildDir(testFile);
  95         this.engineOptions = engineOptions;
  96         this.scriptArguments = scriptArguments;
  97 
  98         this.expectCompileFailure = testOptions.containsKey(OPTIONS_EXPECT_COMPILE_FAIL);
  99         this.shouldRun = testOptions.containsKey(OPTIONS_RUN);
 100         this.expectRunFailure = testOptions.containsKey(OPTIONS_EXPECT_RUN_FAIL);
 101         this.checkCompilerMsg = testOptions.containsKey(OPTIONS_CHECK_COMPILE_MSG);
 102         this.ignoreStdError = testOptions.containsKey(OPTIONS_IGNORE_STD_ERROR);
 103         this.compare = testOptions.containsKey(OPTIONS_COMPARE);
 104         this.fork = testOptions.containsKey(OPTIONS_FORK);
 105 
 106         final String testName = testFile.getName();
 107         this.outputFileName = buildDir + File.separator + testName + ".OUTPUT";
 108         this.errorFileName = buildDir + File.separator + testName + ".ERROR";
 109         this.copyExpectedFileName = buildDir + File.separator + testName + ".EXPECTED";
 110         this.expectedFileName = testFile.getPath() + ".EXPECTED";
 111 
 112         if (failListString != null) {
 113             final String[] failedTests = failListString.split(" ");
 114             for (final String failedTest : failedTests) {
 115                 failList.add(failedTest.trim());
 116             }
 117         }
 118     }
 119 
 120     // run this test - compile or compile-and-run depending on option passed
 121     public void runTest() throws IOException {
 122         log(toString());
 123         Thread.currentThread().setName(testFile.getPath());
 124         if (shouldRun) {
 125             // Analysis of failing tests list -
 126             // if test is in failing list it must fail
 127             // to not wrench passrate (used for crashing tests).
 128             if (failList.contains(testFile.getName())) {
 129                 fail(String.format("Test %s is forced to fail (see %s)", testFile, TEST_JS_FAIL_LIST));
 130             }
 131 
 132             execute();
 133         } else {
 134             compile();
 135         }
 136     }
 137 
 138     @Override
 139     public String toString() {
 140         return "Test(compile" + (expectCompileFailure ? "-" : "") + (shouldRun ? ", run" : "") + (expectRunFailure ? "-" : "") + "): " + testFile;
 141     }
 142 
 143     // compile-only command line arguments
 144     protected List<String> getCompilerArgs() {
 145         final List<String> args = new ArrayList<>();
 146         args.add("--compile-only");
 147         args.addAll(engineOptions);
 148         args.add(testFile.getPath());
 149         return args;
 150     }
 151 
 152     // shared context or not?
 153     protected static final boolean sharedContext = Boolean.getBoolean(TEST_JS_SHARED_CONTEXT);
 154     protected static final String failListString = System.getProperty(TEST_JS_FAIL_LIST);
 155     // VM options when a @fork test is executed by a separate process
 156     protected static final String[] forkJVMOptions;
 157     static {
 158         final String vmOptions = System.getProperty(TestConfig.TEST_FORK_JVM_OPTIONS);
 159         forkJVMOptions = (vmOptions != null)? vmOptions.split(" ") : new String[0];
 160     }
 161 
 162     private static final ThreadLocal<ScriptEvaluator> EVALUATORS = new ThreadLocal<>();
 163 
 164     /**
 165      * Create a script evaluator or return from cache
 166      * @return a ScriptEvaluator object
 167      */
 168     protected ScriptEvaluator getEvaluator() {
 169         synchronized (AbstractScriptRunnable.class) {
 170             ScriptEvaluator evaluator = EVALUATORS.get();
 171             if (evaluator == null) {
 172                 if (sharedContext) {
 173                     final String[] args;
 174                     if (framework.indexOf(' ') > 0) {
 175                         args = framework.split("\\s+");
 176                     } else {
 177                         args = new String[] { framework };
 178                     }
 179                     evaluator = new SharedContextEvaluator(args);
 180                     EVALUATORS.set(evaluator);
 181                 } else {
 182                     evaluator = new SeparateContextEvaluator();
 183                     EVALUATORS.set(evaluator);
 184                 }
 185             }
 186             return evaluator;
 187         }
 188     }
 189 
 190     /**
 191      * Evaluate one or more scripts with given output and error streams
 192      *
 193      * @param out OutputStream for script output
 194      * @param err OutputStream for script errors
 195      * @param args arguments for script evaluation
 196      * @return success or error code from script execution
 197      */
 198     protected int evaluateScript(final OutputStream out, final OutputStream err, final String[] args) {
 199         try {
 200             return getEvaluator().run(out, err, args);
 201         } catch (final IOException e) {
 202             throw new UnsupportedOperationException("I/O error in initializing shell - cannot redirect output to file", e);
 203         }
 204     }
 205 
 206     // arguments to be passed to compile-and-run this script
 207     protected List<String> getRuntimeArgs() {
 208         final ArrayList<String> args = new ArrayList<>();
 209         // add engine options first
 210         args.addAll(engineOptions);
 211 
 212         // framework script if any
 213         if (framework != null) {
 214             if (framework.indexOf(' ') > 0) {
 215                 args.addAll(Arrays.asList(framework.split("\\s+")));
 216             } else {
 217                 args.add(framework);
 218             }
 219         }
 220 
 221         // test script
 222         args.add(testFile.getPath());
 223 
 224         // script arguments
 225         if (!scriptArguments.isEmpty()) {
 226             args.add("--");
 227             args.addAll(scriptArguments);
 228         }
 229 
 230         return args;
 231     }
 232 
 233     // compares actual test output with .EXPECTED output
 234     protected void compare(final BufferedReader actual, final BufferedReader expected, final boolean compareCompilerMsg) throws IOException {
 235         int lineCount = 0;
 236         while (true) {
 237             final String es = expected.readLine();
 238             String as = actual.readLine();
 239             if (compareCompilerMsg) {
 240                 while (as != null && as.startsWith("--")) {
 241                     as = actual.readLine();
 242                 }
 243             }
 244             ++lineCount;
 245 
 246             if (es == null && as == null) {
 247                 if (expectRunFailure) {
 248                     fail("Expected runtime failure");
 249                 } else {
 250                     break;
 251                 }
 252             } else if (expectRunFailure && ((es == null) || as == null || !es.equals(as))) {
 253                 break;
 254             } else if (es == null) {
 255                 fail("Expected output for " + testFile + " ends prematurely at line " + lineCount);
 256             } else if (as == null) {
 257                 fail("Program output for " + testFile + " ends prematurely at line " + lineCount);
 258             } else if (es.equals(as)) {
 259                 continue;
 260             } else if (compareCompilerMsg && equalsCompilerMsgs(es, as)) {
 261                 continue;
 262             } else {
 263                 fail("Test " + testFile + " failed at line " + lineCount + " - " + " \n  expected: '" + escape(es) + "'\n     found: '" + escape(as) + "'");
 264             }
 265         }
 266     }
 267 
 268     // logs the message
 269     protected abstract void log(String msg);
 270     // throw failure message
 271     protected abstract void fail(String msg);
 272     // compile this script but don't run it
 273     protected abstract void compile() throws IOException;
 274     // compile and run this script
 275     protected abstract void execute();
 276 
 277     private static boolean equalsCompilerMsgs(final String es, final String as) {
 278         final int split = es.indexOf(':');
 279         // Replace both types of separators ('/' and '\') with the one from
 280         // current environment
 281         return (split >= 0) && as.equals(es.substring(0, split).replaceAll("[/\\\\]", Matcher.quoteReplacement(File.separator)) + es.substring(split));
 282     }
 283 
 284     private static void escape(final String value, final StringBuilder out) {
 285         final int len = value.length();
 286         for (int i = 0; i < len; i++) {
 287             final char ch = value.charAt(i);
 288             if (ch == '\n') {
 289                 out.append("\\n");
 290             } else if (ch < ' ' || ch == 127) {
 291                 out.append(String.format("\\%03o", (int) ch));
 292             } else if (ch > 127) {
 293                 out.append(String.format("\\u%04x", (int) ch));
 294             } else {
 295                 out.append(ch);
 296             }
 297         }
 298     }
 299 
 300     private static String escape(final String value) {
 301         final StringBuilder sb = new StringBuilder();
 302         escape(value, sb);
 303         return sb.toString();
 304     }
 305 }