1 /*
   2  * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2018 SAP SE. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  */
  24 
  25 /**
  26  * @test
  27  * @summary Validate and test -?, -h and --help flags. All tools in the jdk
  28  *          should take the same flags to display the help message. These
  29  *          flags should be documented in the printed help message. The
  30  *          tool should quit without error code after displaying the
  31  *          help message (if there  is no other problem with the command
  32  *          line).
  33  *          Also check that tools that used to accept -help still do
  34  *          so. Test that tools that never accepted -help don't do so
  35  *          in future. I.e., check that the tool returns with the same
  36  *          return code as called with an invalid flag, and does not
  37  *          print anything containing '-help' in that case.
  38  * @compile HelpFlagsTest.java
  39  * @run main HelpFlagsTest
  40  */
  41 
  42 import java.io.File;
  43 import java.io.FileFilter;
  44 import java.util.Map;
  45 import java.util.ArrayList;
  46 import java.util.HashMap;
  47 import java.util.List;
  48 import java.util.HashSet;
  49 import java.util.Set;
  50 
  51 
  52 public class HelpFlagsTest extends TestHelper {
  53 
  54     // Tools that should not be tested because a usage message is pointless.
  55     static final String[] TOOLS_NOT_TO_TEST = {
  56         "appletviewer",     // deprecated, don't test
  57         "jaccessinspector", // gui, don't test, win only
  58         "jaccesswalker",    // gui, don't test, win only
  59         "jconsole",         // gui, don't test
  60         "servertool",       // none. Shell, don't test.
  61         "javaw",            // don't test, win only
  62         // The flags of these tools need to be fixed in Java EE.
  63         // The tools are deprecated for removal in Java SE. Don't test.
  64         "idlj",
  65         "orbd",
  66         "schemagen",
  67         "tnameserv",
  68         "wsgen",
  69         "wsimport",
  70         "xjc",
  71         // These shall have a help message that resembles that of
  72         // MIT's tools. Thus -?, -h and --help are supported, but not
  73         // mentioned in the help text.
  74         "kinit",
  75         "klist",
  76         "ktab",
  77         // Oracle proprietary tools without help message.
  78         "jmc",
  79         "jweblauncher",
  80         "jcontrol",
  81         "ssvagent"
  82     };
  83 
  84     // Lists which tools support which flags.
  85     private static class ToolHelpSpec {
  86         String toolname;
  87 
  88         // How the flags supposed to be supported are handled.
  89         //
  90         // These flags are supported, i.e.,
  91         // * the tool accepts the flag
  92         // * the tool prints a help message if the flag is specified
  93         // * this help message lists the flag
  94         // * the tool exits with exit code '0'.
  95         boolean supportsQuestionMark;
  96         boolean supportsH;
  97         boolean supportsHelp;
  98 
  99         // One tool returns with exit code != '0'.
 100         int exitcodeOfHelp;
 101 
 102         // How legacy -help is handled.
 103         //
 104         // Tools that so far support -help should still do so, but
 105         // not print documentation about it. Tools that do not
 106         // support -help should not do so in future.
 107         //
 108         // The tools accepts legacy -help. -help should not be
 109         // documented in the usage message.
 110         boolean supportsLegacyHelp;
 111 
 112         // Java itself documents -help. -help prints to stderr,
 113         // while --help prints to stdout. Leave as is.
 114         boolean documentsLegacyHelp;
 115 
 116         // The exit code of the tool if an invalid argument is passed to it.
 117         // An exit code != 0 would be expected, but not all tools handle it
 118         // that way.
 119         int exitcodeOfWrongFlag;
 120 
 121         ToolHelpSpec(String n, int q, int h, int hp, int ex1, int l, int dl, int ex2) {
 122             toolname = n;
 123             supportsQuestionMark = ( q  == 1 ? true : false );
 124             supportsH            = ( h  == 1 ? true : false );
 125             supportsHelp         = ( hp == 1 ? true : false );
 126             exitcodeOfHelp       = ex1;
 127 
 128             supportsLegacyHelp   = (  l == 1 ? true : false );
 129             documentsLegacyHelp  = ( dl == 1 ? true : false );
 130             exitcodeOfWrongFlag  = ex2;
 131         }
 132     }
 133 
 134     static ToolHelpSpec[] jdkTools = {
 135         //               name          -?   -h --help exitcode   -help -help  exitcode
 136         //                                            of help          docu   of wrong
 137         //                                                             mented flag
 138         new ToolHelpSpec("jabswitch",   0,   0,   0,   0,         0,    0,     0),     // /?, prints help message anyways, win only
 139         new ToolHelpSpec("jaotc",       1,   1,   1,   0,         0,    0,     2),     // -?, -h, --help
 140         new ToolHelpSpec("jar",         1,   1,   1,   0,         0,    0,     1),     // -?, -h, --help
 141         new ToolHelpSpec("jarsigner",   1,   1,   1,   0,         1,    0,     1),     // -?, -h, --help, -help accepted but not documented.
 142         new ToolHelpSpec("java",        1,   1,   1,   0,         1,    1,     1),     // -?, -h, --help -help, Documents -help
 143         new ToolHelpSpec("javac",       1,   0,   1,   0,         1,    1,     2),     // -?,     --help -help, Documents -help, -h is already taken for "native header output directory".
 144         new ToolHelpSpec("javadoc",     1,   1,   1,   0,         1,    1,     1),     // -?, -h, --help -help, Documents -help
 145         new ToolHelpSpec("javap",       1,   1,   1,   0,         1,    1,     2),     // -?, -h, --help -help, Documents -help
 146         new ToolHelpSpec("javaw",       1,   1,   1,   0,         1,    1,     1),     // -?, -h, --help -help, Documents -help, win only
 147         new ToolHelpSpec("jcmd",        1,   1,   1,   0,         1,    0,     1),     // -?, -h, --help, -help accepted but not documented.
 148         new ToolHelpSpec("jdb",         1,   1,   1,   0,         1,    1,     0),     // -?, -h, --help -help, Documents -help
 149         new ToolHelpSpec("jdeprscan",   1,   1,   1,   0,         0,    0,     1),     // -?, -h, --help
 150         new ToolHelpSpec("jdeps",       1,   1,   1,   0,         1,    0,     2),     // -?, -h, --help, -help accepted but not documented.
 151         new ToolHelpSpec("jhsdb",       0,   0,   0,   0,         0,    0,     0),     // none, prints help message anyways.
 152         new ToolHelpSpec("jimage",      1,   1,   1,   0,         0,    0,     2),     // -?, -h, --help
 153         new ToolHelpSpec("jinfo",       1,   1,   1,   0,         1,    1,     1),     // -?, -h, --help -help, Documents -help
 154         new ToolHelpSpec("jjs",         0,   1,   1, 100,         0,    0,   100),     //     -h, --help, return code 100
 155         new ToolHelpSpec("jlink",       1,   1,   1,   0,         0,    0,     2),     // -?, -h, --help
 156         new ToolHelpSpec("jmap",        1,   1,   1,   0,         1,    0,     1),     // -?, -h, --help, -help accepted but not documented.
 157         new ToolHelpSpec("jmod",        1,   1,   1,   0,         1,    0,     2),     // -?, -h, --help, -help accepted but not documented.
 158         new ToolHelpSpec("jps",         1,   1,   1,   0,         1,    1,     1),     // -?, -h, --help -help, Documents -help
 159         new ToolHelpSpec("jrunscript",  1,   1,   1,   0,         1,    1,     7),     // -?, -h, --help -help, Documents -help
 160         new ToolHelpSpec("jshell",      1,   1,   1,   0,         1,    0,     1),     // -?, -h, --help, -help accepted but not documented.
 161         new ToolHelpSpec("jstack",      1,   1,   1,   0,         1,    1,     1),     // -?, -h, --help -help, Documents -help
 162         new ToolHelpSpec("jstat",       1,   1,   1,   0,         1,    1,     1),     // -?, -h, --help -help, Documents -help
 163         new ToolHelpSpec("jstatd",      1,   1,   1,   0,         0,    0,     1),     // -?, -h, --help
 164         new ToolHelpSpec("keytool",     1,   1,   1,   0,         1,    0,     1),     // none, prints help message anyways.
 165         new ToolHelpSpec("pack200",     1,   1,   1,   0,         1,    0,     2),     // -?, -h, --help, -help accepted but not documented.
 166         new ToolHelpSpec("rmic",        0,   0,   0,   0,         0,    0,     1),     // none, pirnts help message anyways.
 167         new ToolHelpSpec("rmid",        0,   0,   0,   0,         0,    0,     1),     // none, prints help message anyways.
 168         new ToolHelpSpec("rmiregistry", 0,   0,   0,   0,         0,    0,     1),     // none, prints help message anyways.
 169         new ToolHelpSpec("serialver",   0,   0,   0,   0,         0,    0,     1),     // none, prints help message anyways.
 170         new ToolHelpSpec("unpack200",   1,   1,   1,   0,         1,    0,     2),     // -?, -h, --help, -help accepted but not documented.
 171         // Oracle proprietary tools:
 172         new ToolHelpSpec("javapackager",0,   0,   0,   0,         1,    0,   255),     // -help accepted but not documented.
 173     };
 174 
 175     // Returns true if the file is not a tool.
 176     static boolean notATool(String file) {
 177         if (isWindows && !file.endsWith(EXE_FILE_EXT))
 178             return true;
 179         return false;
 180     }
 181 
 182     // Returns true if tool is listed in TOOLS_NOT_TO_TEST.
 183     static boolean dontTestTool(String tool) {
 184         tool = tool.toLowerCase();
 185         for (String x : TOOLS_NOT_TO_TEST) {
 186             if (tool.toLowerCase().startsWith(x))
 187                 return true;
 188         }
 189         return false;
 190     }
 191 
 192     // Returns corresponding object from jdkTools array.
 193     static ToolHelpSpec getToolHelpSpec(String tool) {
 194         for (ToolHelpSpec x : jdkTools) {
 195             if (tool.toLowerCase().equals(x.toolname) ||
 196                 tool.toLowerCase().equals(x.toolname + ".exe"))
 197                 return x;
 198         }
 199         return null;
 200     }
 201 
 202     // Check whether 'flag' appears in 'line' as a word of itself. It must not
 203     // be a substring of a word, as then similar flags might be matched.
 204     // E.g.: --help matches in the documentation of --help-extra.
 205     // This works only with english locale, as some tools have translated
 206     // usage messages.
 207     static boolean findFlagInLine(String line, String flag) {
 208         if (line.contains(flag) &&
 209             !line.contains("nknown") &&                       // Some tools say 'Unknown option "<flag>"',
 210             !line.contains("invalid flag") &&                 // 'invalid flag: <flag>'
 211             !line.contains("invalid option") &&               // or 'invalid option: <flag>'. Skip that.
 212             !line.contains("FileNotFoundException: -help") && // Special case for idlj.
 213             !line.contains("-h requires an argument") &&      // Special case for javac.
 214             !line.contains("port argument,")) {               // Special case for rmiregistry.
 215             // There might be several appearances of 'flag' in
 216             // 'line'. (-h as substring of --help).
 217             int flagLen = flag.length();
 218             int lineLen = line.length();
 219             for (int i = line.indexOf(flag); i >= 0; i = line.indexOf(flag, i+1)) {
 220                 // There should be a space before 'flag' in 'line', or it's right at the beginning.
 221                 if (i > 0 &&
 222                     line.charAt(i-1) != ' ' &&
 223                     line.charAt(i-1) != '[' &&  // jarsigner
 224                     line.charAt(i-1) != '|' &&  // jstatd
 225                     line.charAt(i-1) != '\t') { // jjs
 226                     continue;
 227                 }
 228                 // There should be a space or comma after 'flag' in 'line', or it's just at the end.
 229                 int posAfter = i + flagLen;
 230                 if (posAfter < lineLen &&
 231                     line.charAt(posAfter) != ' ' &&
 232                     line.charAt(posAfter) != ',' &&
 233                     line.charAt(posAfter) != '[' && // jar
 234                     line.charAt(posAfter) != ']' && // jarsigner
 235                     line.charAt(posAfter) != '|' && // jstatd
 236                     line.charAt(posAfter) != ':' && // jps
 237                     line.charAt(posAfter) != '"') { // keytool
 238                     continue;
 239                 }
 240                 return true;
 241             }
 242         }
 243         return false;
 244     }
 245 
 246     static TestResult runToolWithFlag(File f, String flag) {
 247         String x = f.getAbsolutePath();
 248         TestResult tr = doExec(x, flag);
 249         System.out.println("Testing " + f.getName());
 250         System.out.println("#> " + x + " " + flag);
 251         tr.testOutput.forEach(System.out::println);
 252         System.out.println("#> echo $?");
 253         System.out.println(tr.exitValue);
 254 
 255         return tr;
 256     }
 257 
 258     // Checks whether tool supports flag 'flag' and documents it
 259     // in the help message.
 260     static String testTool(File f, String flag, int exitcode) {
 261         String result = "";
 262         TestResult tr = runToolWithFlag(f, flag);
 263 
 264         // Check that the tool accepted the flag.
 265         if (exitcode == 0 && !tr.isOK()) {
 266             System.out.println("failed");
 267             result = "failed: " + f.getName() + " " + flag + " has exit code " + tr.exitValue + ".\n";
 268         }
 269 
 270         // Check there is a help message listing the flag.
 271         boolean foundFlag = false;
 272         for (String y : tr.testOutput) {
 273             if (!foundFlag && findFlagInLine(y, flag)) { // javac
 274                 foundFlag = true;
 275                 System.out.println("Found documentation of '" + flag + "': '" + y.trim() +"'");
 276             }
 277         }
 278         if (!foundFlag) {
 279             result += "failed: " + f.getName() + " does not document " +
 280                 flag + " in help message.\n";
 281         }
 282 
 283         if (!result.isEmpty())
 284             System.out.println(result);
 285 
 286         return result;
 287     }
 288 
 289     // Test the tool supports legacy option -help, but does
 290     // not document it.
 291     static String testLegacyFlag(File f, int exitcode) {
 292         String result = "";
 293         TestResult tr = runToolWithFlag(f, "-help");
 294 
 295         // Check that the tool accepted the flag.
 296         if (exitcode == 0 && !tr.isOK()) {
 297             System.out.println("failed");
 298             result = "failed: " + f.getName() + " -help has exit code " + tr.exitValue + ".\n";
 299         }
 300 
 301         // Check there is _no_ documentation of -help.
 302         boolean foundFlag = false;
 303         for (String y : tr.testOutput) {
 304             if (!foundFlag && findFlagInLine(y, "-help")) {  // javac
 305                 foundFlag = true;
 306                 System.out.println("Found documentation of '-help': '" + y.trim() +"'");
 307             }
 308         }
 309         if (foundFlag) {
 310             result += "failed: " + f.getName() + " does document -help " +
 311                 "in help message. This legacy flag should not be documented.\n";
 312         }
 313 
 314         if (!result.isEmpty())
 315             System.out.println(result);
 316 
 317         return result;
 318     }
 319 
 320     // Test that the tool exits with the exit code expected for
 321     // invalid flags. In general, one would expect this to be != 0,
 322     // but currently a row of tools exit with 0 in this case.
 323     // The output should not ask to get help with flag '-help'.
 324     static String testInvalidFlag(File f, String flag, int exitcode, boolean documentsLegacyHelp) {
 325         String result = "";
 326         TestResult tr = runToolWithFlag(f, flag);
 327 
 328         // Check that the tool did exit with the expected return code.
 329         if (!((exitcode == tr.exitValue) ||
 330               // Windows reports -1 where unix reports 255.
 331               (tr.exitValue < 0 && exitcode == tr.exitValue + 256))) {
 332             System.out.println("failed");
 333             result = "failed: " + f.getName() + " " + flag + " should not be " +
 334                      "accepted. But it has exit code " + tr.exitValue + ".\n";
 335         }
 336 
 337         if (!documentsLegacyHelp) {
 338             // Check there is _no_ documentation of -help.
 339             boolean foundFlag = false;
 340             for (String y : tr.testOutput) {
 341                 if (!foundFlag && findFlagInLine(y, "-help")) {  // javac
 342                     foundFlag = true;
 343                     System.out.println("Found documentation of '-help': '" + y.trim() +"'");
 344                 }
 345             }
 346             if (foundFlag) {
 347                 result += "failed: " + f.getName() + " does document -help " +
 348                     "in error message. This legacy flag should not be documented.\n";
 349             }
 350         }
 351 
 352         if (!result.isEmpty())
 353             System.out.println(result);
 354 
 355         return result;
 356     }
 357 
 358     public static void main(String[] args) {
 359         String errorMessage = "";
 360 
 361         // The test analyses the help messages printed. It assumes englisch
 362         // help messages. Thus it only works with english locale.
 363         if (!isEnglishLocale()) { return; }
 364 
 365         for (File f : new File(JAVA_BIN).listFiles()) {
 366             String toolName = f.getName();
 367 
 368             if (notATool(toolName)) {
 369                 continue;
 370             }
 371             if (dontTestTool(toolName)) {
 372                 System.out.println("Skipping test of tool " + toolName +
 373                                    ". Tool has no help message.");
 374                 continue;
 375             }
 376 
 377             ToolHelpSpec tool = getToolHelpSpec(toolName);
 378             if (tool == null) {
 379                 errorMessage += "Tool " + toolName + " not covered by this test. " +
 380                     "Add specification to jdkTools array!\n";
 381                 continue;
 382             }
 383 
 384             // Test for help flags to be supported.
 385             if (tool.supportsQuestionMark == true) {
 386                 errorMessage += testTool(f, "-?", tool.exitcodeOfHelp);
 387             } else {
 388                 System.out.println("Skip " + tool.toolname + ". It does not support -?.");
 389             }
 390             if (tool.supportsH == true) {
 391                 errorMessage += testTool(f, "-h", tool.exitcodeOfHelp);
 392             } else {
 393                 System.out.println("Skip " + tool.toolname + ". It does not support -h.");
 394             }
 395             if (tool.supportsHelp == true) {
 396                 errorMessage += testTool(f, "--help", tool.exitcodeOfHelp);
 397             } else {
 398                 System.out.println("Skip " + tool.toolname + ". It does not support --help.");
 399             }
 400 
 401             // Check that the return code listing in jdkTools[] is
 402             // correct for an invalid flag.
 403             errorMessage += testInvalidFlag(f, "-asdfxgr", tool.exitcodeOfWrongFlag, tool.documentsLegacyHelp);
 404 
 405             // Test for legacy -help flag.
 406             if (!tool.documentsLegacyHelp) {
 407                 if (tool.supportsLegacyHelp == true) {
 408                     errorMessage += testLegacyFlag(f, tool.exitcodeOfHelp);
 409                 } else {
 410                     errorMessage += testInvalidFlag(f, "-help", tool.exitcodeOfWrongFlag, false);
 411                 }
 412             }
 413         }
 414 
 415         if (errorMessage.isEmpty()) {
 416             System.out.println("All help string tests: PASS");
 417         } else {
 418             throw new AssertionError("HelpFlagsTest failed:\n" + errorMessage);
 419         }
 420     }
 421 }