1 /*
   2  * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 /**
  25  * @test
  26  * @summary Validate and test -?, -h and --help flags. All tools in the jdk
  27  *          should take the same flags to display the help message. These
  28  *          flags should be documented in the printed help message. The
  29  *          tool should quit without error code after displaying the
  30  *          help message (if there  is no other problem with the command
  31  *          line).
  32  *          Also check that tools that used to accept -help still do so,
  33  *          but do not document it. Test that tools that never accepted
  34  *          -help don't do so in future. I.e., check that the tool returns
  35  *          with the same return code as called with an invalid flag, and
  36  *          does not print anything containing '-help' in that case.
  37  * @compile HelpFlagsTest.java
  38  * @run main HelpFlagsTest
  39  */
  40 
  41 import java.io.File;
  42 import java.io.FileFilter;
  43 import java.util.Map;
  44 import java.util.ArrayList;
  45 import java.util.HashMap;
  46 import java.util.List;
  47 import java.util.HashSet;
  48 import java.util.Set;
  49 
  50 
  51 public class HelpFlagsTest extends TestHelper {
  52 
  53     // Tools that should not be tested because a usage message is pointless.
  54     static final String[] TOOLS_NOT_TO_TEST = {
  55         "appletviewer",     // deprecated, don't test
  56         "jaccessinspector", // gui, don't test, win only
  57         "jaccesswalker",    // gui, don't test, win only
  58         "jconsole",         // gui, don't test
  59         "servertool",       // none. Shell, don't test.
  60         "javaw",            // don't test, win only
  61         // The flags of these tools need to be fixed in Java EE.
  62         // The tools are deprecated for removal in Java SE. Don't test.
  63         "idlj",
  64         "orbd",
  65         "schemagen",
  66         "tnameserv",
  67         "wsgen",
  68         "wsimport",
  69         "xjc",
  70         // These shall have a help message that resembles that of
  71         // MIT's tools. Thus -?, -h and --help are supported, but not
  72         // mentioned in the help text.
  73         "kinit",
  74         "klist",
  75         "ktab"
  76     };
  77 
  78     // Lists which tools support which flags.
  79     private static class ToolHelpSpec {
  80         String toolname;
  81 
  82         // How the flags supposed to be supported are handled.
  83         //
  84         // These flags are supported, i.e.,
  85         // * the tool accepts the flag
  86         // * the tool prints a help message if the flag is specified
  87         // * this help message lists the flag
  88         // * the tool exits with exit code '0'.
  89         boolean supportsQuestionMark;
  90         boolean supportsH;
  91         boolean supportsHelp;
  92 
  93         // One tool returns with exit code != '0'.
  94         int exitcodeOfHelp;
  95 
  96         // How legacy -help is handled.
  97         //
  98         // Tools that so far support -help should still do so, but
  99         // not print documentation about it. Tools that do not
 100         // support -help should not do so in future.
 101         //
 102         // The tools accepts legacy -help. -help should not be
 103         // documented in the usage message.
 104         boolean supportsLegacyHelp;
 105 
 106         // Java itself documents -help. -help prints to stderr,
 107         // while --help prints to stdout. Leave as is.
 108         boolean documentsLegacyHelp;
 109 
 110         // The exit code of the tool if an invalid argument is passed to it.
 111         // An exit code != 0 would be expected, but not all tools handle it
 112         // that way.
 113         int exitcodeOfWrongFlag;
 114 
 115         // Some tools accept the invalid argument and thus hang the test.
 116         // Don't execute these with the wrong flags.
 117         // This actually should be considered a bug in the corresponding tool.
 118         boolean dontExecuteWithWrongFlags;
 119 
 120         ToolHelpSpec(String n, int q, int h, int hp, int ex1, int l, int dl, int ex2, int hangs) {
 121             toolname = n;
 122             supportsQuestionMark = ( q  == 1 ? true : false );
 123             supportsH            = ( h  == 1 ? true : false );
 124             supportsHelp         = ( hp == 1 ? true : false );
 125             exitcodeOfHelp       = ex1;
 126 
 127             supportsLegacyHelp   = (  l == 1 ? true : false );
 128             documentsLegacyHelp  = ( dl == 1 ? true : false );
 129             exitcodeOfWrongFlag  = ex2;
 130 
 131             dontExecuteWithWrongFlags = ( hangs == 1 ? true : false );
 132         }
 133     }
 134 
 135     static ToolHelpSpec[] jdkTools = {
 136         //               name          -?   -h --help exitcode   -help -help  exitcode  Don't
 137         //                                            of help          docu   of wrong  test
 138         //                                                             mented flag      inv flag
 139         new ToolHelpSpec("jabswitch",   0,   0,   0,   0,         0,    0,     0,       0),   // /?, prints help message anyways, win only
 140         new ToolHelpSpec("jaotc",       1,   1,   1,   0,         0,    0,     2,       0),   // -?, -h, --help
 141         new ToolHelpSpec("jar",         1,   1,   1,   0,         0,    0,     1,       0),   // -?, -h, --help
 142         new ToolHelpSpec("jarsigner",   1,   1,   1,   0,         1,    0,     1,       0),   // -?, -h, --help, -help accepted but not documented.
 143         new ToolHelpSpec("java",        1,   1,   1,   0,         1,    1,     1,       0),   // -?, -h, --help -help, Documents -help
 144         new ToolHelpSpec("javac",       1,   0,   1,   0,         1,    1,     2,       0),   // -?,     --help -help, Documents -help, -h is already taken for "native header output directory".
 145         new ToolHelpSpec("javadoc",     1,   1,   1,   0,         0,    0,     1,       0),   // -?, -h, --help
 146         new ToolHelpSpec("javah",       1,   1,   1,   0,         1,    0,     1,       0),   // -?, -h, --help, -help accepted but not documented.
 147         new ToolHelpSpec("javap",       1,   1,   1,   0,         1,    1,     2,       0),   // -?, -h, --help, -help accepted but not documented.
 148         new ToolHelpSpec("javaw",       1,   1,   1,   0,         1,    1,     1,       0),   // -?, -h, --help -help, win only
 149         new ToolHelpSpec("jcmd",        1,   1,   1,   0,         1,    0,     1,       0),   // -?, -h, --help, -help accepted but not documented.
 150         new ToolHelpSpec("jdb",         1,   1,   1,   0,         1,    1,     0,       0),   // -?, -h, --help, -help accepted but not documented.
 151         new ToolHelpSpec("jdeprscan",   1,   1,   1,   0,         0,    0,     1,       0),   // -?, -h, --help
 152         new ToolHelpSpec("jdeps",       1,   1,   1,   0,         1,    0,     2,       0),   // -?, -h, --help, -help accepted but not documented.
 153         new ToolHelpSpec("jhsdb",       0,   0,   0,   0,         0,    0,     0,       0),   // none, prints help message anyways.
 154         new ToolHelpSpec("jimage",      1,   1,   1,   0,         0,    0,     2,       0),   // -?, -h, --help
 155         new ToolHelpSpec("jinfo",       1,   1,   1,   0,         1,    1,     1,       0),   // -?, -h, --help, -help accepted but not documented.
 156         new ToolHelpSpec("jjs",         0,   1,   1, 100,         0,    0,   100,       0),   //     -h, --help, return code 100
 157         new ToolHelpSpec("jlink",       1,   1,   1,   0,         0,    0,     2,       0),   // -?, -h, --help
 158         new ToolHelpSpec("jmap",        1,   1,   1,   0,         1,    0,     1,       0),   // -?, -h, --help, -help accepted but not documented.
 159         new ToolHelpSpec("jmod",        1,   1,   1,   0,         1,    0,     2,       0),   // -?, -h, --help, -help accepted but not documented.
 160         new ToolHelpSpec("jps",         1,   1,   1,   0,         1,    1,     1,       0),   // -?, -h, --help, -help accepted but not documented.
 161         new ToolHelpSpec("jrunscript",  1,   1,   1,   0,         1,    1,     7,       0),   // -?, -h, --help, -help accepted but not documented.
 162         new ToolHelpSpec("jshell",      1,   1,   1,   0,         1,    0,     1,       0),   // -?, -h, --help, -help accepted but not documented.
 163         new ToolHelpSpec("jstack",      1,   1,   1,   0,         1,    1,     1,       0),   // -?, -h, --help, -help accepted but not documented.
 164         new ToolHelpSpec("jstat",       1,   1,   1,   0,         1,    1,     1,       0),   // -?, -h, --help, -help accepted but not documented.
 165         new ToolHelpSpec("jstatd",      1,   1,   1,   0,         0,    0,     1,       0),   // -?, -h, --help
 166         new ToolHelpSpec("keytool",     1,   1,   1,   0,         1,    0,     1,       0),   // none, prints help message anyways.
 167         new ToolHelpSpec("pack200",     1,   1,   1,   0,         1,    0,     2,       0),   // -?, -h, --help, -help accepted but not documented.
 168         new ToolHelpSpec("rmic",        0,   0,   0,   0,         0,    0,     1,       0),   // none, pirnts help message anyways.
 169         new ToolHelpSpec("rmid",        0,   0,   0,   0,         0,    0,     1,       0),   // none, prints help message anyways.
 170         new ToolHelpSpec("rmiregistry", 0,   0,   0,   0,         0,    0,     1,       0),   // none, prints help message anyways.
 171         new ToolHelpSpec("serialver",   0,   0,   0,   0,         0,    0,     1,       0),   // none, prints help message anyways.
 172         new ToolHelpSpec("unpack200",   1,   1,   1,   0,         1,    0,     2,       0),   // -?, -h, --help, -help accepted but not documented.
 173     };
 174 
 175     // Returns true if the file is not a tool.
 176     static boolean notATool(String file) {
 177         if (isWindows && !file.endsWith(EXE_FILE_EXT))
 178             return true;
 179         return false;
 180     }
 181 
 182     // Returns true if tool is listed in TOOLS_NOT_TO_TEST.
 183     static boolean dontTestTool(String tool) {
 184         tool = tool.toLowerCase();
 185         for (String x : TOOLS_NOT_TO_TEST) {
 186             if (tool.toLowerCase().startsWith(x))
 187                 return true;
 188         }
 189         return false;
 190     }
 191 
 192     // Returns corresponding object from jdkTools array.
 193     static ToolHelpSpec getToolHelpSpec(String tool) {
 194         for (ToolHelpSpec x : jdkTools) {
 195             if (tool.toLowerCase().equals(x.toolname) ||
 196                 tool.toLowerCase().equals(x.toolname + ".exe"))
 197                 return x;
 198         }
 199         return null;
 200     }
 201 
 202     // Check whether 'flag' appears in 'line' as a word of itself. It must not
 203     // be a substring of a word, as then similar flags might be matched.
 204     // E.g.: --help matches in the documentation of --help-extra.
 205     // This works only with english locale, as some tools have translated
 206     // usage messages.
 207     static boolean findFlagInLine(String line, String flag) {
 208         if (line.contains(flag) &&
 209             !line.contains("nknown") &&                       // Some tools say 'Unknown option "<flag>"',
 210             !line.contains("invalid flag") &&                 // 'invalid flag: <flag>'
 211             !line.contains("invalid option") &&               // or 'invalid option: <flag>'. Skip that.
 212             !line.contains("FileNotFoundException: -help") && // Special case for idlj.
 213             !line.contains("-h requires an argument") &&      // Special case for javac.
 214             !line.contains("port argument,")) {               // Special case for rmiregistry.
 215             // There might be several appearances of 'flag' in
 216             // 'line'. (-h as substring of --help).
 217             int flagLen = flag.length();
 218             int lineLen = line.length();
 219             for (int i = line.indexOf(flag); i >= 0; i = line.indexOf(flag, i+1)) {
 220                 // There should be a space before 'flag' in 'line', or it's right at the beginning.
 221                 if (i > 0 &&
 222                     line.charAt(i-1) != ' ' &&
 223                     line.charAt(i-1) != '[' &&  // jarsigner
 224                     line.charAt(i-1) != '|' &&  // jstatd
 225                     line.charAt(i-1) != '\t') { // jjs
 226                     continue;
 227                 }
 228                 // There should be a space or comma after 'flag' in 'line', or it's just at the end.
 229                 int posAfter = i + flagLen;
 230                 if (posAfter < lineLen &&
 231                     line.charAt(posAfter) != ' ' &&
 232                     line.charAt(posAfter) != ',' &&
 233                     line.charAt(posAfter) != '[' && // jar
 234                     line.charAt(posAfter) != ']' && // jarsigner
 235                     line.charAt(posAfter) != '|' && // jstatd
 236                     line.charAt(posAfter) != ':' && // jps
 237                     line.charAt(posAfter) != '"') { // keytool
 238                     continue;
 239                 }
 240                 return true;
 241             }
 242         }
 243         return false;
 244     }
 245 
 246     static TestResult runToolWithFlag(File f, String flag) {
 247         String x = f.getAbsolutePath();
 248         TestResult tr = doExec(x, flag);
 249         System.out.println("Testing " + f.getName());
 250         System.out.println("#> " + x + " " + flag);
 251         tr.testOutput.forEach(System.out::println);
 252         System.out.println("#> echo $?");
 253         System.out.println(tr.exitValue);
 254 
 255         return tr;
 256     }
 257 
 258     // Checks whether tool supports flag 'flag' and documents it
 259     // in the help message.
 260     static String testTool(File f, String flag, int exitcode) {
 261         String result = "";
 262         TestResult tr = runToolWithFlag(f, flag);
 263 
 264         // Check that the tool accepted the flag.
 265         if (exitcode == 0 && !tr.isOK()) {
 266             System.out.println("failed");
 267             result = "failed: " + f.getName() + " " + flag + " has exit code " + tr.exitValue + ".\n";
 268         }
 269 
 270         // Check there is a help message listing the flag.
 271         boolean foundFlag = false;
 272         for (String y : tr.testOutput) {
 273             if (!foundFlag && findFlagInLine(y, flag)) { // javac
 274                 foundFlag = true;
 275                 System.out.println("Found documentation of '" + flag + "': '" + y.trim() +"'");
 276             }
 277         }
 278         if (!foundFlag) {
 279             result += "failed: " + f.getName() + " does not document " +
 280                 flag + " in help message.\n";
 281         }
 282 
 283         if (!result.isEmpty()) 
 284             System.out.println(result);
 285 
 286         return result;
 287     }
 288 
 289     // Test the tool supports legacy option -help, but does
 290     // not document it.
 291     static String testLegacyFlag(File f, int exitcode) {
 292         String result = "";
 293         TestResult tr = runToolWithFlag(f, "-help");
 294 
 295         // Check that the tool accepted the flag.
 296         if (exitcode == 0 && !tr.isOK()) {
 297             System.out.println("failed");
 298             result = "failed: " + f.getName() + " -help has exit code " + tr.exitValue + ".\n";
 299         }
 300 
 301         // Check there is _no_ documentation of -help.
 302         boolean foundFlag = false;
 303         for (String y : tr.testOutput) {
 304             if (!foundFlag && findFlagInLine(y, "-help")) {  // javac
 305                 foundFlag = true;
 306                 System.out.println("Found documentation of '-help': '" + y.trim() +"'");
 307             }
 308         }
 309         if (foundFlag) {
 310             result += "failed: " + f.getName() + " does document -help " +
 311                 "in help message. This legacy flag should not be documented.\n";
 312         }
 313 
 314         if (!result.isEmpty())
 315             System.out.println(result);
 316 
 317         return result;
 318     }
 319 
 320     // Test that the tool exits with the exit code expected for
 321     // invalid flags. In general, one would expect this to be != 0,
 322     // but currently a row of tools exit with 0 in this case.
 323     // The output should not ask to get help with flag '-help'.
 324     static String testInvalidFlag(File f, String flag, int exitcode, boolean documentsLegacyHelp) {
 325         String result = "";
 326         TestResult tr = runToolWithFlag(f, flag);
 327 
 328         // Check that the tool did exit with the expected return code.
 329         if (!((exitcode == tr.exitValue) ||
 330               // Windows reports -1 where unix reports 255.
 331               (tr.exitValue < 0 && exitcode == tr.exitValue + 256))) {
 332             System.out.println("failed");
 333             result = "failed: " + f.getName() + " " + flag + " should not be " +
 334                      "accepted. But it has exit code " + tr.exitValue + ".\n";
 335         }
 336 
 337         if (!documentsLegacyHelp) {        
 338             // Check there is _no_ documentation of -help.
 339             boolean foundFlag = false;
 340             for (String y : tr.testOutput) {
 341                 if (!foundFlag && findFlagInLine(y, "-help")) {  // javac
 342                     foundFlag = true;
 343                     System.out.println("Found documentation of '-help': '" + y.trim() +"'");
 344                 }
 345             }
 346             if (foundFlag) {
 347                 result += "failed: " + f.getName() + " does document -help " +
 348                     "in error message. This legacy flag should not be documented.\n";
 349             }
 350         }
 351 
 352         if (!result.isEmpty())
 353             System.out.println(result);
 354 
 355         return result;
 356     }
 357 
 358     public static void main(String[] args) {
 359         String errorMessage = "";
 360 
 361         // The test analyses the help messages printed. It assumes englisch
 362         // help messages. Thus it only works with english locale.
 363         if (!isEnglishLocale()) { return; }
 364 
 365         for (File f : new File(JAVA_BIN).listFiles()) {
 366             String toolName = f.getName();
 367 
 368             if (notATool(toolName)) {
 369                 continue;
 370             }
 371             if (dontTestTool(toolName)) {
 372                 System.out.println("Skipping test of tool " + toolName +
 373                                    ". Tool has no help message.");
 374                 continue;
 375             }
 376 
 377             ToolHelpSpec tool = getToolHelpSpec(toolName);
 378             if (tool == null) {
 379                 errorMessage += "Tool " + toolName + " not covered by this test. " +
 380                     "Add specification to jdkTools array!\n";
 381                 continue;
 382             }
 383 
 384             // Test for help flags to be supported.
 385             if (tool.supportsQuestionMark == true) {
 386                 errorMessage += testTool(f, "-?", tool.exitcodeOfHelp);
 387             } else {
 388                 System.out.println("Skip " + tool.toolname + ". It does not support -?.");
 389             }
 390             if (tool.supportsH == true) {
 391                 errorMessage += testTool(f, "-h", tool.exitcodeOfHelp);
 392             } else {
 393                 System.out.println("Skip " + tool.toolname + ". It does not support -h.");
 394             }
 395             if (tool.supportsHelp == true) {
 396                 errorMessage += testTool(f, "--help", tool.exitcodeOfHelp);
 397             } else {
 398                 System.out.println("Skip " + tool.toolname + ". It does not support --help.");
 399             }
 400 
 401             // Check that the return code listing in jdkTools[] is
 402             // correct for an invalid flag.
 403             if (!tool.dontExecuteWithWrongFlags) {
 404                 errorMessage += testInvalidFlag(f, "-asdfxgr", tool.exitcodeOfWrongFlag, tool.documentsLegacyHelp);
 405             }
 406 
 407             // Test for legacy -help flag.
 408             if (!tool.documentsLegacyHelp) {
 409                 if (tool.supportsLegacyHelp == true) {
 410                     errorMessage += testLegacyFlag(f, tool.exitcodeOfHelp);
 411                 } else {
 412                     if (!tool.dontExecuteWithWrongFlags) {
 413                         errorMessage += testInvalidFlag(f, "-help", tool.exitcodeOfWrongFlag, false);
 414                     }
 415                 }
 416             }
 417         }
 418 
 419         if (errorMessage.isEmpty()) {
 420             System.out.println("All help string tests: PASS");
 421         } else {
 422             throw new AssertionError("HelpFlagsTest failed:\n" + errorMessage);
 423         }
 424     }
 425 }