1 /*
   2  * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
   3  * ORACLE PROPRIETARY/CONFIDENTIAL. Use is subject to license terms.
   4  *
   5  *
   6  *
   7  *
   8  *
   9  *
  10  *
  11  *
  12  *
  13  *
  14  *
  15  *
  16  *
  17  *
  18  *
  19  *
  20  *
  21  *
  22  */
  23 
  24 /**
  25  * @test
  26  * @bug 6545058 6611182 8016209 8139986 8162746
  27  * @summary Validate and test -?, -h and --help flags. All tools should take
  28  *          the same flags to display the help message.
  29  * @compile HelpCheck.java
  30  * @run main HelpCheck
  31  */
  32 
  33 import java.io.File;
  34 import java.io.FileFilter;
  35 import java.util.Map;
  36 import java.util.ArrayList;
  37 import java.util.HashMap;
  38 import java.util.List;
  39 import java.util.HashSet;
  40 import java.util.Set;
  41 
  42 public class HelpCheck extends TestHelper {
  43 
  44     // Tools that do not accept -?.
  45     static final String[] BLACKLIST_QOPTION = {
  46         "appletviewer",     // deprecated, don't test
  47         "idlj",             // none, prints help message anyways.
  48         "jabswitch",        // /?, prints help message anyways, win only
  49         "jaccessinspector", // gui, don't test, win only
  50         "jaccesswalker",    // gui, don't test, win only
  51         "jaotc",            // --help
  52         //"jar",            // -?, -h, --help
  53         //"jarsigner",      // -?, -h, --help, -help accepted but not documented.
  54         //"java",           // -?, -h, --help
  55         //"javac",          // -?,     --help, -h is already taken for "native header output directory".
  56         //"javadoc",        // -?, -h, --help
  57         //"javah",          // -?, -h, --help
  58         //"javap",          // -?, -h, --help, -help accepted but not documented.
  59         //"javaw",          // -?, -h, --help, win only
  60         //"jcmd",           // -?, -h, --help, -help accepted but not documented.
  61         "jconsole",         // gui, don't test
  62         //"jdb",            // -?, -h, --help, -help accepted but not documented.
  63         //"jdeprscan",      // -?, -h, --help
  64         "jdeps",            // none, prints help message anyways.
  65         "jhsdb",            // none, prints help message anyways.    
  66         "jimage",           // -h, --help
  67         //"jinfo",          // -?, -h, --help, -help accepted but not documented.
  68         "jjs",              // -h, --help, return code 100
  69         "jlink",            // -h, --help
  70         "jmap",             // none, prints help message anyways.
  71         "jmod",             // -h, --help
  72         //"jps",            // -?, -h, --help, -help accepted but not documented.
  73         //"jrunscript",     // -?, -help, Displays help message if no arguments are given or if -h, --help.
  74         //"jshell",         // -?, -h, --help
  75         "jstack",           // -h, prints help message anyways.
  76         //"jstat",          // -?, -h, --help, -help accepted but not documented.
  77         "jstatd",           // All work, none documented, return code 1.
  78         "keytool",          // none, prints help message anyways.
  79         "kinit",            // none, interactive, asks for password, don't test, win only
  80         "klist",            // none, prints help message anyways, win only
  81         "ktab",             // none, prints help message anyways, win only
  82         "orbd",             // Throws exceptions on all three flags
  83         //"pack200",        // -? -h --help, return code '1'
  84         "policytool",       // none, prints help message anyways.
  85         "rmic",             // none, prints help message anyways.
  86         "rmid",             // none, prints help message anyways.
  87         "rmiregistry",      // none, prints help message anyways.
  88         "schemagen",        // -help, prints help message anyways.
  89         "serialver",        // none, prints help message anyways.
  90         "servertool",       // none. Shell, don't test. 
  91         "tnameserv",        // Throws exceptions on all three flags
  92         //"unpack200",      // -? -h --help, return code '1'
  93         "wsgen",            // -help, prints help message anyways.
  94         "wsimport",         // -help, exception on other flags
  95         "xjc",              // -help, prints help message anyways.
  96     };
  97 
  98     // Tools that do not accept -h.
  99     static final String[] BLACKLIST_HOPTION = {
 100         "appletviewer",     // deprecated, don't test
 101         "idlj",             // none, prints help message anyways.
 102         "jabswitch",        // win: /?, prints help message anyways.
 103         "jaccessinspector", // win: gui, don't test
 104         "jaccesswalker",    // win: gui, don't test
 105         "jaotc",            // --help
 106         //"jar",            // -h, --help
 107         //"jarsigner",      // -h, --help, not in help message -- fixed
 108         //"java",           // -?, -h, --help
 109         "javac",            // --help, -h is already taken for "native header output directory". -- fixed -?
 110         "javadoc",          // --help
 111         //"javah",          // -?, -h, --help
 112         //"javap",          // -?, --help -- fixed
 113         //"javaw",          // win: -?, -h, --help
 114         //"jcmd",           // -h, but exits with return code
 115         "jconsole",         // gui, don't test
 116         //"jdb",            // -help -- fixed: all
 117         //"jdeprscan",      // -h, --help, return code 1
 118         //"jdeps",          // none, prints help message anyways.
 119         "jhsdb",            // Strange ...
 120         //"jimage",         // -h, --help
 121         //"jinfo",          // Displays help message if no arguments are given or wrong ones. Documents -h. -- fixed.
 122         //"jjs",            // -h, return code 100
 123         //"jlink",          // -h, --help
 124         "jmap",             // none, prints help message anyways.
 125         //"jmod",           // -h, --help
 126         //"jps",            // -?, not documented -- fixed
 127         "jrunscript",       // -?, -help, Displays help message if no arguments are given or if -h, --help.
 128         //"jshell",         // -h, --help, -h not documented. -- fixed
 129         //"jstack",         // -h, prints help message anyways.
 130         //"jstat",          // All work, -help documented -- fixed
 131         "jstatd",           // All work, none documented, return code 1.
 132         "keytool",          // none, prints help message anyways.
 133         "kinit",            // win: none, interactive, asks for password, don't test
 134         "klist",            // win: none, prints help message anyways.
 135         "ktab",             // win: none, prints help message anyways.
 136         "orbd",             // Throws exceptions on all three flags
 137         //"pack200",        // -? -h --help documented, but return code '1'
 138         "policytool",       // none, prints help message anyways.
 139         "rmic",             // none, prints help message anyways.
 140         "rmid",             // none, prints help message anyways.
 141         "rmiregistry",      // none, prints help message anyways.
 142         "schemagen",        // -help, prints help message anyways.
 143         "serialver",        // none, prints help message anyways.
 144         "servertool",       // none. Shell, don't test.
 145         "tnameserv",        // Throws exceptions on all three flags
 146         //"unpack200",      // -? -h --help documented, but return code '1'
 147         "wsgen",            // -help, prints help message anyways.
 148         "wsimport",         // -help, exception on other flags
 149         "xjc",              // -help, prints help message anyways.
 150     };
 151 
 152     // Tools that do not accept --help.
 153     static final String[] BLACKLIST_HELPOPTION = {
 154         "appletviewer",     // deprecated, don't test
 155         "idlj",             // none, prints help message anyways.
 156         "jabswitch",        // win: /?, prints help message anyways.
 157         "jaccessinspector", // win: gui, don't test
 158         "jaccesswalker",    // win: gui, don't test
 159         //"jaotc",          // --help
 160         //"jar",            // -h, --help
 161         //"jarsigner",      // -h, --help, not in help message -- fixed
 162         //"java",           // -?, -h, --help
 163         //"javac",          // --help, -h is already taken for "native header output directory". -- fixed -?
 164         //"javadoc",        // --help
 165         //"javah",          // -?, -h, --help
 166         //"javap",          // -?, --help -- fixed
 167         //"javaw",          // win: -?, -h, --help
 168         //"jcmd",           // -h, but exits with return code -- fixed
 169         "jconsole",         // gui, don't test
 170         //"jdb",            // -help -- fixed: all
 171         //"jdeprscan",      // -h, --help, return code 1
 172         "jdeps",            // none, prints help message anyways.
 173         "jhsdb",            // Strange ...
 174         //"jimage",         // -h, --help
 175         //"jinfo",          // Displays help message if no arguments are given or wrong ones. Documents -h. -- fixed.
 176         //"jjs",            // -h, return code 100 -- fixed --help
 177         //"jlink",          // -h, --help
 178         "jmap",             // none, prints help message anyways.
 179         //"jmod",           // -h, --help
 180         "jps",              // -?, not documented -- fixed
 181         "jrunscript",       // -?, -help, Displays help message if no arguments are given or if -h, --help.
 182         //"jshell",         // -h, --help, -h not documented. -- fixed
 183         "jstack",           // -h, prints help message anyways.
 184         //"jstat",          // All work, -help documented -- fixed
 185         "jstatd",           // All work, none documented, return code 1.
 186         "keytool",          // none, prints help message anyways.
 187         "kinit",            // win: none, interactive, asks for password, don't test
 188         "klist",            // win: none, prints help message anyways.
 189         "ktab",             // win: none, prints help message anyways.
 190         "orbd",             // Throws exceptions on all three flags
 191         //"pack200",        // -? -h --help documented, but return code '1'
 192         "policytool",       // none, prints help message anyways.
 193         "rmic",             // none, prints help message anyways.
 194         "rmid",             // none, prints help message anyways.
 195         "rmiregistry",      // none, prints help message anyways.
 196         "schemagen",        // -help, prints help message anyways.
 197         "serialver",        // none, prints help message anyways.
 198         "servertool",       // none. Shell, don't test.
 199         "tnameserv",        // Throws exceptions on all three flags
 200         //"unpack200",      // -? -h --help documented, but return code '1'
 201         "wsgen",            // -help, prints help message anyways.
 202         "wsimport",         // -help, exception on other flags
 203         "xjc",              // -help, prints help message anyways.
 204     };
 205 
 206     // Tools that exit with a return code != 0 after accepting the 
 207     // flag and printing the help message.
 208     static final String[] ERROR_CODE_ACCEPTABLE = {
 209         "appletviewer",       // deprecated, don't test
 210         //"idlj",             // none, prints help message anyways.
 211         //"jabswitch",        // win: /?, prints help message anyways.
 212         //"jaccessinspector", // win: gui, don't test
 213         //"jaccesswalker",    // win: gui, don't test
 214         //"jaotc",            // --help
 215         //"jar",              // -h, --help
 216         //"jarsigner",        // -h, --help, not in help message -- fixed
 217         //"java",             // -?, -h, --help
 218         //"javac",            // --help
 219         //"javadoc",          // --help
 220         //"javah",            // -?, -h, --help
 221         //"javap",            // -?, --help
 222         //"javaw",            // win: -?, -h, --help
 223         //"jcmd",             // -h, but exits with return code
 224         //"jconsole",         // gui, don't test
 225         "jdb",                // -help
 226         "jdeprscan",          // -h, --help, return code 1 -- fixed
 227         //"jdeps",            // none, prints help message anyways.
 228         //"jhsdb",            // Strange ...
 229         //"jimage",           // -h, --help
 230         //"jinfo",            // Displays help message if no arguments are given or wrong ones. Documents -h.
 231         "jjs",                // -h, return code 100
 232         //"jlink",            // -h, --help
 233         //"jmap",             // none, prints help message anyways.
 234         //"jmod",             // -h, --help
 235         //"jps",              // -?, not documented -- fixed
 236         //"jrunscript",       // -?, -help, Displays help message if no arguments are given or if -h, --help.
 237         //"jshell",           // -h, --help, -h not documented. -- fixed
 238         //"jstack",           // -h with return code 0.
 239         //"jstat",            // All work, -help documented -- fixed
 240         "jstatd",             // All work, none documented, return code 1.
 241         //"keytool",          // none, prints help message anyways.
 242         //"kinit",            // win: none, interactive, asks for password, don't test
 243         //"klist",            // win: none, prints help message anyways.
 244         //"ktab",             // win: none, prints help message anyways.
 245         //"orbd",             // Throws exceptions on all three flags
 246         "pack200",            // -? -h --help documented, but return code '1'
 247         //"policytool",       // none, prints help message anyways.
 248         //"rmic",             // none, prints help message anyways.
 249         //"rmid",             // none, prints help message anyways.
 250         //"rmiregistry",      // none, prints help message anyways.
 251         //"schemagen",        // -help, prints help message anyways.
 252         //"serialver",        // none, prints help message anyways.
 253         //"servertool",       // none. Shell, don't test.
 254         //"tnameserv",        // Throws exceptions on all three flags
 255         "unpack200",          // -? -h --help documented, but return code '1'
 256         //"wsgen",            // -help, prints help message anyways.
 257         //"wsimport",         // -help, exception on other flags
 258         //"xjc",              // -help, prints help message anyways.
 259     };
 260 
 261 
 262     // Returns true if tool is listed in ERROR_CODE_ACCEPTABLE.
 263     static boolean errorCodeAcceptable(String tool) {
 264         for (String x : ERROR_CODE_ACCEPTABLE) {
 265             if (tool.toLowerCase().startsWith(x)) return true;
 266         }
 267         return false;
 268     }
 269 
 270     /*
 271      * Checks if the tools accept the option <flag> and that the
 272      * exit code is zero.
 273      * The output of the tools run with <flag> is verified to
 274      * contain the flag string, i.e. it's checked that the help flag 
 275      * itself is documented.
 276      */
 277     static String testHelpOption(String flag, String[] blacklist) {
 278         System.out.println("=== test " + flag + " option === ");
 279         Set<String> failed = new HashSet<>();
 280         Set<String> helpTextFailed = new HashSet<>();
 281         for (File f : new File(JAVA_BIN).listFiles(new ToolFilter(blacklist))) {
 282             String x = f.getAbsolutePath();
 283             TestResult tr = doExec(x, flag);
 284             System.out.println("Testing " + f.getName());
 285             System.out.println("#> " + x + " " + flag);
 286             tr.testOutput.forEach(System.out::println);
 287             System.out.println("#> echo $?");
 288             System.out.println(tr.exitValue);
 289             if (!tr.isOK() && !errorCodeAcceptable(f.getName())) {
 290                 System.out.println("failed");
 291                 failed.add(f.getName());
 292             }
 293             boolean foundFlag = false;
 294             for (String y : tr.testOutput) {
 295                 // Some tools say 'Unknown option "-?"'. Skip that.
 296                 foundFlag |= (y.contains(flag) && !y.contains("nknown"));
 297             }
 298             if (!foundFlag) { helpTextFailed.add(f.getName()); }
 299         }
 300         if (failed.isEmpty() && helpTextFailed.isEmpty()) {
 301             System.out.println("testing flag '" + flag + "' passed");
 302             return "";
 303         } else {
 304             System.out.println("testing flag '" + flag + "' failed");
 305             String res = "";
 306             if (!failed.isEmpty()) {
 307                 res = "\nHelp flag '" + flag + "' not accepted by: " + failed + ".";
 308             }
 309             if (!helpTextFailed.isEmpty()) {
 310                 res += "\nHelp flag '" + flag + "' not documented for: " + helpTextFailed + ".";
 311             }
 312             return res;
 313         }
 314 
 315     }
 316 
 317     public static void main(String[] args) {
 318         String errorMessage = "";
 319         errorMessage += testHelpOption("-?",     BLACKLIST_QOPTION);
 320         errorMessage += testHelpOption("-h",     BLACKLIST_HOPTION);
 321         errorMessage += testHelpOption("--help", BLACKLIST_HELPOPTION);
 322         if (errorMessage.isEmpty()) {
 323             System.out.println("All help string tests: PASS");
 324         } else {
 325             throw new AssertionError("HelpCheck failed: " + errorMessage);
 326         }
 327     }
 328 
 329     static class ToolFilter implements FileFilter {
 330         final Iterable<String> exclude;
 331         protected ToolFilter(String... exclude) {
 332             List<String> tlist = new ArrayList<>();
 333             this.exclude = tlist;
 334             for (String x : exclude) {
 335                 String str = x + ((isWindows) ? EXE_FILE_EXT : "");
 336                 tlist.add(str.toLowerCase());
 337             }
 338         }
 339         @Override
 340         public boolean accept(File pathname) {
 341             if (!pathname.isFile() || !pathname.canExecute()) {
 342                 return false;
 343             }
 344             String name = pathname.getName().toLowerCase();
 345             if (isWindows && !name.endsWith(EXE_FILE_EXT)) {
 346                 return false;
 347             }
 348             for (String x : exclude) {
 349                 if (name.endsWith(x)) {
 350                     return false;
 351                 }
 352             }
 353             return true;
 354         }
 355     }
 356 }