1 /*
2 * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23
24 /**
25 * @test
26 * @summary Validate and test -?, -h and --help flags. All tools in the jdk
27 * should take the same flags to display the help message. These
28 * flags should be documented in the printed help message. The
29 * tool should quit without error code after displaying the
30 * help message (if there is no other problem with the command
31 * line).
32 * Also check that tools that used to accept -help still do so,
33 * but do not document it. Test that tools that never accepted
34 * -help don't do so in future. I.e., check that the tool returns
35 * with the same return code as called with an invalid flag, and
36 * does not print anything containing '-help' in that case.
37 * @compile HelpFlagsTest.java
38 * @run main HelpFlagsTest
39 */
40
41 import java.io.File;
42 import java.io.FileFilter;
43 import java.util.Map;
44 import java.util.ArrayList;
45 import java.util.HashMap;
46 import java.util.List;
47 import java.util.HashSet;
48 import java.util.Set;
49
50
51 public class HelpFlagsTest extends TestHelper {
52
53
54 // Tools that should not be tested because a usage message is pointless.
55 static final String[] TOOLS_NOT_TO_TEST = {
56 "appletviewer", // deprecated, don't test
57 "jaccessinspector", // gui, don't test, win only
58 "jaccesswalker", // gui, don't test, win only
59 "jconsole", // gui, don't test
60 "servertool", // none. Shell, don't test.
61 "javaw", // don't test, win only
62 };
63
64 // Lists which tools support which flags.
65 private static class ToolHelpSpec {
66 String toolname;
67
68 // How the flags supposed to be supported are handled.
69 //
70 // These flags are supported, i.e.,
71 // * the tool accepts the flag
72 // * the tool prints a help message if the flag is specified
73 // * this help message lists the flag
74 // * the tool exits with exit code '0'.
75 boolean supports_q;
76 boolean supports_h;
77 boolean supports_help;
78 // One tool returns with exit code != '0'.
79 int exitcode_of_help;
80
81 // How legacy -help is handled.
82 //
83 // Tools that so far support -help should still do so, but
84 // not print documentation about it. Tools that do not
85 // support -help should not do so in future.
86 //
87 // The tools accepts legacy -help. -help should not be
88 // documented in the usage message.
89 boolean supports_legacy_help;
90 // Java itself documents -help. -help prints to stderr,
91 // while --help prints to stdout. Leave as is.
92 boolean documents_legacy_help;
93
94 // The exit code of the tool if an invalid argument is passed to it.
95 // An exit code != 0 would be expected, but not all tools handle it
96 // that way.
97 int exitcode_of_wrong_flag;
98
99 // Some tools accept the invalid argument and thus hang the test.
100 // Don't execute these with the wrong flags.
101 // This actually should be considered a bug in the corresponding tool.
102 boolean dont_execute_with_wrong_flags;
103
104 ToolHelpSpec(String n, int q, int h, int hp, int ex1, int l, int dl, int ex2, int hangs) {
105 toolname = n;
106 supports_q = ( q == 1 ? true : false );
107 supports_h = ( h == 1 ? true : false );
108 supports_help = ( hp == 1 ? true : false );
109 exitcode_of_help = ex1;
110
111 supports_legacy_help = ( l == 1 ? true : false );
112 documents_legacy_help = ( dl == 1 ? true : false );
113 exitcode_of_wrong_flag = ex2;
114
115 dont_execute_with_wrong_flags = ( hangs == 1 ? true : false );
116 }
117 }
118
119 static ToolHelpSpec[] jdkTools = {
120 // name -? -h --help exitcode -help -help exitcode Don't
121 // of help docu of wrong test
122 // mented flag inv flag
123 new ToolHelpSpec("idlj", 0, 0, 0, 0, 0, 0, 0, 0), // none, prints help message anyways.
124 new ToolHelpSpec("jabswitch", 0, 0, 0, 0, 0, 0, 0, 0), // /?, prints help message anyways, win only
125 new ToolHelpSpec("jaotc", 1, 1, 1, 0, 0, 0, 2, 0), // -?, -h, --help
126 new ToolHelpSpec("jar", 1, 1, 1, 0, 0, 0, 1, 0), // -?, -h, --help
127 new ToolHelpSpec("jarsigner", 1, 1, 1, 0, 1, 0, 1, 0), // -?, -h, --help, -help accepted but not documented.
128 new ToolHelpSpec("java", 1, 1, 1, 0, 1, 1, 1, 0), // -?, -h, --help -help, Documents -help
129 new ToolHelpSpec("javac", 1, 0, 1, 0, 0, 0, 2, 0), // -?, --help, -h is already taken for "native header output directory".
130 new ToolHelpSpec("javadoc", 1, 1, 1, 0, 0, 0, 1, 0), // -?, -h, --help
131 new ToolHelpSpec("javah", 1, 1, 1, 0, 1, 0, 1, 0), // -?, -h, --help, -help accepted but not documented.
132 new ToolHelpSpec("javap", 1, 1, 1, 0, 1, 0, 2, 0), // -?, -h, --help, -help accepted but not documented.
133 new ToolHelpSpec("javaw", 1, 1, 1, 0, 1, 1, 1, 0), // -?, -h, --help -help, win only
134 new ToolHelpSpec("jcmd", 1, 1, 1, 0, 1, 0, 1, 0), // -?, -h, --help, -help accepted but not documented.
135 new ToolHelpSpec("jdb", 1, 1, 1, 0, 1, 0, 0, 0), // -?, -h, --help, -help accepted but not documented.
136 new ToolHelpSpec("jdeprscan", 1, 1, 1, 0, 0, 0, 1, 0), // -?, -h, --help
137 new ToolHelpSpec("jdeps", 1, 1, 1, 0, 1, 0, 2, 0), // -?, -h, --help, -help accepted but not documented.
138 new ToolHelpSpec("jhsdb", 0, 0, 0, 0, 0, 0, 0, 0), // none, prints help message anyways.
139 new ToolHelpSpec("jimage", 1, 1, 1, 0, 0, 0, 2, 0), // -?, -h, --help
140 new ToolHelpSpec("jinfo", 1, 1, 1, 0, 1, 0, 1, 0), // -?, -h, --help, -help accepted but not documented.
141 new ToolHelpSpec("jjs", 0, 1, 1, 100, 0, 0, 100, 0), // -h, --help, return code 100
142 new ToolHelpSpec("jlink", 1, 1, 1, 0, 0, 0, 2, 0), // -?, -h, --help
143 new ToolHelpSpec("jmap", 1, 1, 1, 0, 1, 0, 1, 0), // -?, -h, --help, -help accepted but not documented.
144 new ToolHelpSpec("jmod", 1, 1, 1, 0, 1, 0, 2, 0), // -?, -h, --help, -help accepted but not documented.
145 new ToolHelpSpec("jps", 1, 1, 1, 0, 1, 0, 1, 0), // -?, -h, --help, -help accepted but not documented.
146 new ToolHelpSpec("jrunscript", 1, 1, 1, 0, 1, 0, 7, 0), // -?, -h, --help, -help accepted but not documented.
147 new ToolHelpSpec("jshell", 1, 1, 1, 0, 1, 0, 1, 0), // -?, -h, --help, -help accepted but not documented.
148 new ToolHelpSpec("jstack", 1, 1, 1, 0, 1, 0, 1, 0), // -?, -h, --help, -help accepted but not documented.
149 new ToolHelpSpec("jstat", 1, 1, 1, 0, 1, 0, 1, 0), // -?, -h, --help, -help accepted but not documented.
150 new ToolHelpSpec("jstatd", 1, 1, 1, 0, 0, 0, 1, 0), // -?, -h, --help
151 new ToolHelpSpec("keytool", 1, 1, 1, 0, 1, 0, 1, 0), // none, prints help message anyways.
152 new ToolHelpSpec("kinit", 1, 1, 1, 0, 1, 0, 0, 1), // -?, -h, --help, -help accepted but not documented.
153 new ToolHelpSpec("klist", 1, 1, 1, 0, 0, 0, -1, 0), // -?, -h, --help, win only
154 new ToolHelpSpec("ktab", 1, 1, 1, 0, 1, 0, -1, 0), // -?, -h, --help, -help accepted but not documented. win only.
155 new ToolHelpSpec("orbd", 1, 1, 1, 0, 0, 0, 0, 1), // -?, -h, --help
156 new ToolHelpSpec("pack200", 1, 1, 1, 0, 1, 0, 2, 0), // -?, -h, --help, -help accepted but not documented.
157 new ToolHelpSpec("policytool", 0, 0, 0, 0, 0, 0, 0, 0), // none, prints help message anyways.
158 new ToolHelpSpec("rmic", 0, 0, 0, 0, 0, 0, 1, 0), // none, pirnts help message anyways.
159 new ToolHelpSpec("rmid", 0, 0, 0, 0, 0, 0, 1, 0), // none, prints help message anyways.
160 new ToolHelpSpec("rmiregistry", 0, 0, 0, 0, 0, 0, 1, 0), // none, prints help message anyways.
161 new ToolHelpSpec("schemagen", 1, 1, 1, 0, 1, 0, 255, 0), // -?, -h, --help, -help accepted but not documented.
162 new ToolHelpSpec("serialver", 0, 0, 0, 0, 0, 0, 1, 0), // none, prints help message anyways.
163 new ToolHelpSpec("tnameserv", 0, 0, 0, 0, 0, 0, 0, 1), // Just starts given any flag.
164 new ToolHelpSpec("unpack200", 1, 1, 1, 0, 1, 0, 2, 0), // -?, -h, --help, -help accepted but not documented.
165 new ToolHelpSpec("wsgen", 1, 1, 1, 0, 1, 0, 1, 0), // -?, -h, --help, -help accepted but not documented.
166 new ToolHelpSpec("wsimport", 1, 1, 1, 0, 1, 0, 1, 0), // -?, -h, --help, -help accepted but not documented.
167 new ToolHelpSpec("xjc", 1, 1, 1, 0, 1, 0, 255, 0) // -?, -h, --help, -help accepted but not documented.
168 };
169
170
171 // Returns true if the file is not a tool.
172 static boolean notATool(String file) {
173 file = file.toLowerCase();
174 if (file.endsWith(".dll") ||
175 file.endsWith(".map") ||
176 file.endsWith(".pdb") ||
177 file.equals("server")) { // server subdir on windows.
178 return true;
179 }
180 return false;
181 }
182
183 // Returns true if tool is listed in TOOLS_NOT_TO_TEST.
184 static boolean dontTestTool(String tool) {
185 tool = tool.toLowerCase();
186 for (String x : TOOLS_NOT_TO_TEST) {
187 if (tool.toLowerCase().startsWith(x)) return true;
188 }
189 return false;
190 }
191
192 // Returns corresponding object from jdkTools array.
193 static ToolHelpSpec getToolHelpSpec(String tool) {
194 for (ToolHelpSpec x : jdkTools) {
195 if (tool.toLowerCase().equals(x.toolname) ||
196 tool.toLowerCase().equals(x.toolname + ".exe")) return x;
197 }
198 return null;
199 }
200
201 // Check whether 'flag' appears in 'line' as a word of itself. I must not
202 // be a substring of a word, as then similar flags might be matched.
203 // E.g.: --help matches in the documentation of --help-extra.
204 // This works only with english locale, as some tools have translated
205 // usage messages.
206 static boolean findFlagInLine(String line, String flag) {
207 if (line.contains(flag) &&
208 !line.contains("nknown") && // Some tools say 'Unknown option "<flag>"',
209 !line.contains("invalid flag") && // 'invalid flag: <flag>'
210 !line.contains("invalid option") && // or 'invalid option: <flag>'. Skip that.
211 !line.contains("FileNotFoundException: -help") && // Special case for idlj.
212 !line.contains("-h requires an argument") && // Special case for javac.
213 !line.contains("port argument,")) { // Special case for rmiregistry.
214 // There might be several appearances of 'flag' in
215 // 'line'. (-h as substring of --help).
216 int flagLen = flag.length();
217 int lineLen = line.length();
218 for (int i = line.indexOf(flag); i >= 0; i = line.indexOf(flag, i+1)) {
219 // There should be a space before 'flag' in 'line', or it's right at the beginning.
220 if (i > 0 &&
221 line.charAt(i-1) != ' ' &&
222 line.charAt(i-1) != '[' && // jarsigner
223 line.charAt(i-1) != '|' && // jstatd
224 line.charAt(i-1) != '\t') { // jjs
225 continue;
226 }
227 // There should be a space or comma after 'flag' in 'line', or it's just at the end.
228 int posAfter = i + flagLen;
229 if (posAfter < lineLen &&
230 line.charAt(posAfter) != ' ' &&
231 line.charAt(posAfter) != ',' &&
232 line.charAt(posAfter) != '[' && // jar
233 line.charAt(posAfter) != ']' && // jarsigner
234 line.charAt(posAfter) != '|' && // jstatd
235 line.charAt(posAfter) != ':' && // jps
236 line.charAt(posAfter) != '"') { // keytool
237 continue;
238 }
239 return true;
240 }
241 }
242 return false;
243 }
244
245 static TestResult runToolWithFlag(File f, String flag) {
246 String x = f.getAbsolutePath();
247 TestResult tr = doExec(x, flag);
248 System.out.println("Testing " + f.getName());
249 System.out.println("#> " + x + " " + flag);
250 tr.testOutput.forEach(System.out::println);
251 System.out.println("#> echo $?");
252 System.out.println(tr.exitValue);
253
254 return tr;
255 }
256
257 // Checks whether tool supports flag 'flag' and documents it
258 // in the help message.
259 static String testTool(File f, String flag, int exitcode) {
260 String result = "";
261 TestResult tr = runToolWithFlag(f, flag);
262
263 // Check that the tool accepted the flag.
264 if (exitcode == 0 && !tr.isOK()) {
265 System.out.println("failed");
266 result = "failed: " + f.getName() + " " + flag + " has exit code " + tr.exitValue + ".\n";
267 }
268
269 // Check there is a help message listing the flag.
270 boolean foundFlag = false;
271 for (String y : tr.testOutput) {
272 if (!foundFlag && findFlagInLine(y, flag)) { // javac
273 foundFlag = true;
274 System.out.println("Found documentation of '" + flag + "': '" + y.trim() +"'");
275 }
276 }
277 if (!foundFlag) {
278 result += "failed: " + f.getName() + " does not document " +
279 flag + " in help message.\n";
280 }
281 if (!result.isEmpty()) { System.out.println(result); }
282 return result;
283 }
284
285 // Test the tool supports legacy option -help, but does
286 // not document it.
287 static String testLegacyFlag(File f, int exitcode) {
288 String result = "";
289 TestResult tr = runToolWithFlag(f, "-help");
290
291 // Check that the tool accepted the flag.
292 if (exitcode == 0 && !tr.isOK()) {
293 System.out.println("failed");
294 result = "failed: " + f.getName() + " -help has exit code " + tr.exitValue + ".\n";
295 }
296
297
298 // Check there is _no_ documentation of -help.
299 boolean foundFlag = false;
300 for (String y : tr.testOutput) {
301 if (!foundFlag && findFlagInLine(y, "-help")) { // javac
302 foundFlag = true;
303 System.out.println("Found documentation of '-help': '" + y.trim() +"'");
304 }
305 }
306 if (foundFlag) {
307 result += "failed: " + f.getName() + " does document -help " +
308 "in help message. This legacy flag should not be documented.\n";
309 }
310 if (!result.isEmpty()) { System.out.println(result); }
311 return result;
312 }
313
314 // Test that the tool exits with the exit code expected for
315 // invalid flags. In general, one would expect this to be != 0,
316 // but currently a row of tools exit with 0 in this case.
317 // The output should not ask to get help with flag '-help'.
318 static String testInvalidFlag(File f, String flag, int exitcode) {
319 String result = "";
320 TestResult tr = runToolWithFlag(f, flag);
321
322 // Check that the tool did exit with the expected return code.
323 if (!((exitcode == tr.exitValue) ||
324 // Windows reports -1 where unix reports 255.
325 (tr.exitValue < 0 && exitcode == tr.exitValue + 256))) {
326 System.out.println("failed");
327 result = "failed: " + f.getName() + " " + flag + " should not be " +
328 "accepted. But it has exit code " + tr.exitValue + ".\n";
329 }
330
331 // Check there is _no_ documentation of -help.
332 boolean foundFlag = false;
333 for (String y : tr.testOutput) {
334 if (!foundFlag && findFlagInLine(y, "-help")) { // javac
335 foundFlag = true;
336 System.out.println("Found documentation of '-help': '" + y.trim() +"'");
337 }
338 }
339 if (foundFlag) {
340 result += "failed: " + f.getName() + " does document -help " +
341 "in error message. This legacy flag should not be documented.\n";
342 }
343 if (!result.isEmpty()) { System.out.println(result); }
344 return result;
345 }
346
347 public static void main(String[] args) {
348 String errorMessage = "";
349
350 for (File f : new File(JAVA_BIN).listFiles()) {
351 String toolName = f.getName();
352
353 if (notATool(toolName)) {
354 continue;
355 }
356 if (dontTestTool(toolName)) {
357 System.out.println("Skipping test of tool " + toolName +
358 ". Tool has no help message.");
359 continue;
360 }
361
362 ToolHelpSpec tool = getToolHelpSpec(toolName);
363 if (tool == null) {
364 errorMessage += "Tool " + toolName + " not covered by this test. " +
365 "Add specification to jdkTools array!\n";
366 continue;
367 }
368
369 // Test for help flags to be supported.
370 if (tool.supports_q == true) {
371 errorMessage += testTool(f, "-?", tool.exitcode_of_help);
372 } else {
373 System.out.println("Skip " + tool.toolname + ". It does not support -?.");
374 }
375 if (tool.supports_h == true) {
376 errorMessage += testTool(f, "-h", tool.exitcode_of_help);
377 } else {
378 System.out.println("Skip " + tool.toolname + ". It does not support -h.");
379 }
380 if (tool.supports_help == true) {
381 errorMessage += testTool(f, "--help", tool.exitcode_of_help);
382 } else {
383 System.out.println("Skip " + tool.toolname + ". It does not support --help.");
384 }
385
386 // Check that the return code listing in jdkTools[] is
387 // correct for an invalid flag.
388 if (!tool.dont_execute_with_wrong_flags) {
389 errorMessage += testInvalidFlag(f, "-asdfxgr", tool.exitcode_of_wrong_flag);
390 }
391
392 // Test for legacy -help flag.
393 if (!tool.documents_legacy_help) {
394 if (tool.supports_legacy_help == true) {
395 errorMessage += testLegacyFlag(f, tool.exitcode_of_help);
396 } else {
397 if (!tool.dont_execute_with_wrong_flags) {
398 errorMessage += testInvalidFlag(f, "-help", tool.exitcode_of_wrong_flag);
399 }
400 }
401 }
402 }
403
404 if (errorMessage.isEmpty()) {
405 System.out.println("All help string tests: PASS");
406 } else {
407 throw new AssertionError("HelpFlagsTest failed:\n" + errorMessage);
408 }
409 }
410 }