1 /*
2 * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2018 SAP SE. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 */
24
25 /**
26 * @test
27 * @summary Validate and test -?, -h and --help flags. All tools in the jdk
28 * should take the same flags to display the help message. These
29 * flags should be documented in the printed help message. The
30 * tool should quit without error code after displaying the
31 * help message (if there is no other problem with the command
32 * line).
33 * Also check that tools that used to accept -help still do
34 * so. Test that tools that never accepted -help don't do so
35 * in future. I.e., check that the tool returns with the same
36 * return code as called with an invalid flag, and does not
37 * print anything containing '-help' in that case.
38 * @compile HelpFlagsTest.java
39 * @run main HelpFlagsTest
40 */
41
42 import java.io.File;
43 import java.io.FileFilter;
44 import java.util.Map;
45 import java.util.ArrayList;
46 import java.util.HashMap;
47 import java.util.List;
48 import java.util.HashSet;
49 import java.util.Set;
50
51
52 public class HelpFlagsTest extends TestHelper {
53
54 // Tools that should not be tested because a usage message is pointless.
55 static final String[] TOOLS_NOT_TO_TEST = {
56 "appletviewer", // deprecated, don't test
57 "jaccessinspector", // gui, don't test, win only
58 "jaccesswalker", // gui, don't test, win only
59 "jconsole", // gui, don't test
60 "servertool", // none. Shell, don't test.
61 "javaw", // don't test, win only
62 // The flags of these tools need to be fixed in Java EE.
63 // The tools are deprecated for removal in Java SE. Don't test.
64 "idlj",
65 "orbd",
66 "schemagen",
67 "tnameserv",
68 "wsgen",
69 "wsimport",
70 "xjc",
71 // These shall have a help message that resembles that of
72 // MIT's tools. Thus -?, -h and --help are supported, but not
73 // mentioned in the help text.
74 "kinit",
75 "klist",
76 "ktab"
77 };
78
79 // Lists which tools support which flags.
80 private static class ToolHelpSpec {
81 String toolname;
82
83 // How the flags supposed to be supported are handled.
84 //
85 // These flags are supported, i.e.,
86 // * the tool accepts the flag
87 // * the tool prints a help message if the flag is specified
88 // * this help message lists the flag
89 // * the tool exits with exit code '0'.
90 boolean supportsQuestionMark;
91 boolean supportsH;
92 boolean supportsHelp;
93
94 // One tool returns with exit code != '0'.
95 int exitcodeOfHelp;
96
97 // How legacy -help is handled.
98 //
99 // Tools that so far support -help should still do so, but
100 // not print documentation about it. Tools that do not
101 // support -help should not do so in future.
102 //
103 // The tools accepts legacy -help. -help should not be
104 // documented in the usage message.
105 boolean supportsLegacyHelp;
106
107 // Java itself documents -help. -help prints to stderr,
108 // while --help prints to stdout. Leave as is.
109 boolean documentsLegacyHelp;
110
111 // The exit code of the tool if an invalid argument is passed to it.
112 // An exit code != 0 would be expected, but not all tools handle it
113 // that way.
114 int exitcodeOfWrongFlag;
115
116 // Some tools accept the invalid argument and thus hang the test.
117 // Don't execute these with the wrong flags.
118 // This actually should be considered a bug in the corresponding tool.
119 boolean dontExecuteWithWrongFlags;
120
121 ToolHelpSpec(String n, int q, int h, int hp, int ex1, int l, int dl, int ex2, int hangs) {
122 toolname = n;
123 supportsQuestionMark = ( q == 1 ? true : false );
124 supportsH = ( h == 1 ? true : false );
125 supportsHelp = ( hp == 1 ? true : false );
126 exitcodeOfHelp = ex1;
127
128 supportsLegacyHelp = ( l == 1 ? true : false );
129 documentsLegacyHelp = ( dl == 1 ? true : false );
130 exitcodeOfWrongFlag = ex2;
131
132 dontExecuteWithWrongFlags = ( hangs == 1 ? true : false );
133 }
134 }
135
136 static ToolHelpSpec[] jdkTools = {
137 // name -? -h --help exitcode -help -help exitcode Don't
138 // of help docu of wrong test
139 // mented flag inv flag
140 new ToolHelpSpec("jabswitch", 0, 0, 0, 0, 0, 0, 0, 0), // /?, prints help message anyways, win only
141 new ToolHelpSpec("jaotc", 1, 1, 1, 0, 0, 0, 2, 0), // -?, -h, --help
142 new ToolHelpSpec("jar", 1, 1, 1, 0, 0, 0, 1, 0), // -?, -h, --help
143 new ToolHelpSpec("jarsigner", 1, 1, 1, 0, 1, 0, 1, 0), // -?, -h, --help, -help accepted but not documented.
144 new ToolHelpSpec("java", 1, 1, 1, 0, 1, 1, 1, 0), // -?, -h, --help -help, Documents -help
145 new ToolHelpSpec("javac", 1, 0, 1, 0, 1, 1, 2, 0), // -?, --help -help, Documents -help, -h is already taken for "native header output directory".
146 new ToolHelpSpec("javadoc", 1, 1, 1, 0, 1, 1, 1, 0), // -?, -h, --help -help, Documents -help
147 new ToolHelpSpec("javap", 1, 1, 1, 0, 1, 1, 2, 0), // -?, -h, --help, -help accepted but not documented.
148 new ToolHelpSpec("javaw", 1, 1, 1, 0, 1, 1, 1, 0), // -?, -h, --help -help, win only
149 new ToolHelpSpec("jcmd", 1, 1, 1, 0, 1, 0, 1, 0), // -?, -h, --help, -help accepted but not documented.
150 new ToolHelpSpec("jdb", 1, 1, 1, 0, 1, 1, 0, 0), // -?, -h, --help, -help accepted but not documented.
151 new ToolHelpSpec("jdeprscan", 1, 1, 1, 0, 0, 0, 1, 0), // -?, -h, --help
152 new ToolHelpSpec("jdeps", 1, 1, 1, 0, 1, 0, 2, 0), // -?, -h, --help, -help accepted but not documented.
153 new ToolHelpSpec("jhsdb", 0, 0, 0, 0, 0, 0, 0, 0), // none, prints help message anyways.
154 new ToolHelpSpec("jimage", 1, 1, 1, 0, 0, 0, 2, 0), // -?, -h, --help
155 new ToolHelpSpec("jinfo", 1, 1, 1, 0, 1, 1, 1, 0), // -?, -h, --help, -help accepted but not documented.
156 new ToolHelpSpec("jjs", 0, 1, 1, 100, 0, 0, 100, 0), // -h, --help, return code 100
157 new ToolHelpSpec("jlink", 1, 1, 1, 0, 0, 0, 2, 0), // -?, -h, --help
158 new ToolHelpSpec("jmap", 1, 1, 1, 0, 1, 0, 1, 0), // -?, -h, --help, -help accepted but not documented.
159 new ToolHelpSpec("jmod", 1, 1, 1, 0, 1, 0, 2, 0), // -?, -h, --help, -help accepted but not documented.
160 new ToolHelpSpec("jps", 1, 1, 1, 0, 1, 1, 1, 0), // -?, -h, --help, -help accepted but not documented.
161 new ToolHelpSpec("jrunscript", 1, 1, 1, 0, 1, 1, 7, 0), // -?, -h, --help, -help accepted but not documented.
162 new ToolHelpSpec("jshell", 1, 1, 1, 0, 1, 0, 1, 0), // -?, -h, --help, -help accepted but not documented.
163 new ToolHelpSpec("jstack", 1, 1, 1, 0, 1, 1, 1, 0), // -?, -h, --help, -help accepted but not documented.
164 new ToolHelpSpec("jstat", 1, 1, 1, 0, 1, 1, 1, 0), // -?, -h, --help, -help accepted but not documented.
165 new ToolHelpSpec("jstatd", 1, 1, 1, 0, 0, 0, 1, 0), // -?, -h, --help
166 new ToolHelpSpec("keytool", 1, 1, 1, 0, 1, 0, 1, 0), // none, prints help message anyways.
167 new ToolHelpSpec("pack200", 1, 1, 1, 0, 1, 0, 2, 0), // -?, -h, --help, -help accepted but not documented.
168 new ToolHelpSpec("rmic", 0, 0, 0, 0, 0, 0, 1, 0), // none, pirnts help message anyways.
169 new ToolHelpSpec("rmid", 0, 0, 0, 0, 0, 0, 1, 0), // none, prints help message anyways.
170 new ToolHelpSpec("rmiregistry", 0, 0, 0, 0, 0, 0, 1, 0), // none, prints help message anyways.
171 new ToolHelpSpec("serialver", 0, 0, 0, 0, 0, 0, 1, 0), // none, prints help message anyways.
172 new ToolHelpSpec("unpack200", 1, 1, 1, 0, 1, 0, 2, 0), // -?, -h, --help, -help accepted but not documented.
173 };
174
175 // Returns true if the file is not a tool.
176 static boolean notATool(String file) {
177 if (isWindows && !file.endsWith(EXE_FILE_EXT))
178 return true;
179 return false;
180 }
181
182 // Returns true if tool is listed in TOOLS_NOT_TO_TEST.
183 static boolean dontTestTool(String tool) {
184 tool = tool.toLowerCase();
185 for (String x : TOOLS_NOT_TO_TEST) {
186 if (tool.toLowerCase().startsWith(x))
187 return true;
188 }
189 return false;
190 }
191
192 // Returns corresponding object from jdkTools array.
193 static ToolHelpSpec getToolHelpSpec(String tool) {
194 for (ToolHelpSpec x : jdkTools) {
195 if (tool.toLowerCase().equals(x.toolname) ||
196 tool.toLowerCase().equals(x.toolname + ".exe"))
197 return x;
198 }
199 return null;
200 }
201
202 // Check whether 'flag' appears in 'line' as a word of itself. It must not
203 // be a substring of a word, as then similar flags might be matched.
204 // E.g.: --help matches in the documentation of --help-extra.
205 // This works only with english locale, as some tools have translated
206 // usage messages.
207 static boolean findFlagInLine(String line, String flag) {
208 if (line.contains(flag) &&
209 !line.contains("nknown") && // Some tools say 'Unknown option "<flag>"',
210 !line.contains("invalid flag") && // 'invalid flag: <flag>'
211 !line.contains("invalid option") && // or 'invalid option: <flag>'. Skip that.
212 !line.contains("FileNotFoundException: -help") && // Special case for idlj.
213 !line.contains("-h requires an argument") && // Special case for javac.
214 !line.contains("port argument,")) { // Special case for rmiregistry.
215 // There might be several appearances of 'flag' in
216 // 'line'. (-h as substring of --help).
217 int flagLen = flag.length();
218 int lineLen = line.length();
219 for (int i = line.indexOf(flag); i >= 0; i = line.indexOf(flag, i+1)) {
220 // There should be a space before 'flag' in 'line', or it's right at the beginning.
221 if (i > 0 &&
222 line.charAt(i-1) != ' ' &&
223 line.charAt(i-1) != '[' && // jarsigner
224 line.charAt(i-1) != '|' && // jstatd
225 line.charAt(i-1) != '\t') { // jjs
226 continue;
227 }
228 // There should be a space or comma after 'flag' in 'line', or it's just at the end.
229 int posAfter = i + flagLen;
230 if (posAfter < lineLen &&
231 line.charAt(posAfter) != ' ' &&
232 line.charAt(posAfter) != ',' &&
233 line.charAt(posAfter) != '[' && // jar
234 line.charAt(posAfter) != ']' && // jarsigner
235 line.charAt(posAfter) != '|' && // jstatd
236 line.charAt(posAfter) != ':' && // jps
237 line.charAt(posAfter) != '"') { // keytool
238 continue;
239 }
240 return true;
241 }
242 }
243 return false;
244 }
245
246 static TestResult runToolWithFlag(File f, String flag) {
247 String x = f.getAbsolutePath();
248 TestResult tr = doExec(x, flag);
249 System.out.println("Testing " + f.getName());
250 System.out.println("#> " + x + " " + flag);
251 tr.testOutput.forEach(System.out::println);
252 System.out.println("#> echo $?");
253 System.out.println(tr.exitValue);
254
255 return tr;
256 }
257
258 // Checks whether tool supports flag 'flag' and documents it
259 // in the help message.
260 static String testTool(File f, String flag, int exitcode) {
261 String result = "";
262 TestResult tr = runToolWithFlag(f, flag);
263
264 // Check that the tool accepted the flag.
265 if (exitcode == 0 && !tr.isOK()) {
266 System.out.println("failed");
267 result = "failed: " + f.getName() + " " + flag + " has exit code " + tr.exitValue + ".\n";
268 }
269
270 // Check there is a help message listing the flag.
271 boolean foundFlag = false;
272 for (String y : tr.testOutput) {
273 if (!foundFlag && findFlagInLine(y, flag)) { // javac
274 foundFlag = true;
275 System.out.println("Found documentation of '" + flag + "': '" + y.trim() +"'");
276 }
277 }
278 if (!foundFlag) {
279 result += "failed: " + f.getName() + " does not document " +
280 flag + " in help message.\n";
281 }
282
283 if (!result.isEmpty())
284 System.out.println(result);
285
286 return result;
287 }
288
289 // Test the tool supports legacy option -help, but does
290 // not document it.
291 static String testLegacyFlag(File f, int exitcode) {
292 String result = "";
293 TestResult tr = runToolWithFlag(f, "-help");
294
295 // Check that the tool accepted the flag.
296 if (exitcode == 0 && !tr.isOK()) {
297 System.out.println("failed");
298 result = "failed: " + f.getName() + " -help has exit code " + tr.exitValue + ".\n";
299 }
300
301 // Check there is _no_ documentation of -help.
302 boolean foundFlag = false;
303 for (String y : tr.testOutput) {
304 if (!foundFlag && findFlagInLine(y, "-help")) { // javac
305 foundFlag = true;
306 System.out.println("Found documentation of '-help': '" + y.trim() +"'");
307 }
308 }
309 if (foundFlag) {
310 result += "failed: " + f.getName() + " does document -help " +
311 "in help message. This legacy flag should not be documented.\n";
312 }
313
314 if (!result.isEmpty())
315 System.out.println(result);
316
317 return result;
318 }
319
320 // Test that the tool exits with the exit code expected for
321 // invalid flags. In general, one would expect this to be != 0,
322 // but currently a row of tools exit with 0 in this case.
323 // The output should not ask to get help with flag '-help'.
324 static String testInvalidFlag(File f, String flag, int exitcode, boolean documentsLegacyHelp) {
325 String result = "";
326 TestResult tr = runToolWithFlag(f, flag);
327
328 // Check that the tool did exit with the expected return code.
329 if (!((exitcode == tr.exitValue) ||
330 // Windows reports -1 where unix reports 255.
331 (tr.exitValue < 0 && exitcode == tr.exitValue + 256))) {
332 System.out.println("failed");
333 result = "failed: " + f.getName() + " " + flag + " should not be " +
334 "accepted. But it has exit code " + tr.exitValue + ".\n";
335 }
336
337 if (!documentsLegacyHelp) {
338 // Check there is _no_ documentation of -help.
339 boolean foundFlag = false;
340 for (String y : tr.testOutput) {
341 if (!foundFlag && findFlagInLine(y, "-help")) { // javac
342 foundFlag = true;
343 System.out.println("Found documentation of '-help': '" + y.trim() +"'");
344 }
345 }
346 if (foundFlag) {
347 result += "failed: " + f.getName() + " does document -help " +
348 "in error message. This legacy flag should not be documented.\n";
349 }
350 }
351
352 if (!result.isEmpty())
353 System.out.println(result);
354
355 return result;
356 }
357
358 public static void main(String[] args) {
359 String errorMessage = "";
360
361 // The test analyses the help messages printed. It assumes englisch
362 // help messages. Thus it only works with english locale.
363 if (!isEnglishLocale()) { return; }
364
365 for (File f : new File(JAVA_BIN).listFiles()) {
366 String toolName = f.getName();
367
368 if (notATool(toolName)) {
369 continue;
370 }
371 if (dontTestTool(toolName)) {
372 System.out.println("Skipping test of tool " + toolName +
373 ". Tool has no help message.");
374 continue;
375 }
376
377 ToolHelpSpec tool = getToolHelpSpec(toolName);
378 if (tool == null) {
379 errorMessage += "Tool " + toolName + " not covered by this test. " +
380 "Add specification to jdkTools array!\n";
381 continue;
382 }
383
384 // Test for help flags to be supported.
385 if (tool.supportsQuestionMark == true) {
386 errorMessage += testTool(f, "-?", tool.exitcodeOfHelp);
387 } else {
388 System.out.println("Skip " + tool.toolname + ". It does not support -?.");
389 }
390 if (tool.supportsH == true) {
391 errorMessage += testTool(f, "-h", tool.exitcodeOfHelp);
392 } else {
393 System.out.println("Skip " + tool.toolname + ". It does not support -h.");
394 }
395 if (tool.supportsHelp == true) {
396 errorMessage += testTool(f, "--help", tool.exitcodeOfHelp);
397 } else {
398 System.out.println("Skip " + tool.toolname + ". It does not support --help.");
399 }
400
401 // Check that the return code listing in jdkTools[] is
402 // correct for an invalid flag.
403 if (!tool.dontExecuteWithWrongFlags) {
404 errorMessage += testInvalidFlag(f, "-asdfxgr", tool.exitcodeOfWrongFlag, tool.documentsLegacyHelp);
405 }
406
407 // Test for legacy -help flag.
408 if (!tool.documentsLegacyHelp) {
409 if (tool.supportsLegacyHelp == true) {
410 errorMessage += testLegacyFlag(f, tool.exitcodeOfHelp);
411 } else {
412 if (!tool.dontExecuteWithWrongFlags) {
413 errorMessage += testInvalidFlag(f, "-help", tool.exitcodeOfWrongFlag, false);
414 }
415 }
416 }
417 }
418
419 if (errorMessage.isEmpty()) {
420 System.out.println("All help string tests: PASS");
421 } else {
422 throw new AssertionError("HelpFlagsTest failed:\n" + errorMessage);
423 }
424 }
425 }