1 /*
   2  * Copyright (c) 2002, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 import java.io.BufferedReader;
  25 import java.io.BufferedWriter;
  26 import java.io.ByteArrayOutputStream;
  27 import java.io.File;
  28 import java.io.FileNotFoundException;
  29 import java.io.FileWriter;
  30 import java.io.FilenameFilter;
  31 import java.io.InputStreamReader;
  32 import java.io.IOException;
  33 import java.io.PrintStream;
  34 import java.io.PrintWriter;
  35 import java.io.StringReader;
  36 import java.io.StringWriter;
  37 import java.lang.annotation.Annotation;
  38 import java.lang.annotation.Retention;
  39 import java.lang.annotation.RetentionPolicy;
  40 import java.lang.ref.SoftReference;
  41 import java.lang.reflect.InvocationTargetException;
  42 import java.lang.reflect.Method;
  43 import java.net.URI;
  44 import java.net.URISyntaxException;
  45 import java.nio.charset.Charset;
  46 import java.nio.charset.CharsetDecoder;
  47 import java.nio.charset.CodingErrorAction;
  48 import java.nio.charset.UnsupportedCharsetException;
  49 import java.nio.file.FileVisitResult;
  50 import java.nio.file.Files;
  51 import java.nio.file.Path;
  52 import java.nio.file.Paths;
  53 import java.nio.file.SimpleFileVisitor;
  54 import java.nio.file.attribute.BasicFileAttributes;
  55 import java.util.ArrayList;
  56 import java.util.Arrays;
  57 import java.util.Collection;
  58 import java.util.Collections;
  59 import java.util.Comparator;
  60 import java.util.EnumMap;
  61 import java.util.HashMap;
  62 import java.util.LinkedHashMap;
  63 import java.util.List;
  64 import java.util.Locale;
  65 import java.util.Map;
  66 import java.util.Objects;
  67 import java.util.Set;
  68 import java.util.TreeMap;
  69 import java.util.TreeSet;
  70 import java.util.function.Function;
  71 import java.util.regex.Pattern;
  72 import java.util.stream.Collectors;
  73 
  74 
  75 /**
  76  * Test framework for running javadoc and performing tests on the resulting output.
  77  *
  78  * <p>
  79  * Tests are typically written as subtypes of JavadocTester, with a main
  80  * method that creates an instance of the test class and calls the runTests()
  81  * method. The runTests() methods calls all the test methods declared in the class,
  82  * and then calls a method to print a summary, and throw an exception if
  83  * any of the test methods reported a failure.
  84  *
  85  * <p>
  86  * Test methods are identified with a @Test annotation. They have no parameters.
  87  * The name of the method is not important, but if you have more than one, it is
  88  * recommended that the names be meaningful and suggestive of the test case
  89  * contained therein.
  90  *
  91  * <p>
  92  * Typically, a test method will invoke javadoc, and then perform various
  93  * checks on the results. The standard checks are:
  94  *
  95  * <dl>
  96  * <dt>checkExitCode
  97  * <dd>Check the exit code returned from javadoc.
  98  * <dt>checkOutput
  99  * <dd>Perform a series of checks on the contents on a file or output stream
 100  *     generated by javadoc.
 101  *     The checks can be either that a series of strings are found or are not found.
 102  * <dt>checkFiles
 103  * <dd>Perform a series of checks on the files generated by javadoc.
 104  *     The checks can be that a series of files are found or are not found.
 105  * </dl>
 106  *
 107  * <pre><code>
 108  *  public class MyTester extends JavadocTester {
 109  *      public static void main(String... args) throws Exception {
 110  *          MyTester tester = new MyTester();
 111  *          tester.runTests();
 112  *      }
 113  *
 114  *      // test methods...
 115  *      @Test
 116  *      void test() {
 117  *          javadoc(<i>args</i>);
 118  *          checkExit(Exit.OK);
 119  *          checkOutput(<i>file</i>, true,
 120  *              <i>strings-to-find</i>);
 121  *          checkOutput(<i>file</i>, false,
 122  *              <i>strings-to-not-find</i>);
 123  *      }
 124  *  }
 125  * </code></pre>
 126  *
 127  * <p>
 128  * If javadoc is run more than once in a test method, you can compare the
 129  * results that are generated with the diff method. Since files written by
 130  * javadoc typically contain a timestamp, you may want to use the -notimestamp
 131  * option if you are going to compare the results from two runs of javadoc.
 132  *
 133  * <p>
 134  * If you have many calls of checkOutput that are very similar, you can write
 135  * your own check... method to reduce the amount of duplication. For example,
 136  * if you want to check that many files contain the same string, you could
 137  * write a method that takes a varargs list of files and calls checkOutput
 138  * on each file in turn with the string to be checked.
 139  *
 140  * <p>
 141  * You can also write you own custom check methods, which can use
 142  * readFile to get the contents of a file generated by javadoc,
 143  * and then use pass(...) or fail(...) to report whether the check
 144  * succeeded or not.
 145  *
 146  * <p>
 147  * You can have many separate test methods, each identified with a @Test
 148  * annotation. However, you should <b>not</b> assume they will be called
 149  * in the order declared in your source file.  If the order of a series
 150  * of javadoc invocations is important, do that within a single method.
 151  * If the invocations are independent, for better clarity, use separate
 152  * test methods, each with their own set of checks on the results.
 153  *
 154  * @author Doug Kramer
 155  * @author Jamie Ho
 156  * @author Jonathan Gibbons (rewrite)
 157  */
 158 public abstract class JavadocTester {
 159 
 160     public static final String FS = System.getProperty("file.separator");
 161     public static final String PS = System.getProperty("path.separator");
 162     public static final String NL = System.getProperty("line.separator");
 163     public static final Path currDir = Paths.get(".").toAbsolutePath().normalize();
 164 
 165     public enum Output {
 166         /** The name of the output stream from javadoc. */
 167         OUT,
 168         /** The name for any output written to System.out. */
 169         STDOUT,
 170         /** The name for any output written to System.err. */
 171         STDERR
 172     }
 173 
 174     /** The output directory used in the most recent call of javadoc. */
 175     protected File outputDir;
 176 
 177     /** The output charset used in the most recent call of javadoc. */
 178     protected Charset charset = Charset.defaultCharset();
 179 
 180     /** The exit code of the most recent call of javadoc. */
 181     private int exitCode;
 182 
 183     /** The output generated by javadoc to the various writers and streams. */
 184     private final Map<Output, String> outputMap = new EnumMap<>(Output.class);
 185 
 186     /** A cache of file content, to avoid reading files unnecessarily. */
 187     private final Map<File,SoftReference<String>> fileContentCache = new HashMap<>();
 188     /** The charset used for files in the fileContentCache. */
 189     private Charset fileContentCacheCharset = null;
 190 
 191     /** Stream used for logging messages. */
 192     protected final PrintStream out = System.out;
 193 
 194     /** The directory containing the source code for the test. */
 195     public static final String testSrc = System.getProperty("test.src");
 196 
 197     /**
 198      * Get the path for a source file in the test source directory.
 199      * @param path the path of a file or directory in the source directory
 200      * @return the full path of the specified file
 201      */
 202     public static String testSrc(String path) {
 203         return new File(testSrc, path).getPath();
 204     }
 205 
 206     /**
 207      * Alternatives for checking the contents of a directory.
 208      */
 209     public enum DirectoryCheck {
 210         /**
 211          * Check that the directory is empty.
 212          */
 213         EMPTY((file, name) -> true),
 214         /**
 215          * Check that the directory does not contain any HTML files,
 216          * such as may have been generated by a prior run of javadoc
 217          * using this directory.
 218          * For now, the check is only performed on the top level directory.
 219          */
 220         NO_HTML_FILES((file, name) -> name.endsWith(".html")),
 221         /**
 222          * No check is performed on the directory contents.
 223          */
 224         NONE(null) { @Override void check(File dir) { } };
 225 
 226         /** The filter used to detect that files should <i>not</i> be present. */
 227         FilenameFilter filter;
 228 
 229         DirectoryCheck(FilenameFilter f) {
 230             filter = f;
 231         }
 232 
 233         void check(File dir) {
 234             if (dir.isDirectory()) {
 235                 String[] contents = dir.list(filter);
 236                 if (contents == null)
 237                     throw new Error("cannot list directory: " + dir);
 238                 if (contents.length > 0) {
 239                     System.err.println("Found extraneous files in dir:" + dir.getAbsolutePath());
 240                     for (String x : contents) {
 241                         System.err.println(x);
 242                     }
 243                     throw new Error("directory has unexpected content: " + dir);
 244                 }
 245             }
 246         }
 247     }
 248 
 249     private DirectoryCheck outputDirectoryCheck = DirectoryCheck.EMPTY;
 250 
 251     private boolean automaticCheckLinks = true;
 252 
 253     /** The current subtest number. Incremented when checking(...) is called. */
 254     private int numTestsRun = 0;
 255 
 256     /** The number of subtests passed. Incremented when passed(...) is called. */
 257     private int numTestsPassed = 0;
 258 
 259     /** The current run of javadoc. Incremented when javadoc is called. */
 260     private int javadocRunNum = 0;
 261 
 262     /** The current subtest number for this run of javadoc. Incremented when checking(...) is called. */
 263     private int javadocTestNum = 0;
 264 
 265     /** Marker annotation for test methods to be invoked by runTests. */
 266     @Retention(RetentionPolicy.RUNTIME)
 267     @interface Test { }
 268 
 269     /**
 270      * Run all methods annotated with @Test, followed by printSummary.
 271      * Typically called on a tester object in main()
 272      * @throws Exception if any errors occurred
 273      */
 274     public void runTests() throws Exception {
 275         runTests(m -> new Object[0]);
 276     }
 277 
 278     /**
 279      * Run all methods annotated with @Test, followed by printSummary.
 280      * Typically called on a tester object in main()
 281      * @param f a function which will be used to provide arguments to each
 282      *          invoked method
 283      * @throws Exception if any errors occurred
 284      */
 285     public void runTests(Function<Method, Object[]> f) throws Exception {
 286         for (Method m: getClass().getDeclaredMethods()) {
 287             Annotation a = m.getAnnotation(Test.class);
 288             if (a != null) {
 289                 try {
 290                     out.println("Running test " + m.getName());
 291                     m.invoke(this, f.apply(m));
 292                 } catch (InvocationTargetException e) {
 293                     Throwable cause = e.getCause();
 294                     throw (cause instanceof Exception) ? ((Exception) cause) : e;
 295                 }
 296                 out.println();
 297             }
 298         }
 299         printSummary();
 300     }
 301 
 302     /**
 303      * Run javadoc.
 304      * The output directory used by this call and the final exit code
 305      * will be saved for later use.
 306      * To aid the reader, it is recommended that calls to this method
 307      * put each option and the arguments it takes on a separate line.
 308      *
 309      * Example:
 310      * <pre><code>
 311      *  javadoc("-d", "out",
 312      *          "-sourcepath", testSrc,
 313      *          "-notimestamp",
 314      *          "pkg1", "pkg2", "pkg3/C.java");
 315      * </code></pre>
 316      *
 317      * @param args the arguments to pass to javadoc
 318      */
 319     public void javadoc(String... args) {
 320         outputMap.clear();
 321         fileContentCache.clear();
 322 
 323         javadocRunNum++;
 324         javadocTestNum = 0; // reset counter for this run of javadoc
 325         if (javadocRunNum == 1) {
 326             out.println("Running javadoc...");
 327         } else {
 328             out.println("Running javadoc (run "+ javadocRunNum + ")...");
 329         }
 330 
 331         outputDir = new File(".");
 332         String charsetArg = null;
 333         String docencodingArg = null;
 334         String encodingArg = null;
 335         for (int i = 0; i < args.length - 2; i++) {
 336             switch (args[i]) {
 337                 case "-d":
 338                     outputDir = new File(args[++i]);
 339                     break;
 340                 case "-charset":
 341                     charsetArg = args[++i];
 342                     break;
 343                 case "-docencoding":
 344                     docencodingArg = args[++i];
 345                     break;
 346                 case "-encoding":
 347                     encodingArg = args[++i];
 348                     break;
 349             }
 350         }
 351 
 352         // The following replicates HtmlConfiguration.finishOptionSettings0
 353         // and sets up the charset used to read files.
 354         String cs;
 355         if (docencodingArg == null) {
 356             if (charsetArg == null) {
 357                 cs = (encodingArg == null) ? "UTF-8" : encodingArg;
 358             } else {
 359                 cs = charsetArg;
 360             }
 361         } else {
 362            cs = docencodingArg;
 363         }
 364         try {
 365             charset = Charset.forName(cs);
 366         } catch (UnsupportedCharsetException e) {
 367             charset = Charset.defaultCharset();
 368         }
 369 
 370         out.println("args: " + Arrays.toString(args));
 371 //        log.setOutDir(outputDir);
 372 
 373         outputDirectoryCheck.check(outputDir);
 374 
 375         // This is the sole stream used by javadoc
 376         WriterOutput outOut = new WriterOutput();
 377 
 378         // These are to catch output to System.out and System.err,
 379         // in case these are used instead of the primary streams
 380         StreamOutput sysOut = new StreamOutput(System.out, System::setOut);
 381         StreamOutput sysErr = new StreamOutput(System.err, System::setErr);
 382 
 383         try {
 384             exitCode = jdk.javadoc.internal.tool.Main.execute(args, outOut.pw);
 385         } finally {
 386             outputMap.put(Output.STDOUT, sysOut.close());
 387             outputMap.put(Output.STDERR, sysErr.close());
 388             outputMap.put(Output.OUT, outOut.close());
 389         }
 390 
 391         outputMap.forEach((name, text) -> {
 392             if (!text.isEmpty()) {
 393                 out.println("javadoc " + name + ":");
 394                 out.println(text);
 395             }
 396         });
 397 
 398         if (automaticCheckLinks && exitCode == Exit.OK.code && outputDir.exists()) {
 399             checkLinks();
 400         }
 401     }
 402 
 403     /**
 404      * Set the kind of check for the initial contents of the output directory
 405      * before javadoc is run.
 406      * The filter should return true for files that should <b>not</b> appear.
 407      * @param c the kind of check to perform
 408      */
 409     public void setOutputDirectoryCheck(DirectoryCheck c) {
 410         outputDirectoryCheck = c;
 411     }
 412 
 413     /**
 414      * Set whether or not to perform an automatic call of checkLinks.
 415      */
 416     public void setAutomaticCheckLinks(boolean b) {
 417         automaticCheckLinks = b;
 418     }
 419 
 420     /**
 421      * The exit codes returned by the javadoc tool.
 422      * @see jdk.javadoc.internal.tool.Main.Result
 423      */
 424     public enum Exit {
 425         OK(0),        // Javadoc completed with no errors.
 426         ERROR(1),     // Completed but reported errors.
 427         CMDERR(2),    // Bad command-line arguments
 428         SYSERR(3),    // System error or resource exhaustion.
 429         ABNORMAL(4);  // Javadoc terminated abnormally
 430 
 431         Exit(int code) {
 432             this.code = code;
 433         }
 434 
 435         final int code;
 436 
 437         @Override
 438         public String toString() {
 439             return name() + '(' + code + ')';
 440         }
 441     }
 442 
 443     /**
 444      * Check the exit code of the most recent call of javadoc.
 445      *
 446      * @param expected the exit code that is required for the test
 447      * to pass.
 448      */
 449     public void checkExit(Exit expected) {
 450         checking("check exit code");
 451         if (exitCode == expected.code) {
 452             passed("return code " + exitCode);
 453         } else {
 454             failed("return code " + exitCode +"; expected " + expected);
 455         }
 456     }
 457 
 458     /**
 459      * Check for content in (or not in) the generated output.
 460      * Within the search strings, the newline character \n
 461      * will be translated to the platform newline character sequence.
 462      * @param path a path within the most recent output directory
 463      *  or the name of one of the output buffers, identifying
 464      *  where to look for the search strings.
 465      * @param expectedFound true if all of the search strings are expected
 466      *  to be found, or false if the file is not expected to be found
 467      * @param strings the strings to be searched for
 468      */
 469     public void checkFileAndOutput(String path, boolean expectedFound, String... strings) {
 470         if (expectedFound) {
 471             checkOutput(path, true, strings);
 472         } else {
 473             checkFiles(false, path);
 474         }
 475     }
 476 
 477     /**
 478      * Check for content in (or not in) the generated output.
 479      * Within the search strings, the newline character \n
 480      * will be translated to the platform newline character sequence.
 481      * @param path a path within the most recent output directory, identifying
 482      *  where to look for the search strings.
 483      * @param expectedFound true if all of the search strings are expected
 484      *  to be found, or false if all of the strings are expected to be
 485      *  not found
 486      * @param strings the strings to be searched for
 487      */
 488     public void checkOutput(String path, boolean expectedFound, String... strings) {
 489         // Read contents of file
 490         try {
 491             String fileString = readFile(outputDir, path);
 492             checkOutput(new File(outputDir, path).getPath(), fileString, expectedFound, strings);
 493         } catch (Error e) {
 494             checking("Read file");
 495             failed("Error reading file: " + e);
 496         }
 497     }
 498 
 499     /**
 500      * Check for content in (or not in) the one of the output streams written by
 501      * javadoc. Within the search strings, the newline character \n
 502      * will be translated to the platform newline character sequence.
 503      * @param output the output stream to check
 504      * @param expectedFound true if all of the search strings are expected
 505      *  to be found, or false if all of the strings are expected to be
 506      *  not found
 507      * @param strings the strings to be searched for
 508      */
 509     public void checkOutput(Output output, boolean expectedFound, String... strings) {
 510         checkOutput(output.toString(), outputMap.get(output), expectedFound, strings);
 511     }
 512 
 513     // NOTE: path may be the name of an Output stream as well as a file path
 514     private void checkOutput(String path, String fileString, boolean expectedFound, String... strings) {
 515         for (String stringToFind : strings) {
 516 //            log.logCheckOutput(path, expectedFound, stringToFind);
 517             checking("checkOutput");
 518             // Find string in file's contents
 519             boolean isFound = findString(fileString, stringToFind);
 520             if (isFound == expectedFound) {
 521                 passed(path + ": following text " + (isFound ? "found:" : "not found:") + "\n"
 522                         + stringToFind);
 523             } else {
 524                 failed(path + ": following text " + (isFound ? "found:" : "not found:") + "\n"
 525                         + stringToFind + '\n' +
 526                         "found \n" +
 527                         fileString);
 528             }
 529         }
 530     }
 531 
 532     public void checkLinks() {
 533         checking("Check links");
 534         LinkChecker c = new LinkChecker(out, this::readFile);
 535         try {
 536             c.checkDirectory(outputDir.toPath());
 537             c.report();
 538             int errors = c.getErrorCount();
 539             if (errors == 0) {
 540                 passed("Links are OK");
 541             } else {
 542                 failed(errors + " errors found when checking links");
 543             }
 544         } catch (IOException e) {
 545             failed("exception thrown when reading files: " + e);
 546         }
 547     }
 548 
 549     /**
 550      * Get the content of the one of the output streams written by javadoc.
 551      * @param output the name of the output stream
 552      * @return the content of the output stream
 553      */
 554     public String getOutput(Output output) {
 555         return outputMap.get(output);
 556     }
 557 
 558     /**
 559      * Get the content of the one of the output streams written by javadoc.
 560      * @param output the name of the output stream
 561      * @return the content of the output stream, as a line of lines
 562      */
 563     public List<String> getOutputLines(Output output) {
 564         String text = outputMap.get(output);
 565         return (text == null) ? Collections.emptyList() : Arrays.asList(text.split(NL));
 566     }
 567 
 568     /**
 569      * Check for files in (or not in) the generated output.
 570      * @param expectedFound true if all of the files are expected
 571      *  to be found, or false if all of the files are expected to be
 572      *  not found
 573      * @param paths the files to check, within the most recent output directory.
 574      * */
 575     public void checkFiles(boolean expectedFound, String... paths) {
 576         checkFiles(expectedFound, Arrays.asList(paths));
 577     }
 578 
 579     /**
 580      * Check for files in (or not in) the generated output.
 581      * @param expectedFound true if all of the files are expected
 582      *  to be found, or false if all of the files are expected to be
 583      *  not found
 584      * @param paths the files to check, within the most recent output directory.
 585      * */
 586     public void checkFiles(boolean expectedFound, Collection<String> paths) {
 587         for (String path: paths) {
 588 //            log.logCheckFile(path, expectedFound);
 589             checking("checkFile");
 590             File file = new File(outputDir, path);
 591             boolean isFound = file.exists();
 592             if (isFound == expectedFound) {
 593                 passed(file, "file " + (isFound ? "found:" : "not found:") + "\n");
 594             } else {
 595                 failed(file, "file " + (isFound ? "found:" : "not found:") + "\n");
 596             }
 597         }
 598     }
 599 
 600     /**
 601      * Check that a series of strings are found in order in a file in
 602      * the generated output.
 603      * @param path the file to check
 604      * @param strings  the strings whose order to check
 605      */
 606     public void checkOrder(String path, String... strings) {
 607         File file = new File(outputDir, path);
 608         String fileString = readOutputFile(path);
 609         int prevIndex = -1;
 610         for (String s : strings) {
 611             s = s.replace("\n", NL); // normalize new lines
 612             int currentIndex = fileString.indexOf(s, prevIndex + 1);
 613             checking("file: " + file + ": " + s + " at index " + currentIndex);
 614             if (currentIndex == -1) {
 615                 failed(file, s + " not found.");
 616                 continue;
 617             }
 618             if (currentIndex > prevIndex) {
 619                 passed(file, s + " is in the correct order");
 620             } else {
 621                 failed(file, s + " is in the wrong order.");
 622             }
 623             prevIndex = currentIndex;
 624         }
 625     }
 626 
 627     /**
 628      * Ensures that a series of strings appear only once, in the generated output,
 629      * noting that, this test does not exhaustively check for all other possible
 630      * duplicates once one is found.
 631      * @param path the file to check
 632      * @param strings ensure each are unique
 633      */
 634     public void checkUnique(String path, String... strings) {
 635         File file = new File(outputDir, path);
 636         String fileString = readOutputFile(path);
 637         for (String s : strings) {
 638             int currentIndex = fileString.indexOf(s);
 639             checking(s + " at index " + currentIndex);
 640             if (currentIndex == -1) {
 641                 failed(file, s + " not found.");
 642                 continue;
 643             }
 644             int nextindex = fileString.indexOf(s, currentIndex + s.length());
 645             if (nextindex == -1) {
 646                 passed(file, s + " is unique");
 647             } else {
 648                 failed(file, s + " is not unique, found at " + nextindex);
 649             }
 650         }
 651     }
 652 
 653     /**
 654      * Compare a set of files in each of two directories.
 655      *
 656      * @param baseDir1 the directory containing the first set of files
 657      * @param baseDir2 the directory containing the second set of files
 658      * @param files the set of files to be compared
 659      */
 660     public void diff(String baseDir1, String baseDir2, String... files) {
 661         File bd1 = new File(baseDir1);
 662         File bd2 = new File(baseDir2);
 663         for (String file : files) {
 664             diff(bd1, bd2, file);
 665         }
 666     }
 667 
 668     /**
 669      * A utility to copy a directory from one place to another.
 670      *
 671      * @param targetDir the directory to copy.
 672      * @param destDir the destination to copy the directory to.
 673      */
 674     // TODO: convert to using java.nio.Files.walkFileTree
 675     public void copyDir(String targetDir, String destDir) {
 676         try {
 677             File targetDirObj = new File(targetDir);
 678             File destDirParentObj = new File(destDir);
 679             File destDirObj = new File(destDirParentObj, targetDirObj.getName());
 680             if (! destDirParentObj.exists()) {
 681                 destDirParentObj.mkdir();
 682             }
 683             if (! destDirObj.exists()) {
 684                 destDirObj.mkdir();
 685             }
 686             String[] files = targetDirObj.list();
 687             for (String file : files) {
 688                 File srcFile = new File(targetDirObj, file);
 689                 File destFile = new File(destDirObj, file);
 690                 if (srcFile.isFile()) {
 691                     out.println("Copying " + srcFile + " to " + destFile);
 692                     copyFile(destFile, srcFile);
 693                 } else if(srcFile.isDirectory()) {
 694                     copyDir(srcFile.getAbsolutePath(), destDirObj.getAbsolutePath());
 695                 }
 696             }
 697         } catch (IOException exc) {
 698             throw new Error("Could not copy " + targetDir + " to " + destDir);
 699         }
 700     }
 701 
 702     /**
 703      * Copy source file to destination file.
 704      *
 705      * @param destfile the destination file
 706      * @param srcfile the source file
 707      * @throws IOException
 708      */
 709     public void copyFile(File destfile, File srcfile) throws IOException {
 710         Files.copy(srcfile.toPath(), destfile.toPath());
 711     }
 712 
 713     /**
 714      * Read a file from the output directory.
 715      *
 716      * @param fileName  the name of the file to read
 717      * @return          the file in string format
 718      */
 719     public String readOutputFile(String fileName) throws Error {
 720         return readFile(outputDir, fileName);
 721     }
 722 
 723     protected String readFile(String fileName) throws Error {
 724         return readFile(outputDir, fileName);
 725     }
 726 
 727     protected String readFile(String baseDir, String fileName) throws Error {
 728         return readFile(new File(baseDir), fileName);
 729     }
 730 
 731     private String readFile(Path file) {
 732         File baseDir;
 733         if (file.startsWith(outputDir.toPath())) {
 734             baseDir = outputDir;
 735         } else if (file.startsWith(currDir)) {
 736             baseDir = currDir.toFile();
 737         } else {
 738             baseDir = file.getParent().toFile();
 739         }
 740         String fileName = baseDir.toPath().relativize(file).toString();
 741         return readFile(baseDir, fileName);
 742     }
 743 
 744     /**
 745      * Read the file and return it as a string.
 746      *
 747      * @param baseDir   the directory in which to locate the file
 748      * @param fileName  the name of the file to read
 749      * @return          the file in string format
 750      */
 751     private String readFile(File baseDir, String fileName) throws Error {
 752         if (!Objects.equals(fileContentCacheCharset, charset)) {
 753             fileContentCache.clear();
 754             fileContentCacheCharset = charset;
 755         }
 756         try {
 757             File file = new File(baseDir, fileName);
 758             SoftReference<String> ref = fileContentCache.get(file);
 759             String content = (ref == null) ? null : ref.get();
 760             if (content != null)
 761                 return content;
 762 
 763             // charset defaults to a value inferred from latest javadoc run
 764             content = new String(Files.readAllBytes(file.toPath()), charset);
 765             fileContentCache.put(file, new SoftReference<>(content));
 766             return content;
 767         } catch (FileNotFoundException e) {
 768             throw new Error("File not found: " + fileName + ": " + e);
 769         } catch (IOException e) {
 770             throw new Error("Error reading file: " + fileName + ": " + e);
 771         }
 772     }
 773 
 774     protected void checking(String message) {
 775         numTestsRun++;
 776         javadocTestNum++;
 777         print("Starting subtest " + javadocRunNum + "." + javadocTestNum, message);
 778     }
 779 
 780     protected void passed(File file, String message) {
 781         passed(file + ": " + message);
 782     }
 783 
 784     protected void passed(String message) {
 785         numTestsPassed++;
 786         print("Passed", message);
 787         out.println();
 788     }
 789 
 790     protected void failed(File file, String message) {
 791         failed(file + ": " + message);
 792     }
 793 
 794     protected void failed(String message) {
 795         print("FAILED", message);
 796         StackWalker.getInstance().walk(s -> {
 797             s.dropWhile(f -> f.getMethodName().equals("failed"))
 798                     .takeWhile(f -> !f.getMethodName().equals("runTests"))
 799                     .forEach(f -> out.println("        at "
 800                             + f.getClassName() + "." + f.getMethodName()
 801                             + "(" + f.getFileName() + ":" + f.getLineNumber() + ")"));
 802             return null;
 803         });
 804         out.println();
 805     }
 806 
 807     private void print(String prefix, String message) {
 808         if (message.isEmpty())
 809             out.println(prefix);
 810         else {
 811             out.print(prefix);
 812             out.print(": ");
 813             out.print(message.replace("\n", NL));
 814             if (!(message.endsWith("\n") || message.endsWith(NL))) {
 815                 out.println();
 816             }
 817         }
 818     }
 819 
 820     /**
 821      * Print a summary of the test results.
 822      */
 823     protected void printSummary() {
 824         String javadocRuns = (javadocRunNum <= 1) ? ""
 825                 : ", in " + javadocRunNum + " runs of javadoc";
 826 
 827         if (numTestsRun != 0 && numTestsPassed == numTestsRun) {
 828             // Test passed
 829             out.println();
 830             out.println("All " + numTestsPassed + " subtests passed" + javadocRuns);
 831         } else {
 832             // Test failed
 833             throw new Error((numTestsRun - numTestsPassed)
 834                     + " of " + (numTestsRun)
 835                     + " subtests failed"
 836                     + javadocRuns);
 837         }
 838     }
 839 
 840     /**
 841      * Search for the string in the given file and return true
 842      * if the string was found.
 843      *
 844      * @param fileString    the contents of the file to search through
 845      * @param stringToFind  the string to search for
 846      * @return              true if the string was found
 847      */
 848     private boolean findString(String fileString, String stringToFind) {
 849         // javadoc (should) always use the platform newline sequence,
 850         // but in the strings to find it is more convenient to use the Java
 851         // newline character. So we translate \n to NL before we search.
 852         stringToFind = stringToFind.replace("\n", NL);
 853         return fileString.contains(stringToFind);
 854     }
 855 
 856     /**
 857      * Compare the two given files.
 858      *
 859      * @param baseDir1 the directory in which to locate the first file
 860      * @param baseDir2 the directory in which to locate the second file
 861      * @param file the file to compare in the two base directories
 862      * @param throwErrorIFNoMatch flag to indicate whether or not to throw
 863      * an error if the files do not match.
 864      * @return true if the files are the same and false otherwise.
 865      */
 866     private void diff(File baseDir1, File baseDir2, String file) {
 867         String file1Contents = readFile(baseDir1, file);
 868         String file2Contents = readFile(baseDir2, file);
 869         checking("diff " + new File(baseDir1, file) + ", " + new File(baseDir2, file));
 870         if (file1Contents.trim().compareTo(file2Contents.trim()) == 0) {
 871             passed("files are equal");
 872         } else {
 873             failed("files differ");
 874         }
 875     }
 876 
 877     /**
 878      * Utility class to simplify the handling of temporarily setting a
 879      * new stream for System.out or System.err.
 880      */
 881     private static class StreamOutput {
 882         // functional interface to set a stream.
 883         private interface Initializer {
 884             void set(PrintStream s);
 885         }
 886 
 887         private final ByteArrayOutputStream baos = new ByteArrayOutputStream();
 888         private final PrintStream ps = new PrintStream(baos);
 889         private final PrintStream prev;
 890         private final Initializer init;
 891 
 892         StreamOutput(PrintStream s, Initializer init) {
 893             prev = s;
 894             init.set(ps);
 895             this.init = init;
 896         }
 897 
 898         String close() {
 899             init.set(prev);
 900             ps.close();
 901             return baos.toString();
 902         }
 903     }
 904 
 905     /**
 906      * Utility class to simplify the handling of creating an in-memory PrintWriter.
 907      */
 908     private static class WriterOutput {
 909         private final StringWriter sw = new StringWriter();
 910         final PrintWriter pw = new PrintWriter(sw);
 911         String close() {
 912             pw.close();
 913             return sw.toString();
 914         }
 915     }
 916 
 917 
 918 //    private final Logger log = new Logger();
 919 
 920     //--------- Logging --------------------------------------------------------
 921     //
 922     // This class writes out the details of calls to checkOutput and checkFile
 923     // in a canonical way, so that the resulting file can be checked against
 924     // similar files from other versions of JavadocTester using the same logging
 925     // facilities.
 926 
 927     static class Logger {
 928         private static final int PREFIX = 40;
 929         private static final int SUFFIX = 20;
 930         private static final int MAX = PREFIX + SUFFIX;
 931         List<String> tests = new ArrayList<>();
 932         String outDir;
 933         String rootDir = rootDir();
 934 
 935         static String rootDir() {
 936             File f = new File(".").getAbsoluteFile();
 937             while (!new File(f, ".hg").exists())
 938                 f = f.getParentFile();
 939             return f.getPath();
 940         }
 941 
 942         void setOutDir(File outDir) {
 943             this.outDir = outDir.getPath();
 944         }
 945 
 946         void logCheckFile(String file, boolean positive) {
 947             // Strip the outdir because that will typically not be the same
 948             if (file.startsWith(outDir + "/"))
 949                 file = file.substring(outDir.length() + 1);
 950             tests.add(file + " " + positive);
 951         }
 952 
 953         void logCheckOutput(String file, boolean positive, String text) {
 954             // Compress the string to be displayed in the log file
 955             String simpleText = text.replaceAll("\\s+", " ").replace(rootDir, "[ROOT]");
 956             if (simpleText.length() > MAX)
 957                 simpleText = simpleText.substring(0, PREFIX)
 958                         + "..." + simpleText.substring(simpleText.length() - SUFFIX);
 959             // Strip the outdir because that will typically not be the same
 960             if (file.startsWith(outDir + "/"))
 961                 file = file.substring(outDir.length() + 1);
 962             // The use of text.hashCode ensure that all of "text" is taken into account
 963             tests.add(file + " " + positive + " " + text.hashCode() + " " + simpleText);
 964         }
 965 
 966         void write() {
 967             // sort the log entries because the subtests may not be executed in the same order
 968             tests.sort((a, b) -> a.compareTo(b));
 969             try (BufferedWriter bw = new BufferedWriter(new FileWriter("tester.log"))) {
 970                 for (String t: tests) {
 971                     bw.write(t);
 972                     bw.newLine();
 973                 }
 974             } catch (IOException e) {
 975                 throw new Error("problem writing log: " + e);
 976             }
 977         }
 978     }
 979 
 980     // Support classes for checkLinks
 981 
 982     /**
 983      * A basic HTML parser. Override the protected methods as needed to get notified
 984      * of significant items in any file that is read.
 985      */
 986     static abstract class HtmlParser {
 987 
 988         protected final PrintStream out;
 989         protected final Function<Path,String> fileReader;
 990 
 991         private Path file;
 992         private StringReader in;
 993         private int ch;
 994         private int lineNumber;
 995         private boolean inScript;
 996         private boolean xml;
 997 
 998         HtmlParser(PrintStream out, Function<Path,String> fileReader) {
 999             this.out = out;
1000             this.fileReader = fileReader;
1001         }
1002 
1003         /**
1004          * Read a file.
1005          * @param file the file to be read
1006          * @throws IOException if an error occurs while reading the file
1007          */
1008         void read(Path file) throws IOException {
1009             try (StringReader r = new StringReader(fileReader.apply(file))) {
1010                 this.file = file;
1011                 this.in = r;
1012 
1013                 startFile(file);
1014                 try {
1015                     lineNumber = 1;
1016                     xml = false;
1017                     nextChar();
1018 
1019                     while (ch != -1) {
1020                         switch (ch) {
1021 
1022                             case '<':
1023                                 html();
1024                                 break;
1025 
1026                             default:
1027                                 nextChar();
1028                         }
1029                     }
1030                 } finally {
1031                     endFile();
1032                 }
1033             } catch (IOException e) {
1034                 error(file, lineNumber, e);
1035             } catch (Throwable t) {
1036                 error(file, lineNumber, t);
1037                 t.printStackTrace(out);
1038             }
1039         }
1040 
1041 
1042         int getLineNumber() {
1043             return lineNumber;
1044         }
1045 
1046         /**
1047          * Called when a file has been opened, before parsing begins.
1048          * This is always the first notification when reading a file.
1049          * This implementation does nothing.
1050          *
1051          * @param file the file
1052          */
1053         protected void startFile(Path file) { }
1054 
1055         /**
1056          * Called when the parser has finished reading a file.
1057          * This is always the last notification when reading a file,
1058          * unless any errors occur while closing the file.
1059          * This implementation does nothing.
1060          */
1061         protected void endFile() { }
1062 
1063         /**
1064          * Called when a doctype declaration is found, at the beginning of the file.
1065          * This implementation does nothing.
1066          * @param s the doctype declaration
1067          */
1068         protected void docType(String s) { }
1069 
1070         /**
1071          * Called when the opening tag of an HTML element is encountered.
1072          * This implementation does nothing.
1073          * @param name the name of the tag
1074          * @param attrs the attribute
1075          * @param selfClosing whether or not this is a self-closing tag
1076          */
1077         protected void startElement(String name, Map<String,String> attrs, boolean selfClosing) { }
1078 
1079         /**
1080          * Called when the closing tag of an HTML tag is encountered.
1081          * This implementation does nothing.
1082          * @param name the name of the tag
1083          */
1084         protected void endElement(String name) { }
1085 
1086         /**
1087          * Called when an error has been encountered.
1088          * @param file the file being read
1089          * @param lineNumber the line number of line containing the error
1090          * @param message a description of the error
1091          */
1092         protected void error(Path file, int lineNumber, String message) {
1093             out.println(file + ":" + lineNumber + ": " + message);
1094         }
1095 
1096         /**
1097          * Called when an exception has been encountered.
1098          * @param file the file being read
1099          * @param lineNumber the line number of the line being read when the exception was found
1100          * @param t the exception
1101          */
1102         protected void error(Path file, int lineNumber, Throwable t) {
1103             out.println(file + ":" + lineNumber + ": " + t);
1104         }
1105 
1106         private void nextChar() throws IOException {
1107             ch = in.read();
1108             if (ch == '\n')
1109                 lineNumber++;
1110         }
1111 
1112         /**
1113          * Read the start or end of an HTML tag, or an HTML comment
1114          * {@literal <identifier attrs> } or {@literal </identifier> }
1115          * @throws java.io.IOException if there is a problem reading the file
1116          */
1117         private void html() throws IOException {
1118             nextChar();
1119             if (isIdentifierStart((char) ch)) {
1120                 String name = readIdentifier().toLowerCase(Locale.US);
1121                 Map<String,String> attrs = htmlAttrs();
1122                 if (attrs != null) {
1123                     boolean selfClosing = false;
1124                     if (ch == '/') {
1125                         nextChar();
1126                         selfClosing = true;
1127                     }
1128                     if (ch == '>') {
1129                         nextChar();
1130                         startElement(name, attrs, selfClosing);
1131                         if (name.equals("script")) {
1132                             inScript = true;
1133                         }
1134                         return;
1135                     }
1136                 }
1137             } else if (ch == '/') {
1138                 nextChar();
1139                 if (isIdentifierStart((char) ch)) {
1140                     String name = readIdentifier().toLowerCase(Locale.US);
1141                     skipWhitespace();
1142                     if (ch == '>') {
1143                         nextChar();
1144                         endElement(name);
1145                         if (name.equals("script")) {
1146                             inScript = false;
1147                         }
1148                         return;
1149                     }
1150                 }
1151             } else if (ch == '!') {
1152                 nextChar();
1153                 if (ch == '-') {
1154                     nextChar();
1155                     if (ch == '-') {
1156                         nextChar();
1157                         while (ch != -1) {
1158                             int dash = 0;
1159                             while (ch == '-') {
1160                                 dash++;
1161                                 nextChar();
1162                             }
1163                             // Strictly speaking, a comment should not contain "--"
1164                             // so dash > 2 is an error, dash == 2 implies ch == '>'
1165                             // See http://www.w3.org/TR/html-markup/syntax.html#syntax-comments
1166                             // for more details.
1167                             if (dash >= 2 && ch == '>') {
1168                                 nextChar();
1169                                 return;
1170                             }
1171 
1172                             nextChar();
1173                         }
1174                     }
1175                 } else if (ch == '[') {
1176                     nextChar();
1177                     if (ch == 'C') {
1178                         nextChar();
1179                         if (ch == 'D') {
1180                             nextChar();
1181                             if (ch == 'A') {
1182                                 nextChar();
1183                                 if (ch == 'T') {
1184                                     nextChar();
1185                                     if (ch == 'A') {
1186                                         nextChar();
1187                                         if (ch == '[') {
1188                                             while (true) {
1189                                                 nextChar();
1190                                                 if (ch == ']') {
1191                                                     nextChar();
1192                                                     if (ch == ']') {
1193                                                         nextChar();
1194                                                         if (ch == '>') {
1195                                                             nextChar();
1196                                                             return;
1197                                                         }
1198                                                     }
1199                                                 }
1200                                             }
1201 
1202                                         }
1203                                     }
1204                                 }
1205                             }
1206                         }
1207                     }
1208                 } else {
1209                     StringBuilder sb = new StringBuilder();
1210                     while (ch != -1 && ch != '>') {
1211                         sb.append((char) ch);
1212                         nextChar();
1213                     }
1214                     Pattern p = Pattern.compile("(?is)doctype\\s+html\\s?.*");
1215                     String s = sb.toString();
1216                     if (p.matcher(s).matches()) {
1217                         docType(s);
1218                         return;
1219                     }
1220                 }
1221             } else if (ch == '?') {
1222                 nextChar();
1223                 if (ch == 'x') {
1224                     nextChar();
1225                     if (ch == 'm') {
1226                         nextChar();
1227                         if (ch == 'l') {
1228                             Map<String,String> attrs = htmlAttrs();
1229                             if (ch == '?') {
1230                                 nextChar();
1231                                 if (ch == '>') {
1232                                     nextChar();
1233                                     xml = true;
1234                                     return;
1235                                 }
1236                             }
1237                         }
1238                     }
1239 
1240                 }
1241             }
1242 
1243             if (!inScript) {
1244                 error(file, lineNumber, "bad html");
1245             }
1246         }
1247 
1248         /**
1249          * Read a series of HTML attributes, terminated by {@literal > }.
1250          * Each attribute is of the form {@literal identifier[=value] }.
1251          * "value" may be unquoted, single-quoted, or double-quoted.
1252          */
1253         private Map<String,String> htmlAttrs() throws IOException {
1254             Map<String, String> map = new LinkedHashMap<>();
1255             skipWhitespace();
1256 
1257             loop:
1258             while (isIdentifierStart((char) ch)) {
1259                 String name = readAttributeName().toLowerCase(Locale.US);
1260                 skipWhitespace();
1261                 String value = null;
1262                 if (ch == '=') {
1263                     nextChar();
1264                     skipWhitespace();
1265                     if (ch == '\'' || ch == '"') {
1266                         char quote = (char) ch;
1267                         nextChar();
1268                         StringBuilder sb = new StringBuilder();
1269                         while (ch != -1 && ch != quote) {
1270                             sb.append((char) ch);
1271                             nextChar();
1272                         }
1273                         value = sb.toString() // hack to replace common entities
1274                                 .replace("&lt;", "<")
1275                                 .replace("&gt;", ">")
1276                                 .replace("&amp;", "&");
1277                         nextChar();
1278                     } else {
1279                         StringBuilder sb = new StringBuilder();
1280                         while (ch != -1 && !isUnquotedAttrValueTerminator((char) ch)) {
1281                             sb.append((char) ch);
1282                             nextChar();
1283                         }
1284                         value = sb.toString();
1285                     }
1286                     skipWhitespace();
1287                 }
1288                 map.put(name, value);
1289             }
1290 
1291             return map;
1292         }
1293 
1294         private boolean isIdentifierStart(char ch) {
1295             return Character.isUnicodeIdentifierStart(ch);
1296         }
1297 
1298         private String readIdentifier() throws IOException {
1299             StringBuilder sb = new StringBuilder();
1300             sb.append((char) ch);
1301             nextChar();
1302             while (ch != -1 && Character.isUnicodeIdentifierPart(ch)) {
1303                 sb.append((char) ch);
1304                 nextChar();
1305             }
1306             return sb.toString();
1307         }
1308 
1309         private String readAttributeName() throws IOException {
1310             StringBuilder sb = new StringBuilder();
1311             sb.append((char) ch);
1312             nextChar();
1313             while (ch != -1 && Character.isUnicodeIdentifierPart(ch)
1314                     || ch == '-'
1315                     || xml && ch == ':') {
1316                 sb.append((char) ch);
1317                 nextChar();
1318             }
1319             return sb.toString();
1320         }
1321 
1322         private boolean isWhitespace(char ch) {
1323             return Character.isWhitespace(ch);
1324         }
1325 
1326         private void skipWhitespace() throws IOException {
1327             while (isWhitespace((char) ch)) {
1328                 nextChar();
1329             }
1330         }
1331 
1332         private boolean isUnquotedAttrValueTerminator(char ch) {
1333             switch (ch) {
1334                 case '\f': case '\n': case '\r': case '\t':
1335                 case ' ':
1336                 case '"': case '\'': case '`':
1337                 case '=': case '<': case '>':
1338                     return true;
1339                 default:
1340                     return false;
1341             }
1342         }
1343     }
1344 
1345     /**
1346      * A class to check the links in a set of HTML files.
1347      */
1348     static class LinkChecker extends HtmlParser {
1349         private final Map<Path, IDTable> allFiles;
1350         private final Map<URI, IDTable> allURIs;
1351 
1352         private int files;
1353         private int links;
1354         private int badSchemes;
1355         private int duplicateIds;
1356         private int missingIds;
1357 
1358         private Path currFile;
1359         private IDTable currTable;
1360         private boolean html5;
1361         private boolean xml;
1362 
1363         private int errors;
1364 
1365         LinkChecker(PrintStream out, Function<Path,String> fileReader) {
1366             super(out, fileReader);
1367             allFiles = new HashMap<>();
1368             allURIs = new HashMap<>();
1369         }
1370 
1371         void checkDirectory(Path dir) throws IOException {
1372             checkFiles(List.of(dir), false, Collections.emptySet());
1373         }
1374 
1375         void checkFiles(List<Path> files, boolean skipSubdirs, Set<Path> excludeFiles) throws IOException {
1376             for (Path file : files) {
1377                 Files.walkFileTree(file, new SimpleFileVisitor<Path>() {
1378                     int depth = 0;
1379 
1380                     @Override
1381                     public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
1382                         if ((skipSubdirs && depth > 0) || excludeFiles.contains(dir)) {
1383                             return FileVisitResult.SKIP_SUBTREE;
1384                         }
1385                         depth++;
1386                         return FileVisitResult.CONTINUE;
1387                     }
1388 
1389                     @Override
1390                     public FileVisitResult visitFile(Path p, BasicFileAttributes attrs) {
1391                         if (excludeFiles.contains(p)) {
1392                             return FileVisitResult.CONTINUE;
1393                         }
1394 
1395                         if (Files.isRegularFile(p) && p.getFileName().toString().endsWith(".html")) {
1396                             checkFile(p);
1397                         }
1398                         return FileVisitResult.CONTINUE;
1399                     }
1400 
1401                     @Override
1402                     public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException {
1403                         depth--;
1404                         return super.postVisitDirectory(dir, e);
1405                     }
1406                 });
1407             }
1408         }
1409 
1410         void checkFile(Path file) {
1411             try {
1412                 read(file);
1413             } catch (IOException e) {
1414                 error(file, 0, e);
1415             }
1416         }
1417 
1418         int getErrorCount() {
1419             return errors;
1420         }
1421 
1422         public void report() {
1423             List<Path> missingFiles = getMissingFiles();
1424             if (!missingFiles.isEmpty()) {
1425                 report("Missing files: (" + missingFiles.size() + ")");
1426                 missingFiles.stream()
1427                         .sorted()
1428                         .forEach(this::reportMissingFile);
1429 
1430             }
1431 
1432             if (!allURIs.isEmpty()) {
1433                 report(false, "External URLs:");
1434                 allURIs.keySet().stream()
1435                         .sorted(new URIComparator())
1436                         .forEach(uri -> report(false, "  %s", uri.toString()));
1437             }
1438 
1439             int anchors = 0;
1440             for (IDTable t : allFiles.values()) {
1441                 anchors += t.map.values().stream()
1442                         .filter(e -> !e.getReferences().isEmpty())
1443                         .count();
1444             }
1445             for (IDTable t : allURIs.values()) {
1446                 anchors += t.map.values().stream()
1447                         .filter(e -> !e.references.isEmpty())
1448                         .count();
1449             }
1450 
1451             report(false, "Checked " + files + " files.");
1452             report(false, "Found " + links + " references to " + anchors + " anchors "
1453                     + "in " + allFiles.size() + " files and " + allURIs.size() + " other URIs.");
1454             report(!missingFiles.isEmpty(),   "%6d missing files", missingFiles.size());
1455             report(duplicateIds > 0, "%6d duplicate ids", duplicateIds);
1456             report(missingIds > 0,   "%6d missing ids", missingIds);
1457 
1458             Map<String, Integer> schemeCounts = new TreeMap<>();
1459             Map<String, Integer> hostCounts = new TreeMap<>(new HostComparator());
1460             for (URI uri : allURIs.keySet()) {
1461                 String scheme = uri.getScheme();
1462                 if (scheme != null) {
1463                     schemeCounts.put(scheme, schemeCounts.computeIfAbsent(scheme, s -> 0) + 1);
1464                 }
1465                 String host = uri.getHost();
1466                 if (host != null) {
1467                     hostCounts.put(host, hostCounts.computeIfAbsent(host, h -> 0) + 1);
1468                 }
1469             }
1470 
1471             if (schemeCounts.size() > 0) {
1472                 report(false, "Schemes");
1473                 schemeCounts.forEach((s, n) -> report(!isSchemeOK(s), "%6d %s", n, s));
1474             }
1475 
1476             if (hostCounts.size() > 0) {
1477                 report(false, "Hosts");
1478                 hostCounts.forEach((h, n) -> report(false, "%6d %s", n, h));
1479             }
1480         }
1481 
1482         private void report(String message, Object... args) {
1483             out.println(String.format(message, args));
1484         }
1485 
1486         private void report(boolean highlight, String message, Object... args) {
1487             out.print(highlight ? "* " : "  ");
1488             out.println(String.format(message, args));
1489         }
1490 
1491         private void reportMissingFile(Path file) {
1492             report("%s", relativePath(file));
1493             IDTable table = allFiles.get(file);
1494             Set<Path> refs = new TreeSet<>();
1495             for (ID id : table.map.values()) {
1496                 if (id.references != null) {
1497                     for (Position p : id.references) {
1498                         refs.add(p.path);
1499                     }
1500                 }
1501             }
1502             int n = 0;
1503             int MAX_REFS = 10;
1504             for (Path ref : refs) {
1505                 report("    in " + relativePath(ref));
1506                 if (++n == MAX_REFS) {
1507                     report("    ... and %d more", refs.size() - n);
1508                     break;
1509                 }
1510             }
1511         }
1512 
1513         @Override
1514         public void startFile(Path path) {
1515             currFile = path.toAbsolutePath().normalize();
1516             currTable = allFiles.computeIfAbsent(currFile, p -> new IDTable(p));
1517             html5 = false;
1518             files++;
1519         }
1520 
1521         @Override
1522         public void endFile() {
1523             currTable.check();
1524         }
1525 
1526         @Override
1527         public void docType(String doctype) {
1528             html5 = doctype.matches("(?i)<\\?doctype\\s+html>");
1529         }
1530 
1531         @Override @SuppressWarnings("fallthrough")
1532         public void startElement(String name, Map<String, String> attrs, boolean selfClosing) {
1533             int line = getLineNumber();
1534             switch (name) {
1535                 case "a":
1536                     String nameAttr = html5 ? null : attrs.get("name");
1537                     if (nameAttr != null) {
1538                         foundAnchor(line, nameAttr);
1539                     }
1540                     // fallthrough
1541                 case "link":
1542                     String href = attrs.get("href");
1543                     if (href != null) {
1544                         foundReference(line, href);
1545                     }
1546                     break;
1547             }
1548 
1549             String idAttr = attrs.get("id");
1550             if (idAttr != null) {
1551                 foundAnchor(line, idAttr);
1552             }
1553         }
1554 
1555         @Override
1556         public void endElement(String name) { }
1557 
1558         private void foundAnchor(int line, String name) {
1559             currTable.addID(line, name);
1560         }
1561 
1562         private void foundReference(int line, String ref) {
1563             links++;
1564             try {
1565                 URI uri = new URI(ref);
1566                 if (uri.isAbsolute()) {
1567                     foundReference(line, uri);
1568                 } else {
1569                     Path p;
1570                     String uriPath = uri.getPath();
1571                     if (uriPath == null || uriPath.isEmpty()) {
1572                         p = currFile;
1573                     } else {
1574                         p = currFile.getParent().resolve(uriPath).normalize();
1575                     }
1576                     foundReference(line, p, uri.getFragment());
1577                 }
1578             } catch (URISyntaxException e) {
1579                 error(currFile, line, "invalid URI: " + e);
1580             }
1581         }
1582 
1583         private void foundReference(int line, Path p, String fragment) {
1584             IDTable t = allFiles.computeIfAbsent(p, key -> new IDTable(key));
1585             t.addReference(fragment, currFile, line);
1586         }
1587 
1588         private void foundReference(int line, URI uri) {
1589             if (!isSchemeOK(uri.getScheme())) {
1590                 error(currFile, line, "bad scheme in URI");
1591                 badSchemes++;
1592             }
1593 
1594             String fragment = uri.getFragment();
1595             try {
1596                 URI noFrag = new URI(uri.toString().replaceAll("#\\Q" + fragment + "\\E$", ""));
1597                 IDTable t = allURIs.computeIfAbsent(noFrag, key -> new IDTable(key.toString()));
1598                 t.addReference(fragment, currFile, line);
1599             } catch (URISyntaxException e) {
1600                 throw new Error(e);
1601             }
1602         }
1603 
1604         private boolean isSchemeOK(String uriScheme) {
1605             if (uriScheme == null) {
1606                 return true;
1607             }
1608 
1609             switch (uriScheme) {
1610                 case "file":
1611                 case "ftp":
1612                 case "http":
1613                 case "https":
1614                 case "javascript":
1615                 case "mailto":
1616                     return true;
1617 
1618                 default:
1619                     return false;
1620             }
1621         }
1622 
1623         private List<Path> getMissingFiles() {
1624             return allFiles.entrySet().stream()
1625                     .filter(e -> !Files.exists(e.getKey()))
1626                     .map(e -> e.getKey())
1627                     .collect(Collectors.toList());
1628         }
1629 
1630         @Override
1631         protected void error(Path file, int lineNumber, String message) {
1632             super.error(relativePath(file), lineNumber, message);
1633             errors++;
1634         }
1635 
1636         @Override
1637         protected void error(Path file, int lineNumber, Throwable t) {
1638             super.error(relativePath(file), lineNumber, t);
1639             errors++;
1640         }
1641 
1642         private Path relativePath(Path path) {
1643             return path.startsWith(currDir) ? currDir.relativize(path) : path;
1644         }
1645 
1646         /**
1647          * A position in a file, as identified by a file name and line number.
1648          */
1649         static class Position implements Comparable<Position> {
1650             Path path;
1651             int line;
1652 
1653             Position(Path path, int line) {
1654                 this.path = path;
1655                 this.line = line;
1656             }
1657 
1658             @Override
1659             public int compareTo(Position o) {
1660                 int v = path.compareTo(o.path);
1661                 return v != 0 ? v : Integer.compare(line, o.line);
1662             }
1663 
1664             @Override
1665             public boolean equals(Object obj) {
1666                 if (this == obj) {
1667                     return true;
1668                 } else if (obj == null || getClass() != obj.getClass()) {
1669                     return false;
1670                 } else {
1671                     final Position other = (Position) obj;
1672                     return Objects.equals(this.path, other.path)
1673                             && this.line == other.line;
1674                 }
1675             }
1676 
1677             @Override
1678             public int hashCode() {
1679                 return Objects.hashCode(path) * 37 + line;
1680             }
1681         }
1682 
1683         /**
1684          * Infor for an ID within an HTML file, and a set of positions that reference it.
1685          */
1686         static class ID {
1687             boolean declared;
1688             Set<Position> references;
1689 
1690             Set<Position> getReferences() {
1691                 return (references) == null ? Collections.emptySet() : references;
1692             }
1693         }
1694 
1695         /**
1696          * A table for the set of IDs in an HTML file.
1697          */
1698         class IDTable {
1699             private String name;
1700             private boolean checked;
1701             private final Map<String, ID> map = new HashMap<>();
1702 
1703             IDTable(Path p) {
1704                 this(relativePath(p).toString());
1705             }
1706 
1707             IDTable(String name) {
1708                 this.name = name;
1709             }
1710 
1711             void addID(int line, String name) {
1712                 if (checked) {
1713                     throw new IllegalStateException("Adding ID after file has been read");
1714                 }
1715                 Objects.requireNonNull(name);
1716                 ID id = map.computeIfAbsent(name, x -> new ID());
1717                 if (id.declared) {
1718                     error(currFile, line, "name already declared: " + name);
1719                     duplicateIds++;
1720                 } else {
1721                     id.declared = true;
1722                 }
1723             }
1724 
1725             void addReference(String name, Path from, int line) {
1726                 if (checked) {
1727                     if (name != null) {
1728                         ID id = map.get(name);
1729                         if (id == null || !id.declared) {
1730                             error(from, line, "id not found: " + this.name + "#" + name);
1731                         }
1732                     }
1733                 } else {
1734                     ID id = map.computeIfAbsent(name, x -> new ID());
1735                     if (id.references == null) {
1736                         id.references = new TreeSet<>();
1737                     }
1738                     id.references.add(new Position(from, line));
1739                 }
1740             }
1741 
1742             void check() {
1743                 map.forEach((name, id) -> {
1744                     if (name != null && !id.declared) {
1745                         //log.error(currFile, 0, "id not declared: " + name);
1746                         for (Position ref : id.references) {
1747                             error(ref.path, ref.line, "id not found: " + this.name + "#" + name);
1748                         }
1749                         missingIds++;
1750                     }
1751                 });
1752                 checked = true;
1753             }
1754         }
1755 
1756         static class URIComparator implements Comparator<URI> {
1757             final HostComparator hostComparator = new HostComparator();
1758 
1759             @Override
1760             public int compare(URI o1, URI o2) {
1761                 if (o1.isOpaque() || o2.isOpaque()) {
1762                     return o1.compareTo(o2);
1763                 }
1764                 String h1 = o1.getHost();
1765                 String h2 = o2.getHost();
1766                 String s1 = o1.getScheme();
1767                 String s2 = o2.getScheme();
1768                 if (h1 == null || h1.isEmpty() || s1 == null || s1.isEmpty()
1769                         || h2 == null || h2.isEmpty() || s2 == null || s2.isEmpty()) {
1770                     return o1.compareTo(o2);
1771                 }
1772                 int v = hostComparator.compare(h1, h2);
1773                 if (v != 0) {
1774                     return v;
1775                 }
1776                 v = s1.compareTo(s2);
1777                 if (v != 0) {
1778                     return v;
1779                 }
1780                 return o1.compareTo(o2);
1781             }
1782         }
1783 
1784         static class HostComparator implements Comparator<String> {
1785             @Override
1786             public int compare(String h1, String h2) {
1787                 List<String> l1 = new ArrayList<>(Arrays.asList(h1.split("\\.")));
1788                 Collections.reverse(l1);
1789                 String r1 = String.join(".", l1);
1790                 List<String> l2 = new ArrayList<>(Arrays.asList(h2.split("\\.")));
1791                 Collections.reverse(l2);
1792                 String r2 = String.join(".", l2);
1793                 return r1.compareTo(r2);
1794             }
1795         }
1796 
1797     }
1798 }