1 /*
   2  * Copyright (c) 2002, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 
  24 import java.io.BufferedReader;
  25 import java.io.BufferedWriter;
  26 import java.io.ByteArrayOutputStream;
  27 import java.io.File;
  28 import java.io.FileNotFoundException;
  29 import java.io.FileWriter;
  30 import java.io.FilenameFilter;
  31 import java.io.InputStreamReader;
  32 import java.io.IOException;
  33 import java.io.PrintStream;
  34 import java.io.PrintWriter;
  35 import java.io.StringReader;
  36 import java.io.StringWriter;
  37 import java.lang.annotation.Annotation;
  38 import java.lang.annotation.Retention;
  39 import java.lang.annotation.RetentionPolicy;
  40 import java.lang.ref.SoftReference;
  41 import java.lang.reflect.InvocationTargetException;
  42 import java.lang.reflect.Method;
  43 import java.net.URI;
  44 import java.net.URISyntaxException;
  45 import java.nio.charset.Charset;
  46 import java.nio.charset.CharsetDecoder;
  47 import java.nio.charset.CodingErrorAction;
  48 import java.nio.charset.UnsupportedCharsetException;
  49 import java.nio.file.FileVisitResult;
  50 import java.nio.file.Files;
  51 import java.nio.file.Path;
  52 import java.nio.file.Paths;
  53 import java.nio.file.SimpleFileVisitor;
  54 import java.nio.file.attribute.BasicFileAttributes;
  55 import java.util.ArrayList;
  56 import java.util.Arrays;
  57 import java.util.Collection;
  58 import java.util.Collections;
  59 import java.util.Comparator;
  60 import java.util.EnumMap;
  61 import java.util.HashMap;
  62 import java.util.LinkedHashMap;
  63 import java.util.List;
  64 import java.util.Locale;
  65 import java.util.Map;
  66 import java.util.Objects;
  67 import java.util.Set;
  68 import java.util.TreeMap;
  69 import java.util.TreeSet;
  70 import java.util.function.Function;
  71 import java.util.regex.Pattern;
  72 import java.util.stream.Collectors;
  73 
  74 
  75 /**
  76  * Test framework for running javadoc and performing tests on the resulting output.
  77  *
  78  * <p>
  79  * Tests are typically written as subtypes of JavadocTester, with a main
  80  * method that creates an instance of the test class and calls the runTests()
  81  * method. The runTests() methods calls all the test methods declared in the class,
  82  * and then calls a method to print a summary, and throw an exception if
  83  * any of the test methods reported a failure.
  84  *
  85  * <p>
  86  * Test methods are identified with a @Test annotation. They have no parameters.
  87  * The name of the method is not important, but if you have more than one, it is
  88  * recommended that the names be meaningful and suggestive of the test case
  89  * contained therein.
  90  *
  91  * <p>
  92  * Typically, a test method will invoke javadoc, and then perform various
  93  * checks on the results. The standard checks are:
  94  *
  95  * <dl>
  96  * <dt>checkExitCode
  97  * <dd>Check the exit code returned from javadoc.
  98  * <dt>checkOutput
  99  * <dd>Perform a series of checks on the contents on a file or output stream
 100  *     generated by javadoc.
 101  *     The checks can be either that a series of strings are found or are not found.
 102  * <dt>checkFiles
 103  * <dd>Perform a series of checks on the files generated by javadoc.
 104  *     The checks can be that a series of files are found or are not found.
 105  * </dl>
 106  *
 107  * <pre><code>
 108  *  public class MyTester extends JavadocTester {
 109  *      public static void main(String... args) throws Exception {
 110  *          MyTester tester = new MyTester();
 111  *          tester.runTests();
 112  *      }
 113  *
 114  *      // test methods...
 115  *      @Test
 116  *      void test() {
 117  *          javadoc(<i>args</i>);
 118  *          checkExit(Exit.OK);
 119  *          checkOutput(<i>file</i>, true,
 120  *              <i>strings-to-find</i>);
 121  *          checkOutput(<i>file</i>, false,
 122  *              <i>strings-to-not-find</i>);
 123  *      }
 124  *  }
 125  * </code></pre>
 126  *
 127  * <p>
 128  * If javadoc is run more than once in a test method, you can compare the
 129  * results that are generated with the diff method. Since files written by
 130  * javadoc typically contain a timestamp, you may want to use the -notimestamp
 131  * option if you are going to compare the results from two runs of javadoc.
 132  *
 133  * <p>
 134  * If you have many calls of checkOutput that are very similar, you can write
 135  * your own check... method to reduce the amount of duplication. For example,
 136  * if you want to check that many files contain the same string, you could
 137  * write a method that takes a varargs list of files and calls checkOutput
 138  * on each file in turn with the string to be checked.
 139  *
 140  * <p>
 141  * You can also write you own custom check methods, which can use
 142  * readFile to get the contents of a file generated by javadoc,
 143  * and then use pass(...) or fail(...) to report whether the check
 144  * succeeded or not.
 145  *
 146  * <p>
 147  * You can have many separate test methods, each identified with a @Test
 148  * annotation. However, you should <b>not</b> assume they will be called
 149  * in the order declared in your source file.  If the order of a series
 150  * of javadoc invocations is important, do that within a single method.
 151  * If the invocations are independent, for better clarity, use separate
 152  * test methods, each with their own set of checks on the results.
 153  *
 154  * @author Doug Kramer
 155  * @author Jamie Ho
 156  * @author Jonathan Gibbons (rewrite)
 157  */
 158 public abstract class JavadocTester {
 159 
 160     public static final String FS = System.getProperty("file.separator");
 161     public static final String PS = System.getProperty("path.separator");
 162     public static final String NL = System.getProperty("line.separator");
 163     public static final Path currDir = Paths.get(".").toAbsolutePath().normalize();
 164 
 165     public enum Output {
 166         /** The name of the output stream from javadoc. */
 167         OUT,
 168         /** The name for any output written to System.out. */
 169         STDOUT,
 170         /** The name for any output written to System.err. */
 171         STDERR
 172     }
 173 
 174     /** The output directory used in the most recent call of javadoc. */
 175     protected File outputDir;
 176 
 177     /** The output charset used in the most recent call of javadoc. */
 178     protected Charset charset = Charset.defaultCharset();
 179 
 180     /** The exit code of the most recent call of javadoc. */
 181     private int exitCode;
 182 
 183     /** The output generated by javadoc to the various writers and streams. */
 184     private final Map<Output, String> outputMap = new EnumMap<>(Output.class);
 185 
 186     /** A cache of file content, to avoid reading files unnecessarily. */
 187     private final Map<File,SoftReference<String>> fileContentCache = new HashMap<>();
 188     /** The charset used for files in the fileContentCache. */
 189     private Charset fileContentCacheCharset = null;
 190 
 191     /** Stream used for logging messages. */
 192     protected final PrintStream out = System.out;
 193 
 194     /** The directory containing the source code for the test. */
 195     public static final String testSrc = System.getProperty("test.src");
 196 
 197     /**
 198      * Get the path for a source file in the test source directory.
 199      * @param path the path of a file or directory in the source directory
 200      * @return the full path of the specified file
 201      */
 202     public static String testSrc(String path) {
 203         return new File(testSrc, path).getPath();
 204     }
 205 
 206     /**
 207      * Alternatives for checking the contents of a directory.
 208      */
 209     public enum DirectoryCheck {
 210         /**
 211          * Check that the directory is empty.
 212          */
 213         EMPTY((file, name) -> true),
 214         /**
 215          * Check that the directory does not contain any HTML files,
 216          * such as may have been generated by a prior run of javadoc
 217          * using this directory.
 218          * For now, the check is only performed on the top level directory.
 219          */
 220         NO_HTML_FILES((file, name) -> name.endsWith(".html")),
 221         /**
 222          * No check is performed on the directory contents.
 223          */
 224         NONE(null) { @Override void check(File dir) { } };
 225 
 226         /** The filter used to detect that files should <i>not</i> be present. */
 227         FilenameFilter filter;
 228 
 229         DirectoryCheck(FilenameFilter f) {
 230             filter = f;
 231         }
 232 
 233         void check(File dir) {
 234             if (dir.isDirectory()) {
 235                 String[] contents = dir.list(filter);
 236                 if (contents == null)
 237                     throw new Error("cannot list directory: " + dir);
 238                 if (contents.length > 0) {
 239                     System.err.println("Found extraneous files in dir:" + dir.getAbsolutePath());
 240                     for (String x : contents) {
 241                         System.err.println(x);
 242                     }
 243                     throw new Error("directory has unexpected content: " + dir);
 244                 }
 245             }
 246         }
 247     }
 248 
 249     private DirectoryCheck outputDirectoryCheck = DirectoryCheck.EMPTY;
 250 
 251     private boolean automaticCheckLinks = true;
 252 
 253     /** The current subtest number. Incremented when checking(...) is called. */
 254     private int numTestsRun = 0;
 255 
 256     /** The number of subtests passed. Incremented when passed(...) is called. */
 257     private int numTestsPassed = 0;
 258 
 259     /** The current run of javadoc. Incremented when javadoc is called. */
 260     private int javadocRunNum = 0;
 261 
 262     /** The current subtest number for this run of javadoc. Incremented when checking(...) is called. */
 263     private int javadocTestNum = 0;
 264 
 265     /** Marker annotation for test methods to be invoked by runTests. */
 266     @Retention(RetentionPolicy.RUNTIME)
 267     @interface Test { }
 268 
 269     /**
 270      * Run all methods annotated with @Test, followed by printSummary.
 271      * Typically called on a tester object in main()
 272      * @throws Exception if any errors occurred
 273      */
 274     public void runTests() throws Exception {
 275         runTests(m -> new Object[0]);
 276     }
 277 
 278     /**
 279      * Run all methods annotated with @Test, followed by printSummary.
 280      * Typically called on a tester object in main()
 281      * @param f a function which will be used to provide arguments to each
 282      *          invoked method
 283      * @throws Exception if any errors occurred
 284      */
 285     public void runTests(Function<Method, Object[]> f) throws Exception {
 286         for (Method m: getClass().getDeclaredMethods()) {
 287             Annotation a = m.getAnnotation(Test.class);
 288             if (a != null) {
 289                 try {
 290                     out.println("Running test " + m.getName());
 291                     m.invoke(this, f.apply(m));
 292                 } catch (InvocationTargetException e) {
 293                     Throwable cause = e.getCause();
 294                     throw (cause instanceof Exception) ? ((Exception) cause) : e;
 295                 }
 296                 out.println();
 297             }
 298         }
 299         printSummary();
 300     }
 301 
 302     /**
 303      * Run javadoc.
 304      * The output directory used by this call and the final exit code
 305      * will be saved for later use.
 306      * To aid the reader, it is recommended that calls to this method
 307      * put each option and the arguments it takes on a separate line.
 308      *
 309      * Example:
 310      * <pre><code>
 311      *  javadoc("-d", "out",
 312      *          "-sourcepath", testSrc,
 313      *          "-notimestamp",
 314      *          "pkg1", "pkg2", "pkg3/C.java");
 315      * </code></pre>
 316      *
 317      * @param args the arguments to pass to javadoc
 318      */
 319     public void javadoc(String... args) {
 320         outputMap.clear();
 321         fileContentCache.clear();
 322 
 323         javadocRunNum++;
 324         javadocTestNum = 0; // reset counter for this run of javadoc
 325         if (javadocRunNum == 1) {
 326             out.println("Running javadoc...");
 327         } else {
 328             out.println("Running javadoc (run "+ javadocRunNum + ")...");
 329         }
 330 
 331         outputDir = new File(".");
 332         String charsetArg = null;
 333         String docencodingArg = null;
 334         String encodingArg = null;
 335         for (int i = 0; i < args.length - 2; i++) {
 336             switch (args[i]) {
 337                 case "-d":
 338                     outputDir = new File(args[++i]);
 339                     break;
 340                 case "-charset":
 341                     charsetArg = args[++i];
 342                     break;
 343                 case "-docencoding":
 344                     docencodingArg = args[++i];
 345                     break;
 346                 case "-encoding":
 347                     encodingArg = args[++i];
 348                     break;
 349             }
 350         }
 351 
 352         // The following replicates HtmlConfiguration.finishOptionSettings0
 353         // and sets up the charset used to read files.
 354         String cs;
 355         if (docencodingArg == null) {
 356             if (charsetArg == null) {
 357                 cs = (encodingArg == null) ? "UTF-8" : encodingArg;
 358             } else {
 359                 cs = charsetArg;
 360             }
 361         } else {
 362            cs = docencodingArg;
 363         }
 364         try {
 365             charset = Charset.forName(cs);
 366         } catch (UnsupportedCharsetException e) {
 367             charset = Charset.defaultCharset();
 368         }
 369 
 370         out.println("args: " + Arrays.toString(args));
 371 //        log.setOutDir(outputDir);
 372 
 373         outputDirectoryCheck.check(outputDir);
 374 
 375         // This is the sole stream used by javadoc
 376         WriterOutput outOut = new WriterOutput();
 377 
 378         // These are to catch output to System.out and System.err,
 379         // in case these are used instead of the primary streams
 380         StreamOutput sysOut = new StreamOutput(System.out, System::setOut);
 381         StreamOutput sysErr = new StreamOutput(System.err, System::setErr);
 382 
 383         try {
 384             exitCode = jdk.javadoc.internal.tool.Main.execute(args, outOut.pw);
 385         } finally {
 386             outputMap.put(Output.STDOUT, sysOut.close());
 387             outputMap.put(Output.STDERR, sysErr.close());
 388             outputMap.put(Output.OUT, outOut.close());
 389         }
 390 
 391         outputMap.forEach((name, text) -> {
 392             if (!text.isEmpty()) {
 393                 out.println("javadoc " + name + ":");
 394                 out.println(text);
 395             }
 396         });
 397 
 398         if (automaticCheckLinks && exitCode == Exit.OK.code && outputDir.exists()) {
 399             checkLinks();
 400         }
 401     }
 402 
 403     /**
 404      * Set the kind of check for the initial contents of the output directory
 405      * before javadoc is run.
 406      * The filter should return true for files that should <b>not</b> appear.
 407      * @param c the kind of check to perform
 408      */
 409     public void setOutputDirectoryCheck(DirectoryCheck c) {
 410         outputDirectoryCheck = c;
 411     }
 412 
 413     /**
 414      * Set whether or not to perform an automatic call of checkLinks.
 415      */
 416     public void setAutomaticCheckLinks(boolean b) {
 417         automaticCheckLinks = b;
 418     }
 419 
 420     /**
 421      * The exit codes returned by the javadoc tool.
 422      * @see jdk.javadoc.internal.tool.Main.Result
 423      */
 424     public enum Exit {
 425         OK(0),        // Javadoc completed with no errors.
 426         ERROR(1),     // Completed but reported errors.
 427         CMDERR(2),    // Bad command-line arguments
 428         SYSERR(3),    // System error or resource exhaustion.
 429         ABNORMAL(4);  // Javadoc terminated abnormally
 430 
 431         Exit(int code) {
 432             this.code = code;
 433         }
 434 
 435         final int code;
 436 
 437         @Override
 438         public String toString() {
 439             return name() + '(' + code + ')';
 440         }
 441     }
 442 
 443     /**
 444      * Check the exit code of the most recent call of javadoc.
 445      *
 446      * @param expected the exit code that is required for the test
 447      * to pass.
 448      */
 449     public void checkExit(Exit expected) {
 450         checking("check exit code");
 451         if (exitCode == expected.code) {
 452             passed("return code " + exitCode);
 453         } else {
 454             failed("return code " + exitCode +"; expected " + expected);
 455         }
 456     }
 457 
 458     /**
 459      * Check for content in (or not in) the generated output.
 460      * Within the search strings, the newline character \n
 461      * will be translated to the platform newline character sequence.
 462      * @param path a path within the most recent output directory
 463      *  or the name of one of the output buffers, identifying
 464      *  where to look for the search strings.
 465      * @param expectedFound true if all of the search strings are expected
 466      *  to be found, or false if the file is not expected to be found
 467      * @param strings the strings to be searched for
 468      */
 469     public void checkFileAndOutput(String path, boolean expectedFound, String... strings) {
 470         if (expectedFound) {
 471             checkOutput(path, true, strings);
 472         } else {
 473             checkFiles(false, path);
 474         }
 475     }
 476 
 477     /**
 478      * Check for content in (or not in) the generated output.
 479      * Within the search strings, the newline character \n
 480      * will be translated to the platform newline character sequence.
 481      * @param path a path within the most recent output directory, identifying
 482      *  where to look for the search strings.
 483      * @param expectedFound true if all of the search strings are expected
 484      *  to be found, or false if all of the strings are expected to be
 485      *  not found
 486      * @param strings the strings to be searched for
 487      */
 488     public void checkOutput(String path, boolean expectedFound, String... strings) {
 489         // Read contents of file
 490         try {
 491             String fileString = readFile(outputDir, path);
 492             checkOutput(new File(outputDir, path).getPath(), fileString, expectedFound, strings);
 493         } catch (Error e) {
 494             checking("Read file");
 495             failed("Error reading file: " + e);
 496         }
 497     }
 498 
 499     /**
 500      * Check for content in (or not in) the one of the output streams written by
 501      * javadoc. Within the search strings, the newline character \n
 502      * will be translated to the platform newline character sequence.
 503      * @param output the output stream to check
 504      * @param expectedFound true if all of the search strings are expected
 505      *  to be found, or false if all of the strings are expected to be
 506      *  not found
 507      * @param strings the strings to be searched for
 508      */
 509     public void checkOutput(Output output, boolean expectedFound, String... strings) {
 510         checkOutput(output.toString(), outputMap.get(output), expectedFound, strings);
 511     }
 512 
 513     // NOTE: path may be the name of an Output stream as well as a file path
 514     private void checkOutput(String path, String fileString, boolean expectedFound, String... strings) {
 515         for (String stringToFind : strings) {
 516 //            log.logCheckOutput(path, expectedFound, stringToFind);
 517             checking("checkOutput");
 518             // Find string in file's contents
 519             boolean isFound = findString(fileString, stringToFind);
 520             if (isFound == expectedFound) {
 521                 passed(path + ": following text " + (isFound ? "found:" : "not found:") + "\n"
 522                         + stringToFind);
 523             } else {
 524                 failed(path + ": following text " + (isFound ? "found:" : "not found:") + "\n"
 525                         + stringToFind);
 526             }
 527         }
 528     }
 529 
 530     public void checkLinks() {
 531         checking("Check links");
 532         LinkChecker c = new LinkChecker(out, this::readFile);
 533         try {
 534             c.checkDirectory(outputDir.toPath());
 535             c.report();
 536             int errors = c.getErrorCount();
 537             if (errors == 0) {
 538                 passed("Links are OK");
 539             } else {
 540                 failed(errors + " errors found when checking links");
 541             }
 542         } catch (IOException e) {
 543             failed("exception thrown when reading files: " + e);
 544         }
 545     }
 546 
 547     /**
 548      * Get the content of the one of the output streams written by javadoc.
 549      * @param output the name of the output stream
 550      * @return the content of the output stream
 551      */
 552     public String getOutput(Output output) {
 553         return outputMap.get(output);
 554     }
 555 
 556     /**
 557      * Get the content of the one of the output streams written by javadoc.
 558      * @param output the name of the output stream
 559      * @return the content of the output stream, as a line of lines
 560      */
 561     public List<String> getOutputLines(Output output) {
 562         String text = outputMap.get(output);
 563         return (text == null) ? Collections.emptyList() : Arrays.asList(text.split(NL));
 564     }
 565 
 566     /**
 567      * Check for files in (or not in) the generated output.
 568      * @param expectedFound true if all of the files are expected
 569      *  to be found, or false if all of the files are expected to be
 570      *  not found
 571      * @param paths the files to check, within the most recent output directory.
 572      * */
 573     public void checkFiles(boolean expectedFound, String... paths) {
 574         checkFiles(expectedFound, Arrays.asList(paths));
 575     }
 576 
 577     /**
 578      * Check for files in (or not in) the generated output.
 579      * @param expectedFound true if all of the files are expected
 580      *  to be found, or false if all of the files are expected to be
 581      *  not found
 582      * @param paths the files to check, within the most recent output directory.
 583      * */
 584     public void checkFiles(boolean expectedFound, Collection<String> paths) {
 585         for (String path: paths) {
 586 //            log.logCheckFile(path, expectedFound);
 587             checking("checkFile");
 588             File file = new File(outputDir, path);
 589             boolean isFound = file.exists();
 590             if (isFound == expectedFound) {
 591                 passed(file, "file " + (isFound ? "found:" : "not found:") + "\n");
 592             } else {
 593                 failed(file, "file " + (isFound ? "found:" : "not found:") + "\n");
 594             }
 595         }
 596     }
 597 
 598     /**
 599      * Check that a series of strings are found in order in a file in
 600      * the generated output.
 601      * @param path the file to check
 602      * @param strings  the strings whose order to check
 603      */
 604     public void checkOrder(String path, String... strings) {
 605         File file = new File(outputDir, path);
 606         String fileString = readOutputFile(path);
 607         int prevIndex = -1;
 608         for (String s : strings) {
 609             s = s.replace("\n", NL); // normalize new lines
 610             int currentIndex = fileString.indexOf(s, prevIndex + 1);
 611             checking("file: " + file + ": " + s + " at index " + currentIndex);
 612             if (currentIndex == -1) {
 613                 failed(file, s + " not found.");
 614                 continue;
 615             }
 616             if (currentIndex > prevIndex) {
 617                 passed(file, s + " is in the correct order");
 618             } else {
 619                 failed(file, s + " is in the wrong order.");
 620             }
 621             prevIndex = currentIndex;
 622         }
 623     }
 624 
 625     /**
 626      * Ensures that a series of strings appear only once, in the generated output,
 627      * noting that, this test does not exhaustively check for all other possible
 628      * duplicates once one is found.
 629      * @param path the file to check
 630      * @param strings ensure each are unique
 631      */
 632     public void checkUnique(String path, String... strings) {
 633         File file = new File(outputDir, path);
 634         String fileString = readOutputFile(path);
 635         for (String s : strings) {
 636             int currentIndex = fileString.indexOf(s);
 637             checking(s + " at index " + currentIndex);
 638             if (currentIndex == -1) {
 639                 failed(file, s + " not found.");
 640                 continue;
 641             }
 642             int nextindex = fileString.indexOf(s, currentIndex + s.length());
 643             if (nextindex == -1) {
 644                 passed(file, s + " is unique");
 645             } else {
 646                 failed(file, s + " is not unique, found at " + nextindex);
 647             }
 648         }
 649     }
 650 
 651     /**
 652      * Compare a set of files in each of two directories.
 653      *
 654      * @param baseDir1 the directory containing the first set of files
 655      * @param baseDir2 the directory containing the second set of files
 656      * @param files the set of files to be compared
 657      */
 658     public void diff(String baseDir1, String baseDir2, String... files) {
 659         File bd1 = new File(baseDir1);
 660         File bd2 = new File(baseDir2);
 661         for (String file : files) {
 662             diff(bd1, bd2, file);
 663         }
 664     }
 665 
 666     /**
 667      * A utility to copy a directory from one place to another.
 668      *
 669      * @param targetDir the directory to copy.
 670      * @param destDir the destination to copy the directory to.
 671      */
 672     // TODO: convert to using java.nio.Files.walkFileTree
 673     public void copyDir(String targetDir, String destDir) {
 674         try {
 675             File targetDirObj = new File(targetDir);
 676             File destDirParentObj = new File(destDir);
 677             File destDirObj = new File(destDirParentObj, targetDirObj.getName());
 678             if (! destDirParentObj.exists()) {
 679                 destDirParentObj.mkdir();
 680             }
 681             if (! destDirObj.exists()) {
 682                 destDirObj.mkdir();
 683             }
 684             String[] files = targetDirObj.list();
 685             for (String file : files) {
 686                 File srcFile = new File(targetDirObj, file);
 687                 File destFile = new File(destDirObj, file);
 688                 if (srcFile.isFile()) {
 689                     out.println("Copying " + srcFile + " to " + destFile);
 690                     copyFile(destFile, srcFile);
 691                 } else if(srcFile.isDirectory()) {
 692                     copyDir(srcFile.getAbsolutePath(), destDirObj.getAbsolutePath());
 693                 }
 694             }
 695         } catch (IOException exc) {
 696             throw new Error("Could not copy " + targetDir + " to " + destDir);
 697         }
 698     }
 699 
 700     /**
 701      * Copy source file to destination file.
 702      *
 703      * @param destfile the destination file
 704      * @param srcfile the source file
 705      * @throws IOException
 706      */
 707     public void copyFile(File destfile, File srcfile) throws IOException {
 708         Files.copy(srcfile.toPath(), destfile.toPath());
 709     }
 710 
 711     /**
 712      * Read a file from the output directory.
 713      *
 714      * @param fileName  the name of the file to read
 715      * @return          the file in string format
 716      */
 717     public String readOutputFile(String fileName) throws Error {
 718         return readFile(outputDir, fileName);
 719     }
 720 
 721     protected String readFile(String fileName) throws Error {
 722         return readFile(outputDir, fileName);
 723     }
 724 
 725     protected String readFile(String baseDir, String fileName) throws Error {
 726         return readFile(new File(baseDir), fileName);
 727     }
 728 
 729     private String readFile(Path file) {
 730         File baseDir;
 731         if (file.startsWith(outputDir.toPath())) {
 732             baseDir = outputDir;
 733         } else if (file.startsWith(currDir)) {
 734             baseDir = currDir.toFile();
 735         } else {
 736             baseDir = file.getParent().toFile();
 737         }
 738         String fileName = baseDir.toPath().relativize(file).toString();
 739         return readFile(baseDir, fileName);
 740     }
 741 
 742     /**
 743      * Read the file and return it as a string.
 744      *
 745      * @param baseDir   the directory in which to locate the file
 746      * @param fileName  the name of the file to read
 747      * @return          the file in string format
 748      */
 749     private String readFile(File baseDir, String fileName) throws Error {
 750         if (!Objects.equals(fileContentCacheCharset, charset)) {
 751             fileContentCache.clear();
 752             fileContentCacheCharset = charset;
 753         }
 754         try {
 755             File file = new File(baseDir, fileName);
 756             SoftReference<String> ref = fileContentCache.get(file);
 757             String content = (ref == null) ? null : ref.get();
 758             if (content != null)
 759                 return content;
 760 
 761             // charset defaults to a value inferred from latest javadoc run
 762             content = new String(Files.readAllBytes(file.toPath()), charset);
 763             fileContentCache.put(file, new SoftReference<>(content));
 764             return content;
 765         } catch (FileNotFoundException e) {
 766             throw new Error("File not found: " + fileName + ": " + e);
 767         } catch (IOException e) {
 768             throw new Error("Error reading file: " + fileName + ": " + e);
 769         }
 770     }
 771 
 772     protected void checking(String message) {
 773         numTestsRun++;
 774         javadocTestNum++;
 775         print("Starting subtest " + javadocRunNum + "." + javadocTestNum, message);
 776     }
 777 
 778     protected void passed(File file, String message) {
 779         passed(file + ": " + message);
 780     }
 781 
 782     protected void passed(String message) {
 783         numTestsPassed++;
 784         print("Passed", message);
 785         out.println();
 786     }
 787 
 788     protected void failed(File file, String message) {
 789         failed(file + ": " + message);
 790     }
 791 
 792     protected void failed(String message) {
 793         print("FAILED", message);
 794         StackWalker.getInstance().walk(s -> {
 795             s.dropWhile(f -> f.getMethodName().equals("failed"))
 796                     .takeWhile(f -> !f.getMethodName().equals("runTests"))
 797                     .forEach(f -> out.println("        at "
 798                             + f.getClassName() + "." + f.getMethodName()
 799                             + "(" + f.getFileName() + ":" + f.getLineNumber() + ")"));
 800             return null;
 801         });
 802         out.println();
 803     }
 804 
 805     private void print(String prefix, String message) {
 806         if (message.isEmpty())
 807             out.println(prefix);
 808         else {
 809             out.print(prefix);
 810             out.print(": ");
 811             out.print(message.replace("\n", NL));
 812             if (!(message.endsWith("\n") || message.endsWith(NL))) {
 813                 out.println();
 814             }
 815         }
 816     }
 817 
 818     /**
 819      * Print a summary of the test results.
 820      */
 821     protected void printSummary() {
 822         String javadocRuns = (javadocRunNum <= 1) ? ""
 823                 : ", in " + javadocRunNum + " runs of javadoc";
 824 
 825         if (numTestsRun != 0 && numTestsPassed == numTestsRun) {
 826             // Test passed
 827             out.println();
 828             out.println("All " + numTestsPassed + " subtests passed" + javadocRuns);
 829         } else {
 830             // Test failed
 831             throw new Error((numTestsRun - numTestsPassed)
 832                     + " of " + (numTestsRun)
 833                     + " subtests failed"
 834                     + javadocRuns);
 835         }
 836     }
 837 
 838     /**
 839      * Search for the string in the given file and return true
 840      * if the string was found.
 841      *
 842      * @param fileString    the contents of the file to search through
 843      * @param stringToFind  the string to search for
 844      * @return              true if the string was found
 845      */
 846     private boolean findString(String fileString, String stringToFind) {
 847         // javadoc (should) always use the platform newline sequence,
 848         // but in the strings to find it is more convenient to use the Java
 849         // newline character. So we translate \n to NL before we search.
 850         stringToFind = stringToFind.replace("\n", NL);
 851         return fileString.contains(stringToFind);
 852     }
 853 
 854     /**
 855      * Compare the two given files.
 856      *
 857      * @param baseDir1 the directory in which to locate the first file
 858      * @param baseDir2 the directory in which to locate the second file
 859      * @param file the file to compare in the two base directories
 860      * @param throwErrorIFNoMatch flag to indicate whether or not to throw
 861      * an error if the files do not match.
 862      * @return true if the files are the same and false otherwise.
 863      */
 864     private void diff(File baseDir1, File baseDir2, String file) {
 865         String file1Contents = readFile(baseDir1, file);
 866         String file2Contents = readFile(baseDir2, file);
 867         checking("diff " + new File(baseDir1, file) + ", " + new File(baseDir2, file));
 868         if (file1Contents.trim().compareTo(file2Contents.trim()) == 0) {
 869             passed("files are equal");
 870         } else {
 871             failed("files differ");
 872         }
 873     }
 874 
 875     /**
 876      * Utility class to simplify the handling of temporarily setting a
 877      * new stream for System.out or System.err.
 878      */
 879     private static class StreamOutput {
 880         // functional interface to set a stream.
 881         private interface Initializer {
 882             void set(PrintStream s);
 883         }
 884 
 885         private final ByteArrayOutputStream baos = new ByteArrayOutputStream();
 886         private final PrintStream ps = new PrintStream(baos);
 887         private final PrintStream prev;
 888         private final Initializer init;
 889 
 890         StreamOutput(PrintStream s, Initializer init) {
 891             prev = s;
 892             init.set(ps);
 893             this.init = init;
 894         }
 895 
 896         String close() {
 897             init.set(prev);
 898             ps.close();
 899             return baos.toString();
 900         }
 901     }
 902 
 903     /**
 904      * Utility class to simplify the handling of creating an in-memory PrintWriter.
 905      */
 906     private static class WriterOutput {
 907         private final StringWriter sw = new StringWriter();
 908         final PrintWriter pw = new PrintWriter(sw);
 909         String close() {
 910             pw.close();
 911             return sw.toString();
 912         }
 913     }
 914 
 915 
 916 //    private final Logger log = new Logger();
 917 
 918     //--------- Logging --------------------------------------------------------
 919     //
 920     // This class writes out the details of calls to checkOutput and checkFile
 921     // in a canonical way, so that the resulting file can be checked against
 922     // similar files from other versions of JavadocTester using the same logging
 923     // facilities.
 924 
 925     static class Logger {
 926         private static final int PREFIX = 40;
 927         private static final int SUFFIX = 20;
 928         private static final int MAX = PREFIX + SUFFIX;
 929         List<String> tests = new ArrayList<>();
 930         String outDir;
 931         String rootDir = rootDir();
 932 
 933         static String rootDir() {
 934             File f = new File(".").getAbsoluteFile();
 935             while (!new File(f, ".hg").exists())
 936                 f = f.getParentFile();
 937             return f.getPath();
 938         }
 939 
 940         void setOutDir(File outDir) {
 941             this.outDir = outDir.getPath();
 942         }
 943 
 944         void logCheckFile(String file, boolean positive) {
 945             // Strip the outdir because that will typically not be the same
 946             if (file.startsWith(outDir + "/"))
 947                 file = file.substring(outDir.length() + 1);
 948             tests.add(file + " " + positive);
 949         }
 950 
 951         void logCheckOutput(String file, boolean positive, String text) {
 952             // Compress the string to be displayed in the log file
 953             String simpleText = text.replaceAll("\\s+", " ").replace(rootDir, "[ROOT]");
 954             if (simpleText.length() > MAX)
 955                 simpleText = simpleText.substring(0, PREFIX)
 956                         + "..." + simpleText.substring(simpleText.length() - SUFFIX);
 957             // Strip the outdir because that will typically not be the same
 958             if (file.startsWith(outDir + "/"))
 959                 file = file.substring(outDir.length() + 1);
 960             // The use of text.hashCode ensure that all of "text" is taken into account
 961             tests.add(file + " " + positive + " " + text.hashCode() + " " + simpleText);
 962         }
 963 
 964         void write() {
 965             // sort the log entries because the subtests may not be executed in the same order
 966             tests.sort((a, b) -> a.compareTo(b));
 967             try (BufferedWriter bw = new BufferedWriter(new FileWriter("tester.log"))) {
 968                 for (String t: tests) {
 969                     bw.write(t);
 970                     bw.newLine();
 971                 }
 972             } catch (IOException e) {
 973                 throw new Error("problem writing log: " + e);
 974             }
 975         }
 976     }
 977 
 978     // Support classes for checkLinks
 979 
 980     /**
 981      * A basic HTML parser. Override the protected methods as needed to get notified
 982      * of significant items in any file that is read.
 983      */
 984     static abstract class HtmlParser {
 985 
 986         protected final PrintStream out;
 987         protected final Function<Path,String> fileReader;
 988 
 989         private Path file;
 990         private StringReader in;
 991         private int ch;
 992         private int lineNumber;
 993         private boolean inScript;
 994         private boolean xml;
 995 
 996         HtmlParser(PrintStream out, Function<Path,String> fileReader) {
 997             this.out = out;
 998             this.fileReader = fileReader;
 999         }
1000 
1001         /**
1002          * Read a file.
1003          * @param file the file to be read
1004          * @throws IOException if an error occurs while reading the file
1005          */
1006         void read(Path file) throws IOException {
1007             try (StringReader r = new StringReader(fileReader.apply(file))) {
1008                 this.file = file;
1009                 this.in = r;
1010 
1011                 startFile(file);
1012                 try {
1013                     lineNumber = 1;
1014                     xml = false;
1015                     nextChar();
1016 
1017                     while (ch != -1) {
1018                         switch (ch) {
1019 
1020                             case '<':
1021                                 html();
1022                                 break;
1023 
1024                             default:
1025                                 nextChar();
1026                         }
1027                     }
1028                 } finally {
1029                     endFile();
1030                 }
1031             } catch (IOException e) {
1032                 error(file, lineNumber, e);
1033             } catch (Throwable t) {
1034                 error(file, lineNumber, t);
1035                 t.printStackTrace(out);
1036             }
1037         }
1038 
1039 
1040         int getLineNumber() {
1041             return lineNumber;
1042         }
1043 
1044         /**
1045          * Called when a file has been opened, before parsing begins.
1046          * This is always the first notification when reading a file.
1047          * This implementation does nothing.
1048          *
1049          * @param file the file
1050          */
1051         protected void startFile(Path file) { }
1052 
1053         /**
1054          * Called when the parser has finished reading a file.
1055          * This is always the last notification when reading a file,
1056          * unless any errors occur while closing the file.
1057          * This implementation does nothing.
1058          */
1059         protected void endFile() { }
1060 
1061         /**
1062          * Called when a doctype declaration is found, at the beginning of the file.
1063          * This implementation does nothing.
1064          * @param s the doctype declaration
1065          */
1066         protected void docType(String s) { }
1067 
1068         /**
1069          * Called when the opening tag of an HTML element is encountered.
1070          * This implementation does nothing.
1071          * @param name the name of the tag
1072          * @param attrs the attribute
1073          * @param selfClosing whether or not this is a self-closing tag
1074          */
1075         protected void startElement(String name, Map<String,String> attrs, boolean selfClosing) { }
1076 
1077         /**
1078          * Called when the closing tag of an HTML tag is encountered.
1079          * This implementation does nothing.
1080          * @param name the name of the tag
1081          */
1082         protected void endElement(String name) { }
1083 
1084         /**
1085          * Called when an error has been encountered.
1086          * @param file the file being read
1087          * @param lineNumber the line number of line containing the error
1088          * @param message a description of the error
1089          */
1090         protected void error(Path file, int lineNumber, String message) {
1091             out.println(file + ":" + lineNumber + ": " + message);
1092         }
1093 
1094         /**
1095          * Called when an exception has been encountered.
1096          * @param file the file being read
1097          * @param lineNumber the line number of the line being read when the exception was found
1098          * @param t the exception
1099          */
1100         protected void error(Path file, int lineNumber, Throwable t) {
1101             out.println(file + ":" + lineNumber + ": " + t);
1102         }
1103 
1104         private void nextChar() throws IOException {
1105             ch = in.read();
1106             if (ch == '\n')
1107                 lineNumber++;
1108         }
1109 
1110         /**
1111          * Read the start or end of an HTML tag, or an HTML comment
1112          * {@literal <identifier attrs> } or {@literal </identifier> }
1113          * @throws java.io.IOException if there is a problem reading the file
1114          */
1115         private void html() throws IOException {
1116             nextChar();
1117             if (isIdentifierStart((char) ch)) {
1118                 String name = readIdentifier().toLowerCase(Locale.US);
1119                 Map<String,String> attrs = htmlAttrs();
1120                 if (attrs != null) {
1121                     boolean selfClosing = false;
1122                     if (ch == '/') {
1123                         nextChar();
1124                         selfClosing = true;
1125                     }
1126                     if (ch == '>') {
1127                         nextChar();
1128                         startElement(name, attrs, selfClosing);
1129                         if (name.equals("script")) {
1130                             inScript = true;
1131                         }
1132                         return;
1133                     }
1134                 }
1135             } else if (ch == '/') {
1136                 nextChar();
1137                 if (isIdentifierStart((char) ch)) {
1138                     String name = readIdentifier().toLowerCase(Locale.US);
1139                     skipWhitespace();
1140                     if (ch == '>') {
1141                         nextChar();
1142                         endElement(name);
1143                         if (name.equals("script")) {
1144                             inScript = false;
1145                         }
1146                         return;
1147                     }
1148                 }
1149             } else if (ch == '!') {
1150                 nextChar();
1151                 if (ch == '-') {
1152                     nextChar();
1153                     if (ch == '-') {
1154                         nextChar();
1155                         while (ch != -1) {
1156                             int dash = 0;
1157                             while (ch == '-') {
1158                                 dash++;
1159                                 nextChar();
1160                             }
1161                             // Strictly speaking, a comment should not contain "--"
1162                             // so dash > 2 is an error, dash == 2 implies ch == '>'
1163                             // See http://www.w3.org/TR/html-markup/syntax.html#syntax-comments
1164                             // for more details.
1165                             if (dash >= 2 && ch == '>') {
1166                                 nextChar();
1167                                 return;
1168                             }
1169 
1170                             nextChar();
1171                         }
1172                     }
1173                 } else if (ch == '[') {
1174                     nextChar();
1175                     if (ch == 'C') {
1176                         nextChar();
1177                         if (ch == 'D') {
1178                             nextChar();
1179                             if (ch == 'A') {
1180                                 nextChar();
1181                                 if (ch == 'T') {
1182                                     nextChar();
1183                                     if (ch == 'A') {
1184                                         nextChar();
1185                                         if (ch == '[') {
1186                                             while (true) {
1187                                                 nextChar();
1188                                                 if (ch == ']') {
1189                                                     nextChar();
1190                                                     if (ch == ']') {
1191                                                         nextChar();
1192                                                         if (ch == '>') {
1193                                                             nextChar();
1194                                                             return;
1195                                                         }
1196                                                     }
1197                                                 }
1198                                             }
1199 
1200                                         }
1201                                     }
1202                                 }
1203                             }
1204                         }
1205                     }
1206                 } else {
1207                     StringBuilder sb = new StringBuilder();
1208                     while (ch != -1 && ch != '>') {
1209                         sb.append((char) ch);
1210                         nextChar();
1211                     }
1212                     Pattern p = Pattern.compile("(?is)doctype\\s+html\\s?.*");
1213                     String s = sb.toString();
1214                     if (p.matcher(s).matches()) {
1215                         docType(s);
1216                         return;
1217                     }
1218                 }
1219             } else if (ch == '?') {
1220                 nextChar();
1221                 if (ch == 'x') {
1222                     nextChar();
1223                     if (ch == 'm') {
1224                         nextChar();
1225                         if (ch == 'l') {
1226                             Map<String,String> attrs = htmlAttrs();
1227                             if (ch == '?') {
1228                                 nextChar();
1229                                 if (ch == '>') {
1230                                     nextChar();
1231                                     xml = true;
1232                                     return;
1233                                 }
1234                             }
1235                         }
1236                     }
1237 
1238                 }
1239             }
1240 
1241             if (!inScript) {
1242                 error(file, lineNumber, "bad html");
1243             }
1244         }
1245 
1246         /**
1247          * Read a series of HTML attributes, terminated by {@literal > }.
1248          * Each attribute is of the form {@literal identifier[=value] }.
1249          * "value" may be unquoted, single-quoted, or double-quoted.
1250          */
1251         private Map<String,String> htmlAttrs() throws IOException {
1252             Map<String, String> map = new LinkedHashMap<>();
1253             skipWhitespace();
1254 
1255             loop:
1256             while (isIdentifierStart((char) ch)) {
1257                 String name = readAttributeName().toLowerCase(Locale.US);
1258                 skipWhitespace();
1259                 String value = null;
1260                 if (ch == '=') {
1261                     nextChar();
1262                     skipWhitespace();
1263                     if (ch == '\'' || ch == '"') {
1264                         char quote = (char) ch;
1265                         nextChar();
1266                         StringBuilder sb = new StringBuilder();
1267                         while (ch != -1 && ch != quote) {
1268                             sb.append((char) ch);
1269                             nextChar();
1270                         }
1271                         value = sb.toString() // hack to replace common entities
1272                                 .replace("&lt;", "<")
1273                                 .replace("&gt;", ">")
1274                                 .replace("&amp;", "&");
1275                         nextChar();
1276                     } else {
1277                         StringBuilder sb = new StringBuilder();
1278                         while (ch != -1 && !isUnquotedAttrValueTerminator((char) ch)) {
1279                             sb.append((char) ch);
1280                             nextChar();
1281                         }
1282                         value = sb.toString();
1283                     }
1284                     skipWhitespace();
1285                 }
1286                 map.put(name, value);
1287             }
1288 
1289             return map;
1290         }
1291 
1292         private boolean isIdentifierStart(char ch) {
1293             return Character.isUnicodeIdentifierStart(ch);
1294         }
1295 
1296         private String readIdentifier() throws IOException {
1297             StringBuilder sb = new StringBuilder();
1298             sb.append((char) ch);
1299             nextChar();
1300             while (ch != -1 && Character.isUnicodeIdentifierPart(ch)) {
1301                 sb.append((char) ch);
1302                 nextChar();
1303             }
1304             return sb.toString();
1305         }
1306 
1307         private String readAttributeName() throws IOException {
1308             StringBuilder sb = new StringBuilder();
1309             sb.append((char) ch);
1310             nextChar();
1311             while (ch != -1 && Character.isUnicodeIdentifierPart(ch)
1312                     || ch == '-'
1313                     || xml && ch == ':') {
1314                 sb.append((char) ch);
1315                 nextChar();
1316             }
1317             return sb.toString();
1318         }
1319 
1320         private boolean isWhitespace(char ch) {
1321             return Character.isWhitespace(ch);
1322         }
1323 
1324         private void skipWhitespace() throws IOException {
1325             while (isWhitespace((char) ch)) {
1326                 nextChar();
1327             }
1328         }
1329 
1330         private boolean isUnquotedAttrValueTerminator(char ch) {
1331             switch (ch) {
1332                 case '\f': case '\n': case '\r': case '\t':
1333                 case ' ':
1334                 case '"': case '\'': case '`':
1335                 case '=': case '<': case '>':
1336                     return true;
1337                 default:
1338                     return false;
1339             }
1340         }
1341     }
1342 
1343     /**
1344      * A class to check the links in a set of HTML files.
1345      */
1346     static class LinkChecker extends HtmlParser {
1347         private final Map<Path, IDTable> allFiles;
1348         private final Map<URI, IDTable> allURIs;
1349 
1350         private int files;
1351         private int links;
1352         private int badSchemes;
1353         private int duplicateIds;
1354         private int missingIds;
1355 
1356         private Path currFile;
1357         private IDTable currTable;
1358         private boolean html5;
1359         private boolean xml;
1360 
1361         private int errors;
1362 
1363         LinkChecker(PrintStream out, Function<Path,String> fileReader) {
1364             super(out, fileReader);
1365             allFiles = new HashMap<>();
1366             allURIs = new HashMap<>();
1367         }
1368 
1369         void checkDirectory(Path dir) throws IOException {
1370             checkFiles(List.of(dir), false, Collections.emptySet());
1371         }
1372 
1373         void checkFiles(List<Path> files, boolean skipSubdirs, Set<Path> excludeFiles) throws IOException {
1374             for (Path file : files) {
1375                 Files.walkFileTree(file, new SimpleFileVisitor<Path>() {
1376                     int depth = 0;
1377 
1378                     @Override
1379                     public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
1380                         if ((skipSubdirs && depth > 0) || excludeFiles.contains(dir)) {
1381                             return FileVisitResult.SKIP_SUBTREE;
1382                         }
1383                         depth++;
1384                         return FileVisitResult.CONTINUE;
1385                     }
1386 
1387                     @Override
1388                     public FileVisitResult visitFile(Path p, BasicFileAttributes attrs) {
1389                         if (excludeFiles.contains(p)) {
1390                             return FileVisitResult.CONTINUE;
1391                         }
1392 
1393                         if (Files.isRegularFile(p) && p.getFileName().toString().endsWith(".html")) {
1394                             checkFile(p);
1395                         }
1396                         return FileVisitResult.CONTINUE;
1397                     }
1398 
1399                     @Override
1400                     public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException {
1401                         depth--;
1402                         return super.postVisitDirectory(dir, e);
1403                     }
1404                 });
1405             }
1406         }
1407 
1408         void checkFile(Path file) {
1409             try {
1410                 read(file);
1411             } catch (IOException e) {
1412                 error(file, 0, e);
1413             }
1414         }
1415 
1416         int getErrorCount() {
1417             return errors;
1418         }
1419 
1420         public void report() {
1421             List<Path> missingFiles = getMissingFiles();
1422             if (!missingFiles.isEmpty()) {
1423                 report("Missing files: (" + missingFiles.size() + ")");
1424                 missingFiles.stream()
1425                         .sorted()
1426                         .forEach(this::reportMissingFile);
1427 
1428             }
1429 
1430             if (!allURIs.isEmpty()) {
1431                 report(false, "External URLs:");
1432                 allURIs.keySet().stream()
1433                         .sorted(new URIComparator())
1434                         .forEach(uri -> report(false, "  %s", uri.toString()));
1435             }
1436 
1437             int anchors = 0;
1438             for (IDTable t : allFiles.values()) {
1439                 anchors += t.map.values().stream()
1440                         .filter(e -> !e.getReferences().isEmpty())
1441                         .count();
1442             }
1443             for (IDTable t : allURIs.values()) {
1444                 anchors += t.map.values().stream()
1445                         .filter(e -> !e.references.isEmpty())
1446                         .count();
1447             }
1448 
1449             report(false, "Checked " + files + " files.");
1450             report(false, "Found " + links + " references to " + anchors + " anchors "
1451                     + "in " + allFiles.size() + " files and " + allURIs.size() + " other URIs.");
1452             report(!missingFiles.isEmpty(),   "%6d missing files", missingFiles.size());
1453             report(duplicateIds > 0, "%6d duplicate ids", duplicateIds);
1454             report(missingIds > 0,   "%6d missing ids", missingIds);
1455 
1456             Map<String, Integer> schemeCounts = new TreeMap<>();
1457             Map<String, Integer> hostCounts = new TreeMap<>(new HostComparator());
1458             for (URI uri : allURIs.keySet()) {
1459                 String scheme = uri.getScheme();
1460                 if (scheme != null) {
1461                     schemeCounts.put(scheme, schemeCounts.computeIfAbsent(scheme, s -> 0) + 1);
1462                 }
1463                 String host = uri.getHost();
1464                 if (host != null) {
1465                     hostCounts.put(host, hostCounts.computeIfAbsent(host, h -> 0) + 1);
1466                 }
1467             }
1468 
1469             if (schemeCounts.size() > 0) {
1470                 report(false, "Schemes");
1471                 schemeCounts.forEach((s, n) -> report(!isSchemeOK(s), "%6d %s", n, s));
1472             }
1473 
1474             if (hostCounts.size() > 0) {
1475                 report(false, "Hosts");
1476                 hostCounts.forEach((h, n) -> report(false, "%6d %s", n, h));
1477             }
1478         }
1479 
1480         private void report(String message, Object... args) {
1481             out.println(String.format(message, args));
1482         }
1483 
1484         private void report(boolean highlight, String message, Object... args) {
1485             out.print(highlight ? "* " : "  ");
1486             out.println(String.format(message, args));
1487         }
1488 
1489         private void reportMissingFile(Path file) {
1490             report("%s", relativePath(file));
1491             IDTable table = allFiles.get(file);
1492             Set<Path> refs = new TreeSet<>();
1493             for (ID id : table.map.values()) {
1494                 if (id.references != null) {
1495                     for (Position p : id.references) {
1496                         refs.add(p.path);
1497                     }
1498                 }
1499             }
1500             int n = 0;
1501             int MAX_REFS = 10;
1502             for (Path ref : refs) {
1503                 report("    in " + relativePath(ref));
1504                 if (++n == MAX_REFS) {
1505                     report("    ... and %d more", refs.size() - n);
1506                     break;
1507                 }
1508             }
1509         }
1510 
1511         @Override
1512         public void startFile(Path path) {
1513             currFile = path.toAbsolutePath().normalize();
1514             currTable = allFiles.computeIfAbsent(currFile, p -> new IDTable(p));
1515             html5 = false;
1516             files++;
1517         }
1518 
1519         @Override
1520         public void endFile() {
1521             currTable.check();
1522         }
1523 
1524         @Override
1525         public void docType(String doctype) {
1526             html5 = doctype.matches("(?i)<\\?doctype\\s+html>");
1527         }
1528 
1529         @Override @SuppressWarnings("fallthrough")
1530         public void startElement(String name, Map<String, String> attrs, boolean selfClosing) {
1531             int line = getLineNumber();
1532             switch (name) {
1533                 case "a":
1534                     String nameAttr = html5 ? null : attrs.get("name");
1535                     if (nameAttr != null) {
1536                         foundAnchor(line, nameAttr);
1537                     }
1538                     // fallthrough
1539                 case "link":
1540                     String href = attrs.get("href");
1541                     if (href != null) {
1542                         foundReference(line, href);
1543                     }
1544                     break;
1545             }
1546 
1547             String idAttr = attrs.get("id");
1548             if (idAttr != null) {
1549                 foundAnchor(line, idAttr);
1550             }
1551         }
1552 
1553         @Override
1554         public void endElement(String name) { }
1555 
1556         private void foundAnchor(int line, String name) {
1557             currTable.addID(line, name);
1558         }
1559 
1560         private void foundReference(int line, String ref) {
1561             links++;
1562             try {
1563                 URI uri = new URI(ref);
1564                 if (uri.isAbsolute()) {
1565                     foundReference(line, uri);
1566                 } else {
1567                     Path p;
1568                     String uriPath = uri.getPath();
1569                     if (uriPath == null || uriPath.isEmpty()) {
1570                         p = currFile;
1571                     } else {
1572                         p = currFile.getParent().resolve(uriPath).normalize();
1573                     }
1574                     foundReference(line, p, uri.getFragment());
1575                 }
1576             } catch (URISyntaxException e) {
1577                 error(currFile, line, "invalid URI: " + e);
1578             }
1579         }
1580 
1581         private void foundReference(int line, Path p, String fragment) {
1582             IDTable t = allFiles.computeIfAbsent(p, key -> new IDTable(key));
1583             t.addReference(fragment, currFile, line);
1584         }
1585 
1586         private void foundReference(int line, URI uri) {
1587             if (!isSchemeOK(uri.getScheme())) {
1588                 error(currFile, line, "bad scheme in URI");
1589                 badSchemes++;
1590             }
1591 
1592             String fragment = uri.getFragment();
1593             try {
1594                 URI noFrag = new URI(uri.toString().replaceAll("#\\Q" + fragment + "\\E$", ""));
1595                 IDTable t = allURIs.computeIfAbsent(noFrag, key -> new IDTable(key.toString()));
1596                 t.addReference(fragment, currFile, line);
1597             } catch (URISyntaxException e) {
1598                 throw new Error(e);
1599             }
1600         }
1601 
1602         private boolean isSchemeOK(String uriScheme) {
1603             if (uriScheme == null) {
1604                 return true;
1605             }
1606 
1607             switch (uriScheme) {
1608                 case "file":
1609                 case "ftp":
1610                 case "http":
1611                 case "https":
1612                 case "javascript":
1613                 case "mailto":
1614                     return true;
1615 
1616                 default:
1617                     return false;
1618             }
1619         }
1620 
1621         private List<Path> getMissingFiles() {
1622             return allFiles.entrySet().stream()
1623                     .filter(e -> !Files.exists(e.getKey()))
1624                     .map(e -> e.getKey())
1625                     .collect(Collectors.toList());
1626         }
1627 
1628         @Override
1629         protected void error(Path file, int lineNumber, String message) {
1630             super.error(relativePath(file), lineNumber, message);
1631             errors++;
1632         }
1633 
1634         @Override
1635         protected void error(Path file, int lineNumber, Throwable t) {
1636             super.error(relativePath(file), lineNumber, t);
1637             errors++;
1638         }
1639 
1640         private Path relativePath(Path path) {
1641             return path.startsWith(currDir) ? currDir.relativize(path) : path;
1642         }
1643 
1644         /**
1645          * A position in a file, as identified by a file name and line number.
1646          */
1647         static class Position implements Comparable<Position> {
1648             Path path;
1649             int line;
1650 
1651             Position(Path path, int line) {
1652                 this.path = path;
1653                 this.line = line;
1654             }
1655 
1656             @Override
1657             public int compareTo(Position o) {
1658                 int v = path.compareTo(o.path);
1659                 return v != 0 ? v : Integer.compare(line, o.line);
1660             }
1661 
1662             @Override
1663             public boolean equals(Object obj) {
1664                 if (this == obj) {
1665                     return true;
1666                 } else if (obj == null || getClass() != obj.getClass()) {
1667                     return false;
1668                 } else {
1669                     final Position other = (Position) obj;
1670                     return Objects.equals(this.path, other.path)
1671                             && this.line == other.line;
1672                 }
1673             }
1674 
1675             @Override
1676             public int hashCode() {
1677                 return Objects.hashCode(path) * 37 + line;
1678             }
1679         }
1680 
1681         /**
1682          * Infor for an ID within an HTML file, and a set of positions that reference it.
1683          */
1684         static class ID {
1685             boolean declared;
1686             Set<Position> references;
1687 
1688             Set<Position> getReferences() {
1689                 return (references) == null ? Collections.emptySet() : references;
1690             }
1691         }
1692 
1693         /**
1694          * A table for the set of IDs in an HTML file.
1695          */
1696         class IDTable {
1697             private String name;
1698             private boolean checked;
1699             private final Map<String, ID> map = new HashMap<>();
1700 
1701             IDTable(Path p) {
1702                 this(relativePath(p).toString());
1703             }
1704 
1705             IDTable(String name) {
1706                 this.name = name;
1707             }
1708 
1709             void addID(int line, String name) {
1710                 if (checked) {
1711                     throw new IllegalStateException("Adding ID after file has been read");
1712                 }
1713                 Objects.requireNonNull(name);
1714                 ID id = map.computeIfAbsent(name, x -> new ID());
1715                 if (id.declared) {
1716                     error(currFile, line, "name already declared: " + name);
1717                     duplicateIds++;
1718                 } else {
1719                     id.declared = true;
1720                 }
1721             }
1722 
1723             void addReference(String name, Path from, int line) {
1724                 if (checked) {
1725                     if (name != null) {
1726                         ID id = map.get(name);
1727                         if (id == null || !id.declared) {
1728                             error(from, line, "id not found: " + this.name + "#" + name);
1729                         }
1730                     }
1731                 } else {
1732                     ID id = map.computeIfAbsent(name, x -> new ID());
1733                     if (id.references == null) {
1734                         id.references = new TreeSet<>();
1735                     }
1736                     id.references.add(new Position(from, line));
1737                 }
1738             }
1739 
1740             void check() {
1741                 map.forEach((name, id) -> {
1742                     if (name != null && !id.declared) {
1743                         //log.error(currFile, 0, "id not declared: " + name);
1744                         for (Position ref : id.references) {
1745                             error(ref.path, ref.line, "id not found: " + this.name + "#" + name);
1746                         }
1747                         missingIds++;
1748                     }
1749                 });
1750                 checked = true;
1751             }
1752         }
1753 
1754         static class URIComparator implements Comparator<URI> {
1755             final HostComparator hostComparator = new HostComparator();
1756 
1757             @Override
1758             public int compare(URI o1, URI o2) {
1759                 if (o1.isOpaque() || o2.isOpaque()) {
1760                     return o1.compareTo(o2);
1761                 }
1762                 String h1 = o1.getHost();
1763                 String h2 = o2.getHost();
1764                 String s1 = o1.getScheme();
1765                 String s2 = o2.getScheme();
1766                 if (h1 == null || h1.isEmpty() || s1 == null || s1.isEmpty()
1767                         || h2 == null || h2.isEmpty() || s2 == null || s2.isEmpty()) {
1768                     return o1.compareTo(o2);
1769                 }
1770                 int v = hostComparator.compare(h1, h2);
1771                 if (v != 0) {
1772                     return v;
1773                 }
1774                 v = s1.compareTo(s2);
1775                 if (v != 0) {
1776                     return v;
1777                 }
1778                 return o1.compareTo(o2);
1779             }
1780         }
1781 
1782         static class HostComparator implements Comparator<String> {
1783             @Override
1784             public int compare(String h1, String h2) {
1785                 List<String> l1 = new ArrayList<>(Arrays.asList(h1.split("\\.")));
1786                 Collections.reverse(l1);
1787                 String r1 = String.join(".", l1);
1788                 List<String> l2 = new ArrayList<>(Arrays.asList(h2.split("\\.")));
1789                 Collections.reverse(l2);
1790                 String r2 = String.join(".", l2);
1791                 return r1.compareTo(r2);
1792             }
1793         }
1794 
1795     }
1796 }