1 /*
   2  * Copyright (c) 1997, 2016, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 
  26 package jdk.javadoc.internal.doclets.toolkit;
  27 
  28 import java.io.*;
  29 import java.util.*;
  30 import java.util.regex.Matcher;
  31 import java.util.regex.Pattern;
  32 
  33 import javax.lang.model.element.Element;
  34 import javax.lang.model.element.PackageElement;
  35 import javax.lang.model.element.TypeElement;
  36 import javax.tools.JavaFileManager;
  37 import javax.tools.JavaFileObject;
  38 
  39 import com.sun.source.util.DocTreePath;
  40 
  41 import jdk.javadoc.doclet.Doclet;
  42 import jdk.javadoc.doclet.DocletEnvironment;
  43 import jdk.javadoc.doclet.Reporter;
  44 import jdk.javadoc.internal.doclets.toolkit.builders.BuilderFactory;
  45 import jdk.javadoc.internal.doclets.toolkit.taglets.TagletManager;
  46 import jdk.javadoc.internal.doclets.toolkit.util.ClassDocCatalog;
  47 import jdk.javadoc.internal.doclets.toolkit.util.DocFile;
  48 import jdk.javadoc.internal.doclets.toolkit.util.DocletAbortException;
  49 import jdk.javadoc.internal.doclets.toolkit.util.DocletConstants;
  50 import jdk.javadoc.internal.doclets.toolkit.util.Extern;
  51 import jdk.javadoc.internal.doclets.toolkit.util.Group;
  52 import jdk.javadoc.internal.doclets.toolkit.util.MessageRetriever;
  53 import jdk.javadoc.internal.doclets.toolkit.util.MetaKeywords;
  54 import jdk.javadoc.internal.doclets.toolkit.util.Utils;
  55 
  56 import static javax.tools.Diagnostic.Kind.*;
  57 
  58 /**
  59  * Configure the output based on the options. Doclets should sub-class
  60  * Configuration, to configure and add their own options. This class contains
  61  * all user options which are supported by the 1.1 doclet and the standard
  62  * doclet.
  63  *
  64  *  <p><b>This is NOT part of any supported API.
  65  *  If you write code that depends on this, you do so at your own risk.
  66  *  This code and its internal interfaces are subject to change or
  67  *  deletion without notice.</b>
  68  *
  69  * @author Robert Field.
  70  * @author Atul Dambalkar.
  71  * @author Jamie Ho
  72  */
  73 public abstract class Configuration {
  74 
  75     /**
  76      * Exception used to report a problem during setOptions.
  77      */
  78     public static class Fault extends Exception {
  79         private static final long serialVersionUID = 0;
  80 
  81         Fault(String msg) {
  82             super(msg);
  83         }
  84 
  85         Fault(String msg, Exception cause) {
  86             super(msg, cause);
  87         }
  88     }
  89 
  90     /**
  91      * The factory for builders.
  92      */
  93     protected BuilderFactory builderFactory;
  94 
  95     /**
  96      * The taglet manager.
  97      */
  98     public TagletManager tagletManager;
  99 
 100     /**
 101      * The path to the builder XML input file.
 102      */
 103     public String builderXMLPath;
 104 
 105     /**
 106      * The default path to the builder XML.
 107      */
 108     private static final String DEFAULT_BUILDER_XML = "resources/doclet.xml";
 109 
 110     /**
 111      * The path to Taglets
 112      */
 113     public String tagletpath = "";
 114 
 115     /**
 116      * This is true if option "-serialwarn" is used. Defualt value is false to
 117      * suppress excessive warnings about serial tag.
 118      */
 119     public boolean serialwarn = false;
 120 
 121     /**
 122      * The specified amount of space between tab stops.
 123      */
 124     public int sourcetab;
 125 
 126     public String tabSpaces;
 127 
 128     /**
 129      * True if we should generate browsable sources.
 130      */
 131     public boolean linksource = false;
 132 
 133     /**
 134      * True if command line option "-nosince" is used. Default value is
 135      * false.
 136      */
 137     public boolean nosince = false;
 138 
 139     /**
 140      * True if we should recursively copy the doc-file subdirectories
 141      */
 142     public boolean copydocfilesubdirs = false;
 143 
 144     /**
 145      * Maintain backward compatibility with previous javadoc version
 146      */
 147     public boolean backwardCompatibility = true;
 148 
 149     /**
 150      * The META charset tag used for cross-platform viewing.
 151      */
 152     public String charset = "";
 153 
 154     /**
 155      * True if user wants to add member names as meta keywords.
 156      * Set to false because meta keywords are ignored in general
 157      * by most Internet search engines.
 158      */
 159     public boolean keywords = false;
 160 
 161     /**
 162      * The meta tag keywords instance.
 163      */
 164     public final MetaKeywords metakeywords;
 165 
 166     /**
 167      * The list of doc-file subdirectories to exclude
 168      */
 169     protected Set<String> excludedDocFileDirs;
 170 
 171     /**
 172      * The list of qualifiers to exclude
 173      */
 174     protected Set<String> excludedQualifiers;
 175 
 176     /**
 177      * The Root of the generated Program Structure from the Doclet API.
 178      */
 179     public DocletEnvironment root;
 180 
 181     /**
 182      * An utility class for commonly used helpers
 183      */
 184     public Utils utils;
 185 
 186     /**
 187      * All the temporary accessors to javac internals.
 188      */
 189     public WorkArounds workArounds;
 190 
 191     /**
 192      * Destination directory name, in which doclet will generate the entire
 193      * documentation. Default is current directory.
 194      */
 195     public String destDirName = "";
 196 
 197     /**
 198      * Destination directory name, in which doclet will copy the doc-files to.
 199      */
 200     public String docFileDestDirName = "";
 201 
 202     /**
 203      * Encoding for this document. Default is default encoding for this
 204      * platform.
 205      */
 206     public String docencoding = null;
 207 
 208     /**
 209      * True if user wants to suppress descriptions and tags.
 210      */
 211     public boolean nocomment = false;
 212 
 213     /**
 214      * Encoding for this document. Default is default encoding for this
 215      * platform.
 216      */
 217     public String encoding = null;
 218 
 219     /**
 220      * Generate author specific information for all the classes if @author
 221      * tag is used in the doc comment and if -author option is used.
 222      * <code>showauthor</code> is set to true if -author option is used.
 223      * Default is don't show author information.
 224      */
 225     public boolean showauthor = false;
 226 
 227     /**
 228      * Generate documentation for JavaFX getters and setters automatically
 229      * by copying it from the appropriate property definition.
 230      */
 231     public boolean javafx = false;
 232 
 233     /**
 234      * Generate version specific information for the all the classes
 235      * if @version tag is used in the doc comment and if -version option is
 236      * used. <code>showversion</code> is set to true if -version option is
 237      * used.Default is don't show version information.
 238      */
 239     public boolean showversion = false;
 240 
 241     /**
 242      * Sourcepath from where to read the source files. Default is classpath.
 243      *
 244      */
 245     public String sourcepath = "";
 246 
 247     /**
 248      * Don't generate deprecated API information at all, if -nodeprecated
 249      * option is used. <code>nodepracted</code> is set to true if
 250      * -nodeprecated option is used. Default is generate deprected API
 251      * information.
 252      */
 253     public boolean nodeprecated = false;
 254 
 255     /**
 256      * The catalog of classes specified on the command-line
 257      */
 258     public ClassDocCatalog classDocCatalog;
 259 
 260     /**
 261      * Message Retriever for the doclet, to retrieve message from the resource
 262      * file for this Configuration, which is common for 1.1 and standard
 263      * doclets.
 264      *
 265      * TODO:  Make this private!!!
 266      */
 267     public MessageRetriever message = null;
 268 
 269     /**
 270      * True if user wants to suppress time stamp in output.
 271      * Default is false.
 272      */
 273     public boolean notimestamp= false;
 274 
 275     /**
 276      * The package grouping instance.
 277      */
 278     public final Group group = new Group(this);
 279 
 280     /**
 281      * The tracker of external package links.
 282      */
 283     public final Extern extern = new Extern(this);
 284 
 285     public  Reporter reporter;
 286 
 287     public Locale locale;
 288 
 289     /**
 290      * Suppress all messages
 291      */
 292     public boolean quiet = false;
 293 
 294     private String urlForLink;
 295 
 296     private String pkglistUrlForLink;
 297 
 298     private String urlForLinkOffline;
 299 
 300     private String pkglistUrlForLinkOffline;
 301 
 302     private List<GroupContainer> groups;
 303 
 304     /**
 305      * Return the build date for the doclet.
 306      */
 307     public abstract String getDocletSpecificBuildDate();
 308 
 309     /**
 310      * This method should be defined in all those doclets(configurations),
 311      * which want to derive themselves from this Configuration. This method
 312      * can be used to finish up the options setup.
 313      */
 314 
 315     public abstract boolean finishOptionSettings();
 316 
 317     /**
 318      * Return the doclet specific {@link MessageRetriever}
 319      * @return the doclet specific MessageRetriever.
 320      */
 321     public abstract MessageRetriever getDocletSpecificMsg();
 322 
 323     public CommentUtils cmtutils;
 324 
 325     /**
 326      * A sorted set of packages specified on the command-line merged with a
 327      * collection of packages that contain the classes specified on the
 328      * command-line.
 329      */
 330     public SortedSet<PackageElement> packages;
 331 
 332     protected final Set<Doclet.Option> optionsProcessed;
 333 
 334     public final OverviewElement overviewElement;
 335 
 336     /**
 337      * Constructor. Constructs the message retriever with resource file.
 338      */
 339     public Configuration() {
 340         message = new MessageRetriever(this, "jdk.javadoc.internal.doclets.toolkit.resources.doclets");
 341         excludedDocFileDirs = new HashSet<>();
 342         excludedQualifiers = new HashSet<>();
 343         setTabWidth(DocletConstants.DEFAULT_TAB_STOP_LENGTH);
 344         metakeywords = new MetaKeywords(this);
 345         optionsProcessed = new HashSet<>();
 346         groups = new ArrayList<>(0);
 347         overviewElement = new OverviewElement(root);
 348     }
 349 
 350     /**
 351      * Return the builder factory for this doclet.
 352      *
 353      * @return the builder factory for this doclet.
 354      */
 355     public BuilderFactory getBuilderFactory() {
 356         if (builderFactory == null) {
 357             builderFactory = new BuilderFactory(this);
 358         }
 359         return builderFactory;
 360     }
 361 
 362     public Reporter getReporter() {
 363         return this.reporter;
 364     }
 365 
 366     private void initPackages() {
 367         packages = new TreeSet<>(utils.makePackageComparator());
 368         packages.addAll(utils.getSpecifiedPackages());
 369         for (TypeElement aClass : utils.getSpecifiedClasses()) {
 370             packages.add(utils.containingPackage(aClass));
 371         }
 372     }
 373 
 374     public Set<Doclet.Option> getSupportedOptions() {
 375         Doclet.Option[] options = {
 376             new Option(this, "author") {
 377                 @Override
 378                 public boolean process(String opt, ListIterator<String> args) {
 379                     optionsProcessed.add(this);
 380                     showauthor = true;
 381                     return true;
 382                 }
 383             },
 384             new Hidden(this, "classpath", 1) {
 385                 @Override
 386                 public boolean process(String opt, ListIterator<String> args) {
 387                     if (sourcepath.length() == 0) {
 388                         optionsProcessed.add(this);
 389                         sourcepath = args.next();
 390                     }
 391                     return true;
 392                 }
 393             },
 394             new Hidden(this, "cp", 1) {
 395                 @Override
 396                 public boolean process(String opt, ListIterator<String> args) {
 397                     if (sourcepath.length() == 0) {
 398                         optionsProcessed.add(this);
 399                         sourcepath = args.next();
 400                     }
 401                     return true;
 402                 }
 403             },
 404             new Option(this, "d", 1) {
 405                 @Override
 406                 public boolean process(String opt, ListIterator<String> args) {
 407                     optionsProcessed.add(this);
 408                     destDirName = addTrailingFileSep(args.next());
 409                     return true;
 410                 }
 411             },
 412             new Option(this, "docencoding", 1) {
 413                 @Override
 414                 public boolean process(String opt, ListIterator<String> args) {
 415                     optionsProcessed.add(this);
 416                     docencoding = args.next();
 417                     return true;
 418                 }
 419             },
 420             new Option(this, "docfilessubdirs") {
 421                 @Override
 422                 public boolean process(String opt, ListIterator<String> args) {
 423                     optionsProcessed.add(this);
 424                     copydocfilesubdirs = true;
 425                     return true;
 426                 }
 427             },
 428             new Hidden(this, "encoding", 1) {
 429                 @Override
 430                 public boolean process(String opt, ListIterator<String> args) {
 431                     optionsProcessed.add(this);
 432                     encoding = args.next();
 433                     return true;
 434                 }
 435             },
 436             new Option(this, "excludedocfilessubdir", 1) {
 437                 @Override
 438                 public boolean process(String opt, ListIterator<String> args) {
 439                     optionsProcessed.add(this);
 440                     addToSet(excludedDocFileDirs, args.next());
 441                     return true;
 442                 }
 443             },
 444             new Option(this, "group", 2) {
 445                 @Override
 446                 public boolean process(String opt, ListIterator<String> args) {
 447                     optionsProcessed.add(this);
 448                     groups.add(new GroupContainer(args.next(), args.next()));
 449                     return true;
 450                 }
 451             },
 452             new Hidden(this, "javafx") {
 453                 @Override
 454                 public boolean process(String opt, ListIterator<String> args) {
 455                     optionsProcessed.add(this);
 456                     javafx = true;
 457                     return true;
 458                 }
 459             },
 460             new Option(this, "keywords") {
 461                 @Override
 462                 public boolean process(String opt, ListIterator<String> args) {
 463                     optionsProcessed.add(this);
 464                     keywords = true;
 465                     return true;
 466                 }
 467             },
 468             new Option(this, "link", 1) {
 469                 @Override
 470                 public boolean process(String opt, ListIterator<String> args) {
 471                     optionsProcessed.add(this);
 472                     urlForLink = args.next();
 473                     pkglistUrlForLink = urlForLink;
 474                     return true;
 475                 }
 476             },
 477             new Option(this, "linksource") {
 478                 @Override
 479                 public boolean process(String opt, ListIterator<String> args) {
 480                     optionsProcessed.add(this);
 481                     linksource = true;
 482                     return true;
 483                 }
 484             },
 485             new Option(this, "linkoffline", 2) {
 486                 @Override
 487                 public boolean process(String opt, ListIterator<String> args) {
 488                     optionsProcessed.add(this);
 489                     urlForLinkOffline = args.next();
 490                     pkglistUrlForLinkOffline = args.next();
 491                     return true;
 492                 }
 493             },
 494             new Option(this, "nocomment") {
 495                 @Override
 496                 public boolean process(String opt, ListIterator<String> args) {
 497                     optionsProcessed.add(this);
 498                     nocomment = true;
 499                     return true;
 500                 }
 501             },
 502             new Option(this, "nodeprecated") {
 503                 @Override
 504                 public boolean process(String opt, ListIterator<String> args) {
 505                     optionsProcessed.add(this);
 506                     nodeprecated = true;
 507                     return true;
 508                 }
 509             },
 510             new Option(this, "nosince") {
 511                 @Override
 512                 public boolean process(String opt, ListIterator<String> args) {
 513                     optionsProcessed.add(this);
 514                     nosince = true;
 515                     return true;
 516                 }
 517             },
 518             new Option(this, "notimestamp") {
 519                 @Override
 520                 public boolean process(String opt, ListIterator<String> args) {
 521                     optionsProcessed.add(this);
 522                     notimestamp = true;
 523                     return true;
 524                 }
 525             },
 526             new Option(this, "noqualifier", 1) {
 527                 @Override
 528                 public boolean process(String opt, ListIterator<String> args) {
 529                     optionsProcessed.add(this);
 530                     addToSet(excludedQualifiers, args.next());
 531                     return true;
 532                 }
 533             },
 534             new Hidden(this, "quiet") {
 535                 @Override
 536                 public boolean process(String opt, ListIterator<String> args) {
 537                     optionsProcessed.add(this);
 538                     quiet = true;
 539                     return true;
 540                 }
 541             },
 542             new Option(this, "serialwarn") {
 543                 @Override
 544                 public boolean process(String opt, ListIterator<String> args) {
 545                     optionsProcessed.add(this);
 546                     serialwarn = true;
 547                     return true;
 548                 }
 549             },
 550             new Hidden(this, "sourcepath", 1) {
 551                 @Override
 552                 public boolean process(String opt, ListIterator<String> args) {
 553                     optionsProcessed.add(this);
 554                     sourcepath = args.next();
 555                     return true;
 556                 }
 557             },
 558             new Option(this, "sourcetab", 1) {
 559                 @Override
 560                 public boolean process(String opt, ListIterator<String> args) {
 561                     optionsProcessed.add(this);
 562                     linksource = true;
 563                     try {
 564                         setTabWidth(Integer.parseInt(args.next()));
 565                     } catch (NumberFormatException e) {
 566                              //Set to -1 so that warning will be printed
 567                         //to indicate what is valid argument.
 568                         sourcetab = -1;
 569                     }
 570                     if (sourcetab <= 0) {
 571                         message.warning("doclet.sourcetab_warning");
 572                         setTabWidth(DocletConstants.DEFAULT_TAB_STOP_LENGTH);
 573                     }
 574                     return true;
 575                 }
 576             },
 577             new Option(this, "tag", 1) {
 578                 @Override
 579                 public boolean process(String opt, ListIterator<String> args) {
 580                     optionsProcessed.add(this);
 581                     ArrayList<String> list = new ArrayList<>();
 582                     list.add(opt);
 583                     list.add(args.next());
 584                     customTagStrs.add(list);
 585                     return true;
 586                 }
 587             },
 588              new Option(this, "taglet", 1) {
 589                 @Override
 590                 public boolean process(String opt, ListIterator<String> args) {
 591                     optionsProcessed.add(this);
 592                     ArrayList<String> list = new ArrayList<>();
 593                     list.add(opt);
 594                     list.add(args.next());
 595                     customTagStrs.add(list);
 596                     return true;
 597                 }
 598             },
 599             new Option(this, "tagletpath", 1) {
 600                 @Override
 601                 public boolean process(String opt, ListIterator<String> args) {
 602                     optionsProcessed.add(this);
 603                     tagletpath = args.next();
 604                     return true;
 605                 }
 606             },
 607             new Option(this, "version") {
 608                 @Override
 609                 public boolean process(String opt, ListIterator<String> args) {
 610                     optionsProcessed.add(this);
 611                     showversion = true;
 612                     return true;
 613                 }
 614             }
 615         };
 616         Set<Doclet.Option> set = new TreeSet<>();
 617         set.addAll(Arrays.asList(options));
 618         return set;
 619     }
 620 
 621     final LinkedHashSet<List<String>> customTagStrs = new LinkedHashSet<>();
 622 
 623     /*
 624      * when this is called all the option have been set, this method,
 625      * initializes certain components before anything else is started.
 626      */
 627     private void finishOptionSettings0() throws Fault {
 628         ensureOutputDirExists();
 629         if (urlForLink != null && pkglistUrlForLink != null)
 630             extern.link(urlForLink, pkglistUrlForLink, reporter, false);
 631         if (urlForLinkOffline != null && pkglistUrlForLinkOffline != null)
 632             extern.link(urlForLinkOffline, pkglistUrlForLinkOffline, reporter, true);
 633         if (sourcepath.length() == 0) {
 634             sourcepath = System.getProperty("env.class.path", "");
 635         }
 636         if (docencoding == null) {
 637             docencoding = encoding;
 638         }
 639         classDocCatalog = new ClassDocCatalog(utils.getSpecifiedClasses(), this);
 640         initTagletManager(customTagStrs);
 641         groups.stream().forEach((grp) -> {
 642             group.checkPackageGroups(grp.value1, grp.value2);
 643         });
 644     }
 645 
 646     /**
 647      * Set the command line options supported by this configuration.
 648      *
 649      * @return
 650      * @throws DocletAbortException
 651      */
 652     public boolean setOptions() {
 653         try {
 654             initPackages();
 655             finishOptionSettings0();
 656             if (!finishOptionSettings())
 657                 return false;
 658 
 659         } catch (Fault f) {
 660             throw new DocletAbortException(f.getMessage());
 661         } catch (DocletAbortException e) {
 662             throw new DocletAbortException(e);
 663         }
 664         return true;
 665     }
 666 
 667     private void ensureOutputDirExists() throws Fault {
 668         DocFile destDir = DocFile.createFileForDirectory(this, destDirName);
 669         if (!destDir.exists()) {
 670             //Create the output directory (in case it doesn't exist yet)
 671             if (!destDirName.isEmpty())
 672                 reporter.print(NOTE, getText("doclet.dest_dir_create", destDirName));
 673             destDir.mkdirs();
 674         } else if (!destDir.isDirectory()) {
 675             throw new Fault(getText(
 676                 "doclet.destination_directory_not_directory_0",
 677                 destDir.getPath()));
 678         } else if (!destDir.canWrite()) {
 679             throw new Fault(getText(
 680                 "doclet.destination_directory_not_writable_0",
 681                 destDir.getPath()));
 682         }
 683     }
 684 
 685     /**
 686      * Initialize the taglet manager.  The strings to initialize the simple custom tags should
 687      * be in the following format:  "[tag name]:[location str]:[heading]".
 688      * @param customTagStrs the set two dimensional arrays of strings.  These arrays contain
 689      * either -tag or -taglet arguments.
 690      */
 691     private void initTagletManager(Set<List<String>> customTagStrs) {
 692         tagletManager = tagletManager == null ?
 693             new TagletManager(nosince, showversion, showauthor, javafx, message) :
 694             tagletManager;
 695         for (List<String> args : customTagStrs) {
 696             if (args.get(0).equals("-taglet")) {
 697                 tagletManager.addCustomTag(args.get(1), getFileManager(), tagletpath);
 698                 continue;
 699             }
 700             List<String> tokens = tokenize(args.get(1), TagletManager.SIMPLE_TAGLET_OPT_SEPARATOR, 3);
 701             if (tokens.size() == 1) {
 702                 String tagName = args.get(1);
 703                 if (tagletManager.isKnownCustomTag(tagName)) {
 704                     //reorder a standard tag
 705                     tagletManager.addNewSimpleCustomTag(tagName, null, "");
 706                 } else {
 707                     //Create a simple tag with the heading that has the same name as the tag.
 708                     StringBuilder heading = new StringBuilder(tagName + ":");
 709                     heading.setCharAt(0, Character.toUpperCase(tagName.charAt(0)));
 710                     tagletManager.addNewSimpleCustomTag(tagName, heading.toString(), "a");
 711                 }
 712             } else if (tokens.size() == 2) {
 713                 //Add simple taglet without heading, probably to excluding it in the output.
 714                 tagletManager.addNewSimpleCustomTag(tokens.get(0), tokens.get(1), "");
 715             } else if (tokens.size() >= 3) {
 716                 tagletManager.addNewSimpleCustomTag(tokens.get(0), tokens.get(2), tokens.get(1));
 717             } else {
 718                 message.error("doclet.Error_invalid_custom_tag_argument", args.get(1));
 719             }
 720         }
 721     }
 722 
 723     /**
 724      * Given a string, return an array of tokens.  The separator can be escaped
 725      * with the '\' character.  The '\' character may also be escaped by the
 726      * '\' character.
 727      *
 728      * @param s         the string to tokenize.
 729      * @param separator the separator char.
 730      * @param maxTokens the maximum number of tokens returned.  If the
 731      *                  max is reached, the remaining part of s is appended
 732      *                  to the end of the last token.
 733      *
 734      * @return an array of tokens.
 735      */
 736     private List<String> tokenize(String s, char separator, int maxTokens) {
 737         List<String> tokens = new ArrayList<>();
 738         StringBuilder  token = new StringBuilder ();
 739         boolean prevIsEscapeChar = false;
 740         for (int i = 0; i < s.length(); i += Character.charCount(i)) {
 741             int currentChar = s.codePointAt(i);
 742             if (prevIsEscapeChar) {
 743                 // Case 1:  escaped character
 744                 token.appendCodePoint(currentChar);
 745                 prevIsEscapeChar = false;
 746             } else if (currentChar == separator && tokens.size() < maxTokens-1) {
 747                 // Case 2:  separator
 748                 tokens.add(token.toString());
 749                 token = new StringBuilder();
 750             } else if (currentChar == '\\') {
 751                 // Case 3:  escape character
 752                 prevIsEscapeChar = true;
 753             } else {
 754                 // Case 4:  regular character
 755                 token.appendCodePoint(currentChar);
 756             }
 757         }
 758         if (token.length() > 0) {
 759             tokens.add(token.toString());
 760         }
 761         return tokens;
 762     }
 763 
 764     private void addToSet(Set<String> s, String str){
 765         StringTokenizer st = new StringTokenizer(str, ":");
 766         String current;
 767         while(st.hasMoreTokens()){
 768             current = st.nextToken();
 769             s.add(current);
 770         }
 771     }
 772 
 773     /**
 774      * Add a trailing file separator, if not found. Remove superfluous
 775      * file separators if any. Preserve the front double file separator for
 776      * UNC paths.
 777      *
 778      * @param path Path under consideration.
 779      * @return String Properly constructed path string.
 780      */
 781     public static String addTrailingFileSep(String path) {
 782         String fs = System.getProperty("file.separator");
 783         String dblfs = fs + fs;
 784         int indexDblfs;
 785         while ((indexDblfs = path.indexOf(dblfs, 1)) >= 0) {
 786             path = path.substring(0, indexDblfs) +
 787                 path.substring(indexDblfs + fs.length());
 788         }
 789         if (!path.endsWith(fs))
 790             path += fs;
 791         return path;
 792     }
 793 
 794     /**
 795      * This checks for the validity of the options used by the user.
 796      * This works exactly like DocErrorReporter. This will validate the options which are shared
 797      * by our doclets. For example, this method will flag an error using
 798      * the DocErrorReporter if user has used "-nohelp" and "-helpfile" option
 799      * together.
 800      *
 801      * @return true if all the options are valid.
 802      */
 803     public boolean generalValidOptions() {
 804         boolean docencodingfound = false;
 805         for (Doclet.Option opt : optionsProcessed) {
 806             if (opt.matches("-docencoding")) {
 807                 docencodingfound = true;
 808                 if (!checkOutputFileEncoding(docencoding)) {
 809                     return false;
 810                 }
 811             };
 812         }
 813         if (!docencodingfound && (encoding != null && !encoding.isEmpty())) {
 814             if (!checkOutputFileEncoding(encoding)) {
 815                 return false;
 816             }
 817         }
 818         return true;
 819     }
 820 
 821     /**
 822      * Check the validity of the given Source or Output File encoding on this
 823      * platform.
 824      *
 825      * @param docencoding output file encoding.
 826      * @param reporter    used to report errors.
 827      */
 828     private boolean checkOutputFileEncoding(String docencoding) {
 829         OutputStream ost= new ByteArrayOutputStream();
 830         OutputStreamWriter osw = null;
 831         try {
 832             osw = new OutputStreamWriter(ost, docencoding);
 833         } catch (UnsupportedEncodingException exc) {
 834             reporter.print(ERROR, getText("doclet.Encoding_not_supported", docencoding));
 835             return false;
 836         } finally {
 837             try {
 838                 if (osw != null) {
 839                     osw.close();
 840                 }
 841             } catch (IOException exc) {
 842             }
 843         }
 844         return true;
 845     }
 846 
 847     /**
 848      * Return true if the given doc-file subdirectory should be excluded and
 849      * false otherwise.
 850      * @param docfilesubdir the doc-files subdirectory to check.
 851      * @return
 852      */
 853     public boolean shouldExcludeDocFileDir(String docfilesubdir){
 854         return excludedDocFileDirs.contains(docfilesubdir);
 855     }
 856 
 857     /**
 858      * Return true if the given qualifier should be excluded and false otherwise.
 859      * @param qualifier the qualifier to check.
 860      */
 861     public boolean shouldExcludeQualifier(String qualifier){
 862         if (excludedQualifiers.contains("all") ||
 863             excludedQualifiers.contains(qualifier) ||
 864             excludedQualifiers.contains(qualifier + ".*")) {
 865             return true;
 866         } else {
 867             int index = -1;
 868             while ((index = qualifier.indexOf(".", index + 1)) != -1) {
 869                 if (excludedQualifiers.contains(qualifier.substring(0, index + 1) + "*")) {
 870                     return true;
 871                 }
 872             }
 873             return false;
 874         }
 875     }
 876 
 877     /**
 878      * Return the qualified name of the Element if it's qualifier is not excluded.
 879      * Otherwise return the unqualified Element name.
 880      * @param te the TypeElement to check.
 881      * @return the class name
 882      */
 883     public String getClassName(TypeElement te) {
 884         PackageElement pkg = utils.containingPackage(te);
 885         return shouldExcludeQualifier(utils.getPackageName(pkg))
 886                 ? utils.getSimpleName(te)
 887                 : utils.getFullyQualifiedName(te);
 888     }
 889 
 890     public String getText(String key) {
 891         try {
 892             //Check the doclet specific properties file.
 893             return getDocletSpecificMsg().getText(key);
 894         } catch (Exception e) {
 895             //Check the shared properties file.
 896             return message.getText(key);
 897         }
 898     }
 899 
 900     public String getText(String key, String a1) {
 901         try {
 902             //Check the doclet specific properties file.
 903             return getDocletSpecificMsg().getText(key, a1);
 904         } catch (MissingResourceException e) {
 905             //Check the shared properties file.
 906             return message.getText(key, a1);
 907         }
 908     }
 909 
 910     public String getText(String key, String a1, String a2) {
 911         try {
 912             //Check the doclet specific properties file.
 913             return getDocletSpecificMsg().getText(key, a1, a2);
 914         } catch (MissingResourceException e) {
 915             //Check the shared properties file.
 916             return message.getText(key, a1, a2);
 917         }
 918     }
 919 
 920     public String getText(String key, String a1, String a2, String a3) {
 921         try {
 922             //Check the doclet specific properties file.
 923             return getDocletSpecificMsg().getText(key, a1, a2, a3);
 924         } catch (MissingResourceException e) {
 925             //Check the shared properties file.
 926             return message.getText(key, a1, a2, a3);
 927         }
 928     }
 929 
 930     public abstract Content newContent();
 931 
 932     /**
 933      * Get the configuration string as a content.
 934      *
 935      * @param key the key to look for in the configuration file
 936      * @return a content tree for the text
 937      */
 938     public Content getResource(String key) {
 939         Content c = newContent();
 940         c.addContent(getText(key));
 941         return c;
 942     }
 943 
 944     /**
 945      * Get the configuration string as a content.
 946      *
 947      * @param key the key to look for in the configuration file
 948      * @param o   string or content argument added to configuration text
 949      * @return a content tree for the text
 950      */
 951     public Content getResource(String key, Object o) {
 952         return getResource(key, o, null, null);
 953     }
 954 
 955     /**
 956      * Get the configuration string as a content.
 957      *
 958      * @param key the key to look for in the configuration file
 959      * @param o1 resource argument
 960      * @param o2 resource argument
 961      * @return a content tree for the text
 962      */
 963     public Content getResource(String key, Object o1, Object o2) {
 964         return getResource(key, o1, o2, null);
 965     }
 966 
 967     /**
 968      * Get the configuration string as a content.
 969      *
 970      * @param key the key to look for in the configuration file
 971      * @param o0  string or content argument added to configuration text
 972      * @param o1  string or content argument added to configuration text
 973      * @param o2  string or content argument added to configuration text
 974      * @return a content tree for the text
 975      */
 976     public Content getResource(String key, Object o0, Object o1, Object o2) {
 977         Content c = newContent();
 978         Pattern p = Pattern.compile("\\{([012])\\}");
 979         String text = getText(key);
 980         Matcher m = p.matcher(text);
 981         int start = 0;
 982         while (m.find(start)) {
 983             c.addContent(text.substring(start, m.start()));
 984 
 985             Object o = null;
 986             switch (m.group(1).charAt(0)) {
 987                 case '0': o = o0; break;
 988                 case '1': o = o1; break;
 989                 case '2': o = o2; break;
 990             }
 991 
 992             if (o == null) {
 993                 c.addContent("{" + m.group(1) + "}");
 994             } else if (o instanceof String) {
 995                 c.addContent((String) o);
 996             } else if (o instanceof Content) {
 997                 c.addContent((Content) o);
 998             }
 999 
1000             start = m.end();
1001         }
1002 
1003         c.addContent(text.substring(start));
1004         return c;
1005     }
1006 
1007 
1008     /**
1009      * Return true if the TypeElement element is getting documented, depending upon
1010      * -nodeprecated option and the deprecation information. Return true if
1011      * -nodeprecated is not used. Return false if -nodeprecated is used and if
1012      * either TypeElement element is deprecated or the containing package is deprecated.
1013      *
1014      * @param te the TypeElement for which the page generation is checked
1015      * @return boolean
1016      */
1017     public boolean isGeneratedDoc(TypeElement te) {
1018         if (!nodeprecated) {
1019             return true;
1020         }
1021         return !(utils.isDeprecated(te) || utils.isDeprecated(utils.containingPackage(te)));
1022     }
1023 
1024     /**
1025      * Return the doclet specific instance of a writer factory.
1026      * @return the {@link WriterFactory} for the doclet.
1027      */
1028     public abstract WriterFactory getWriterFactory();
1029 
1030     /**
1031      * Return the input stream to the builder XML.
1032      *
1033      * @return the input steam to the builder XML.
1034      * @throws FileNotFoundException when the given XML file cannot be found.
1035      */
1036     public InputStream getBuilderXML() throws IOException {
1037         return builderXMLPath == null ?
1038             Configuration.class.getResourceAsStream(DEFAULT_BUILDER_XML) :
1039             DocFile.createFileForInput(this, builderXMLPath).openInputStream();
1040     }
1041 
1042     /**
1043      * Return the Locale for this document.
1044      * @return locale
1045      */
1046     public abstract Locale getLocale();
1047 
1048     /**
1049      * Return the path of the overview file and null if it does not exist.
1050      *
1051      * @return the path of the overview file and null if it does not exist.
1052      */
1053     public abstract JavaFileObject getOverviewPath();
1054 
1055     /**
1056      * Return the current file manager.
1057      * @return JavaFileManager
1058      */
1059     public abstract JavaFileManager getFileManager();
1060 
1061     private void setTabWidth(int n) {
1062         sourcetab = n;
1063         tabSpaces = String.format("%" + n + "s", "");
1064     }
1065 
1066     public abstract boolean showMessage(DocTreePath path, String key);
1067 
1068     public abstract boolean showMessage(Element e, String key);
1069 
1070     public static abstract class Option implements Doclet.Option, Comparable<Option> {
1071         private final String name;
1072         private final String parameters;
1073         private final String description;
1074         private final int argCount;
1075 
1076         protected final Configuration c;
1077 
1078         protected Option(Configuration config, String keyName, String name, int argCount) {
1079             c = config;
1080             String key = keyName + "name";
1081             String oname = getOptionsMessage(key);
1082             if (oname.isEmpty()) {
1083                 this.name = name;
1084                 this.parameters = "<MISSING KEY>";
1085                 this.description = "<MISSING KEY>";
1086             } else {
1087                 this.name = oname;
1088                 this.parameters = getOptionsMessage(keyName + "parameters");
1089                 this.description = getOptionsMessage(keyName + "description");
1090             }
1091             this.argCount = argCount;
1092         }
1093 
1094         protected Option(String prefix, Configuration config, String name, int argCount) {
1095             this(config, prefix + name.toLowerCase() + ".", name, argCount);
1096         }
1097 
1098         protected Option(Configuration config, String name, int argCount) {
1099             this("doclet.usage.", config,  name, argCount);
1100         }
1101 
1102         protected Option(Configuration config, String name) {
1103             this(config, name, 0);
1104         }
1105 
1106         private String getOptionsMessage(String key) {
1107             try {
1108                 return c.getDocletSpecificMsg().getText(key, (Object[]) null);
1109             } catch (MissingResourceException ignore) {
1110                 return "";
1111             }
1112         }
1113 
1114         @Override
1115         public String getDescription() {
1116             return description;
1117         }
1118 
1119         @Override
1120         public Option.Kind getKind() {
1121             return Doclet.Option.Kind.STANDARD;
1122         }
1123 
1124         @Override
1125         public String getName() {
1126             return name;
1127         }
1128 
1129         @Override
1130         public String getParameters() {
1131             return parameters;
1132         }
1133 
1134         /**
1135          * Maintains the formatting for javadoc -help. Note the space
1136          * alignment.
1137          */
1138         @Override
1139         public String toString() {
1140             String opt = name + " " + parameters;
1141             int optlen = opt.length();
1142             int spaces = 32 - optlen;
1143             StringBuffer sb = new StringBuffer("  -").append(opt);
1144             for (int i = 0; i < spaces; i++) {
1145                 sb.append(" ");
1146             }
1147             sb.append(description);
1148             return sb.toString();
1149         }
1150 
1151         @Override
1152         public int getArgumentCount() {
1153             return argCount;
1154         }
1155 
1156         @Override
1157         public boolean matches(String option) {
1158             String arg = option.startsWith("-") ? option.substring(1) : option;
1159             return name.toLowerCase().equals(arg.toLowerCase());
1160         }
1161 
1162         @Override
1163         public int compareTo(Option that) {
1164             return this.getName().compareTo(that.getName());
1165         }
1166     }
1167 
1168     public abstract class XOption extends Option {
1169 
1170         public XOption(Configuration config, String keyname, String name, int argCount) {
1171             super(config, keyname, name, argCount);
1172         }
1173 
1174         public XOption(Configuration config, String name, int argCount) {
1175             super("doclet.xusage.", config, name, argCount);
1176         }
1177 
1178         public XOption(Configuration config, String name) {
1179             this(config, name, 0);
1180         }
1181 
1182         @Override
1183         public Option.Kind getKind() {
1184             return Doclet.Option.Kind.EXTENDED;
1185         }
1186     }
1187 
1188     public abstract class Hidden extends Option {
1189 
1190         public Hidden(Configuration config, String name, int argCount) {
1191             super("doclet.xusage.", config, name, argCount);
1192         }
1193 
1194         public Hidden(Configuration config, String name) {
1195             this(config, name, 0);
1196         }
1197 
1198         @Override
1199         public Option.Kind getKind() {
1200             return Doclet.Option.Kind.OTHER;
1201         }
1202     }
1203 
1204     /*
1205      * Stores a pair of Strings.
1206      */
1207     protected static class GroupContainer {
1208         final String value1;
1209         final String value2;
1210         public GroupContainer(String value1, String value2) {
1211             this.value1 = value1;
1212             this.value2 = value2;
1213         }
1214     }
1215 }