1 /*
   2  * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 
  26 package build.tools.cldrconverter;
  27 
  28 import static build.tools.cldrconverter.Bundle.jreTimeZoneNames;
  29 import build.tools.cldrconverter.BundleGenerator.BundleType;
  30 import java.io.File;
  31 import java.io.IOException;
  32 import java.io.UncheckedIOException;
  33 import java.nio.file.*;
  34 import java.time.*;
  35 import java.util.*;
  36 import java.util.ResourceBundle.Control;
  37 import java.util.logging.Level;
  38 import java.util.logging.Logger;
  39 import java.util.stream.Collectors;
  40 import java.util.stream.IntStream;
  41 import java.util.stream.Stream;
  42 import javax.xml.parsers.SAXParser;
  43 import javax.xml.parsers.SAXParserFactory;
  44 import org.xml.sax.SAXNotRecognizedException;
  45 import org.xml.sax.SAXNotSupportedException;
  46 
  47 
  48 /**
  49  * Converts locale data from "Locale Data Markup Language" format to
  50  * JRE resource bundle format. LDML is the format used by the Common
  51  * Locale Data Repository maintained by the Unicode Consortium.
  52  */
  53 public class CLDRConverter {
  54 
  55     static final String LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldml.dtd";
  56     static final String SPPL_LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldmlSupplemental.dtd";
  57     static final String BCP47_LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldmlBCP47.dtd";
  58 
  59 
  60     private static String CLDR_BASE;
  61     static String LOCAL_LDML_DTD;
  62     static String LOCAL_SPPL_LDML_DTD;
  63     static String LOCAL_BCP47_LDML_DTD;
  64     private static String SOURCE_FILE_DIR;
  65     private static String SPPL_SOURCE_FILE;
  66     private static String SPPL_META_SOURCE_FILE;
  67     private static String NUMBERING_SOURCE_FILE;
  68     private static String METAZONES_SOURCE_FILE;
  69     private static String LIKELYSUBTAGS_SOURCE_FILE;
  70     private static String TIMEZONE_SOURCE_FILE;
  71     static String DESTINATION_DIR = "build/gensrc";
  72 
  73     static final String LOCALE_NAME_PREFIX = "locale.displayname.";
  74     static final String LOCALE_SEPARATOR = LOCALE_NAME_PREFIX + "separator";
  75     static final String LOCALE_KEYTYPE = LOCALE_NAME_PREFIX + "keytype";
  76     static final String LOCALE_KEY_PREFIX = LOCALE_NAME_PREFIX + "key.";
  77     static final String LOCALE_TYPE_PREFIX = LOCALE_NAME_PREFIX + "type.";
  78     static final String LOCALE_TYPE_PREFIX_CA = LOCALE_TYPE_PREFIX + "ca.";
  79     static final String CURRENCY_SYMBOL_PREFIX = "currency.symbol.";
  80     static final String CURRENCY_NAME_PREFIX = "currency.displayname.";
  81     static final String CALENDAR_NAME_PREFIX = "calendarname.";
  82     static final String CALENDAR_FIRSTDAY_PREFIX = "firstDay.";
  83     static final String CALENDAR_MINDAYS_PREFIX = "minDays.";
  84     static final String TIMEZONE_ID_PREFIX = "timezone.id.";
  85     static final String ZONE_NAME_PREFIX = "timezone.displayname.";
  86     static final String METAZONE_ID_PREFIX = "metazone.id.";
  87     static final String PARENT_LOCALE_PREFIX = "parentLocale.";
  88 
  89     private static SupplementDataParseHandler handlerSuppl;
  90     private static SupplementalMetadataParseHandler handlerSupplMeta;
  91     private static LikelySubtagsParseHandler handlerLikelySubtags;
  92     static NumberingSystemsParseHandler handlerNumbering;
  93     static MetaZonesParseHandler handlerMetaZones;
  94     static TimeZoneParseHandler handlerTimeZone;
  95     private static BundleGenerator bundleGenerator;
  96 
  97     // java.base module related
  98     static boolean isBaseModule = false;
  99     static final Set<Locale> BASE_LOCALES = new HashSet<>();
 100 
 101     // "parentLocales" map
 102     private static final Map<String, SortedSet<String>> parentLocalesMap = new HashMap<>();
 103     private static final ResourceBundle.Control defCon =
 104         ResourceBundle.Control.getControl(ResourceBundle.Control.FORMAT_DEFAULT);
 105 
 106     private static final String[] AVAILABLE_TZIDS = TimeZone.getAvailableIDs();
 107     private static String zoneNameTempFile;
 108     private static String tzDataDir;
 109 
 110     static enum DraftType {
 111         UNCONFIRMED,
 112         PROVISIONAL,
 113         CONTRIBUTED,
 114         APPROVED;
 115 
 116         private static final Map<String, DraftType> map = new HashMap<>();
 117         static {
 118             for (DraftType dt : values()) {
 119                 map.put(dt.getKeyword(), dt);
 120             }
 121         }
 122         static private DraftType defaultType = CONTRIBUTED;
 123 
 124         private final String keyword;
 125 
 126         private DraftType() {
 127             keyword = this.name().toLowerCase(Locale.ROOT);
 128 
 129         }
 130 
 131         static DraftType forKeyword(String keyword) {
 132             return map.get(keyword);
 133         }
 134 
 135         static DraftType getDefault() {
 136             return defaultType;
 137         }
 138 
 139         static void setDefault(String keyword) {
 140             defaultType = Objects.requireNonNull(forKeyword(keyword));
 141         }
 142 
 143         String getKeyword() {
 144             return keyword;
 145         }
 146     }
 147 
 148     static boolean USE_UTF8 = false;
 149     private static boolean verbose;
 150 
 151     private CLDRConverter() {
 152        // no instantiation
 153     }
 154 
 155     @SuppressWarnings("AssignmentToForLoopParameter")
 156     public static void main(String[] args) throws Exception {
 157         if (args.length != 0) {
 158             String currentArg = null;
 159             try {
 160                 for (int i = 0; i < args.length; i++) {
 161                     currentArg = args[i];
 162                     switch (currentArg) {
 163                     case "-draft":
 164                         String draftDataType = args[++i];
 165                         try {
 166                             DraftType.setDefault(draftDataType);
 167                         } catch (NullPointerException e) {
 168                             severe("Error: incorrect draft value: %s%n", draftDataType);
 169                             System.exit(1);
 170                         }
 171                         info("Using the specified data type: %s%n", draftDataType);
 172                         break;
 173 
 174                     case "-base":
 175                         // base directory for input files
 176                         CLDR_BASE = args[++i];
 177                         if (!CLDR_BASE.endsWith("/")) {
 178                             CLDR_BASE += "/";
 179                         }
 180                         break;
 181 
 182                     case "-baselocales":
 183                         // base locales
 184                         setupBaseLocales(args[++i]);
 185                         break;
 186 
 187                     case "-basemodule":
 188                         // indicates java.base module resource generation
 189                         isBaseModule = true;
 190                         break;
 191 
 192                     case "-o":
 193                         // output directory
 194                         DESTINATION_DIR = args[++i];
 195                         break;
 196 
 197                     case "-utf8":
 198                         USE_UTF8 = true;
 199                         break;
 200 
 201                     case "-verbose":
 202                         verbose = true;
 203                         break;
 204 
 205                     case "-zntempfile":
 206                         zoneNameTempFile = args[++i];
 207                         break;
 208 
 209                     case "-tzdatadir":
 210                         tzDataDir = args[++i];
 211                         break;
 212 
 213                     case "-help":
 214                         usage();
 215                         System.exit(0);
 216                         break;
 217 
 218                     default:
 219                         throw new RuntimeException();
 220                     }
 221                 }
 222             } catch (RuntimeException e) {
 223                 severe("unknown or imcomplete arg(s): " + currentArg);
 224                 usage();
 225                 System.exit(1);
 226             }
 227         }
 228 
 229         // Set up path names
 230         LOCAL_LDML_DTD = CLDR_BASE + "/dtd/ldml.dtd";
 231         LOCAL_SPPL_LDML_DTD = CLDR_BASE + "/dtd/ldmlSupplemental.dtd";
 232         LOCAL_BCP47_LDML_DTD = CLDR_BASE + "/dtd/ldmlBCP47.dtd";
 233         SOURCE_FILE_DIR = CLDR_BASE + "/main";
 234         SPPL_SOURCE_FILE = CLDR_BASE + "/supplemental/supplementalData.xml";
 235         LIKELYSUBTAGS_SOURCE_FILE = CLDR_BASE + "/supplemental/likelySubtags.xml";
 236         NUMBERING_SOURCE_FILE = CLDR_BASE + "/supplemental/numberingSystems.xml";
 237         METAZONES_SOURCE_FILE = CLDR_BASE + "/supplemental/metaZones.xml";
 238         TIMEZONE_SOURCE_FILE = CLDR_BASE + "/bcp47/timezone.xml";
 239         SPPL_META_SOURCE_FILE = CLDR_BASE + "/supplemental/supplementalMetadata.xml";
 240 
 241         if (BASE_LOCALES.isEmpty()) {
 242             setupBaseLocales("en-US");
 243         }
 244 
 245         bundleGenerator = new ResourceBundleGenerator();
 246 
 247         // Parse data independent of locales
 248         parseSupplemental();
 249         parseBCP47();
 250 
 251         List<Bundle> bundles = readBundleList();
 252         convertBundles(bundles);
 253 
 254         // Generate java.time.format.ZoneName.java
 255         if (isBaseModule) {
 256             generateZoneName();
 257         }
 258     }
 259 
 260     private static void usage() {
 261         errout("Usage: java CLDRConverter [options]%n"
 262                 + "\t-help          output this usage message and exit%n"
 263                 + "\t-verbose       output information%n"
 264                 + "\t-draft [contributed | approved | provisional | unconfirmed]%n"
 265                 + "\t\t       draft level for using data (default: contributed)%n"
 266                 + "\t-base dir      base directory for CLDR input files%n"
 267                 + "\t-basemodule    generates bundles that go into java.base module%n"
 268                 + "\t-baselocales loc(,loc)*      locales that go into the base module%n"
 269                 + "\t-o dir         output directory (default: ./build/gensrc)%n"
 270                 + "\t-zntempfile    template file for java.time.format.ZoneName.java%n"
 271                 + "\t-tzdatadir     tzdata directory for java.time.format.ZoneName.java%n"
 272                 + "\t-utf8          use UTF-8 rather than \\uxxxx (for debug)%n");
 273     }
 274 
 275     static void info(String fmt, Object... args) {
 276         if (verbose) {
 277             System.out.printf(fmt, args);
 278         }
 279     }
 280 
 281     static void info(String msg) {
 282         if (verbose) {
 283             System.out.println(msg);
 284         }
 285     }
 286 
 287     static void warning(String fmt, Object... args) {
 288         System.err.print("Warning: ");
 289         System.err.printf(fmt, args);
 290     }
 291 
 292     static void warning(String msg) {
 293         System.err.print("Warning: ");
 294         errout(msg);
 295     }
 296 
 297     static void severe(String fmt, Object... args) {
 298         System.err.print("Error: ");
 299         System.err.printf(fmt, args);
 300     }
 301 
 302     static void severe(String msg) {
 303         System.err.print("Error: ");
 304         errout(msg);
 305     }
 306 
 307     private static void errout(String msg) {
 308         if (msg.contains("%n")) {
 309             System.err.printf(msg);
 310         } else {
 311             System.err.println(msg);
 312         }
 313     }
 314 
 315     /**
 316      * Configure the parser to allow access to DTDs on the file system.
 317      */
 318     private static void enableFileAccess(SAXParser parser) throws SAXNotSupportedException {
 319         try {
 320             parser.setProperty("http://javax.xml.XMLConstants/property/accessExternalDTD", "file");
 321         } catch (SAXNotRecognizedException ignore) {
 322             // property requires >= JAXP 1.5
 323         }
 324     }
 325 
 326     private static List<Bundle> readBundleList() throws Exception {
 327         List<Bundle> retList = new ArrayList<>();
 328         Path path = FileSystems.getDefault().getPath(SOURCE_FILE_DIR);
 329         try (DirectoryStream<Path> dirStr = Files.newDirectoryStream(path)) {
 330             for (Path entry : dirStr) {
 331                 String fileName = entry.getFileName().toString();
 332                 if (fileName.endsWith(".xml")) {
 333                     String id = fileName.substring(0, fileName.indexOf('.'));
 334                     Locale cldrLoc = Locale.forLanguageTag(toLanguageTag(id));
 335                     StringBuilder sb = getCandLocales(cldrLoc);
 336                     if (sb.indexOf("root") == -1) {
 337                         sb.append("root");
 338                     }
 339                     Bundle b = new Bundle(id, sb.toString(), null, null);
 340                     // Insert the bundle for root at the top so that it will get
 341                     // processed first.
 342                     if ("root".equals(id)) {
 343                         retList.add(0, b);
 344                     } else {
 345                         retList.add(b);
 346                     }
 347                 }
 348             }
 349         }
 350         return retList;
 351     }
 352 
 353     private static final Map<String, Map<String, Object>> cldrBundles = new HashMap<>();
 354 
 355     private static Map<String, SortedSet<String>> metaInfo = new HashMap<>();
 356 
 357     static {
 358         // For generating information on supported locales.
 359         metaInfo.put("AvailableLocales", new TreeSet<>());
 360     }
 361 
 362     static Map<String, Object> getCLDRBundle(String id) throws Exception {
 363         Map<String, Object> bundle = cldrBundles.get(id);
 364         if (bundle != null) {
 365             return bundle;
 366         }
 367         File file = new File(SOURCE_FILE_DIR + File.separator + id + ".xml");
 368         if (!file.exists()) {
 369             // Skip if the file doesn't exist.
 370             return Collections.emptyMap();
 371         }
 372 
 373         info("..... main directory .....");
 374         LDMLParseHandler handler = new LDMLParseHandler(id);
 375         parseLDMLFile(file, handler);
 376 
 377         bundle = handler.getData();
 378         cldrBundles.put(id, bundle);
 379 
 380         if (id.equals("root")) {
 381             // Calendar data (firstDayOfWeek & minDaysInFirstWeek)
 382             bundle = handlerSuppl.getData("root");
 383             if (bundle != null) {
 384                 //merge two maps into one map
 385                 Map<String, Object> temp = cldrBundles.remove(id);
 386                 bundle.putAll(temp);
 387                 cldrBundles.put(id, bundle);
 388             }
 389         }
 390         return bundle;
 391     }
 392 
 393     // Parsers for data in "supplemental" directory
 394     //
 395     private static void parseSupplemental() throws Exception {
 396         // Parse SupplementalData file and store the information in the HashMap
 397         // Calendar information such as firstDay and minDay are stored in
 398         // supplementalData.xml as of CLDR1.4. Individual territory is listed
 399         // with its ISO 3166 country code while default is listed using UNM49
 400         // region and composition numerical code (001 for World.)
 401         //
 402         // SupplementalData file also provides the "parent" locales which
 403         // are othrwise not to be fallen back. Process them here as well.
 404         //
 405         handlerSuppl = new SupplementDataParseHandler();
 406         parseLDMLFile(new File(SPPL_SOURCE_FILE), handlerSuppl);
 407         Map<String, Object> parentData = handlerSuppl.getData("root");
 408         parentData.keySet().stream()
 409                 .filter(key -> key.startsWith(PARENT_LOCALE_PREFIX))
 410                 .forEach(key -> {
 411                 parentLocalesMap.put(key, new TreeSet(
 412                     Arrays.asList(((String)parentData.get(key)).split(" "))));
 413             });
 414 
 415         // Parse numberingSystems to get digit zero character information.
 416         handlerNumbering = new NumberingSystemsParseHandler();
 417         parseLDMLFile(new File(NUMBERING_SOURCE_FILE), handlerNumbering);
 418 
 419         // Parse metaZones to create mappings between Olson tzids and CLDR meta zone names
 420         handlerMetaZones = new MetaZonesParseHandler();
 421         parseLDMLFile(new File(METAZONES_SOURCE_FILE), handlerMetaZones);
 422 
 423         // Parse likelySubtags
 424         handlerLikelySubtags = new LikelySubtagsParseHandler();
 425         parseLDMLFile(new File(LIKELYSUBTAGS_SOURCE_FILE), handlerLikelySubtags);
 426 
 427         // Parse supplementalMetadata
 428         // Currently only interested in deprecated time zone ids.
 429         handlerSupplMeta = new SupplementalMetadataParseHandler();
 430         parseLDMLFile(new File(SPPL_META_SOURCE_FILE), handlerSupplMeta);
 431     }
 432 
 433     // Parsers for data in "bcp47" directory
 434     //
 435     private static void parseBCP47() throws Exception {
 436         // Parse timezone
 437         handlerTimeZone = new TimeZoneParseHandler();
 438         parseLDMLFile(new File(TIMEZONE_SOURCE_FILE), handlerTimeZone);
 439     }
 440 
 441     private static void parseLDMLFile(File srcfile, AbstractLDMLHandler handler) throws Exception {
 442         info("..... Parsing " + srcfile.getName() + " .....");
 443         SAXParserFactory pf = SAXParserFactory.newInstance();
 444         pf.setValidating(true);
 445         SAXParser parser = pf.newSAXParser();
 446         enableFileAccess(parser);
 447         parser.parse(srcfile, handler);
 448     }
 449 
 450     private static StringBuilder getCandLocales(Locale cldrLoc) {
 451         List<Locale> candList = getCandidateLocales(cldrLoc);
 452         StringBuilder sb = new StringBuilder();
 453         for (Locale loc : candList) {
 454             if (!loc.equals(Locale.ROOT)) {
 455                 sb.append(toLocaleName(loc.toLanguageTag()));
 456                 sb.append(",");
 457             }
 458         }
 459         return sb;
 460     }
 461 
 462     private static List<Locale> getCandidateLocales(Locale cldrLoc) {
 463         List<Locale> candList = new ArrayList<>();
 464         candList = applyParentLocales("", defCon.getCandidateLocales("",  cldrLoc));
 465         return candList;
 466     }
 467 
 468     private static void convertBundles(List<Bundle> bundles) throws Exception {
 469         // parent locales map. The mappings are put in base metaInfo file
 470         // for now.
 471         if (isBaseModule) {
 472             metaInfo.putAll(parentLocalesMap);
 473         }
 474 
 475         for (Bundle bundle : bundles) {
 476             // Get the target map, which contains all the data that should be
 477             // visible for the bundle's locale
 478 
 479             Map<String, Object> targetMap = bundle.getTargetMap();
 480 
 481             EnumSet<Bundle.Type> bundleTypes = bundle.getBundleTypes();
 482 
 483             if (bundle.isRoot()) {
 484                 // Add DateTimePatternChars because CLDR no longer supports localized patterns.
 485                 targetMap.put("DateTimePatternChars", "GyMdkHmsSEDFwWahKzZ");
 486             }
 487 
 488             // Now the map contains just the entries that need to be in the resources bundles.
 489             // Go ahead and generate them.
 490             if (bundleTypes.contains(Bundle.Type.LOCALENAMES)) {
 491                 Map<String, Object> localeNamesMap = extractLocaleNames(targetMap, bundle.getID());
 492                 if (!localeNamesMap.isEmpty() || bundle.isRoot()) {
 493                     bundleGenerator.generateBundle("util", "LocaleNames", bundle.getJavaID(), true, localeNamesMap, BundleType.OPEN);
 494                 }
 495             }
 496             if (bundleTypes.contains(Bundle.Type.CURRENCYNAMES)) {
 497                 Map<String, Object> currencyNamesMap = extractCurrencyNames(targetMap, bundle.getID(), bundle.getCurrencies());
 498                 if (!currencyNamesMap.isEmpty() || bundle.isRoot()) {
 499                     bundleGenerator.generateBundle("util", "CurrencyNames", bundle.getJavaID(), true, currencyNamesMap, BundleType.OPEN);
 500                 }
 501             }
 502             if (bundleTypes.contains(Bundle.Type.TIMEZONENAMES)) {
 503                 Map<String, Object> zoneNamesMap = extractZoneNames(targetMap, bundle.getID());
 504                 if (!zoneNamesMap.isEmpty() || bundle.isRoot()) {
 505                     bundleGenerator.generateBundle("util", "TimeZoneNames", bundle.getJavaID(), true, zoneNamesMap, BundleType.TIMEZONE);
 506                 }
 507             }
 508             if (bundleTypes.contains(Bundle.Type.CALENDARDATA)) {
 509                 Map<String, Object> calendarDataMap = extractCalendarData(targetMap, bundle.getID());
 510                 if (!calendarDataMap.isEmpty() || bundle.isRoot()) {
 511                     bundleGenerator.generateBundle("util", "CalendarData", bundle.getJavaID(), true, calendarDataMap, BundleType.PLAIN);
 512                 }
 513             }
 514             if (bundleTypes.contains(Bundle.Type.FORMATDATA)) {
 515                 Map<String, Object> formatDataMap = extractFormatData(targetMap, bundle.getID());
 516                 if (!formatDataMap.isEmpty() || bundle.isRoot()) {
 517                     bundleGenerator.generateBundle("text", "FormatData", bundle.getJavaID(), true, formatDataMap, BundleType.PLAIN);
 518                 }
 519             }
 520 
 521             // For AvailableLocales
 522             metaInfo.get("AvailableLocales").add(toLanguageTag(bundle.getID()));
 523             addLikelySubtags(metaInfo, "AvailableLocales", bundle.getID());
 524         }
 525         bundleGenerator.generateMetaInfo(metaInfo);
 526     }
 527 
 528     static final Map<String, String> aliases = new HashMap<>();
 529 
 530     /**
 531      * Translate the aliases into the real entries in the bundle map.
 532      */
 533     static void handleAliases(Map<String, Object> bundleMap) {
 534         Set bundleKeys = bundleMap.keySet();
 535         try {
 536             for (String key : aliases.keySet()) {
 537                 String targetKey = aliases.get(key);
 538                 if (bundleKeys.contains(targetKey)) {
 539                     bundleMap.putIfAbsent(key, bundleMap.get(targetKey));
 540                 }
 541             }
 542         } catch (Exception ex) {
 543             Logger.getLogger(CLDRConverter.class.getName()).log(Level.SEVERE, null, ex);
 544         }
 545     }
 546 
 547     /*
 548      * Returns the language portion of the given id.
 549      * If id is "root", "" is returned.
 550      */
 551     static String getLanguageCode(String id) {
 552         return "root".equals(id) ? "" : Locale.forLanguageTag(id.replaceAll("_", "-")).getLanguage();
 553     }
 554 
 555     /**
 556      * Examine if the id includes the country (territory) code. If it does, it returns
 557      * the country code.
 558      * Otherwise, it returns null. eg. when the id is "zh_Hans_SG", it return "SG".
 559      * It does NOT return UN M.49 code, e.g., '001', as those three digit numbers cannot
 560      * be translated into package names.
 561      */
 562     static String getCountryCode(String id) {
 563         String rgn = getRegionCode(id);
 564         return rgn.length() == 2 ? rgn: null;
 565     }
 566 
 567     /**
 568      * Examine if the id includes the region code. If it does, it returns
 569      * the region code.
 570      * Otherwise, it returns null. eg. when the id is "zh_Hans_SG", it return "SG".
 571      * It DOES return UN M.49 code, e.g., '001', as well as ISO 3166 two letter country codes.
 572      */
 573     static String getRegionCode(String id) {
 574         return Locale.forLanguageTag(id.replaceAll("_", "-")).getCountry();
 575     }
 576 
 577     private static class KeyComparator implements Comparator<String> {
 578         static KeyComparator INSTANCE = new KeyComparator();
 579 
 580         private KeyComparator() {
 581         }
 582 
 583         @Override
 584         public int compare(String o1, String o2) {
 585             int len1 = o1.length();
 586             int len2 = o2.length();
 587             if (!isDigit(o1.charAt(0)) && !isDigit(o2.charAt(0))) {
 588                 // Shorter string comes first unless either starts with a digit.
 589                 if (len1 < len2) {
 590                     return -1;
 591                 }
 592                 if (len1 > len2) {
 593                     return 1;
 594                 }
 595             }
 596             return o1.compareTo(o2);
 597         }
 598 
 599         private boolean isDigit(char c) {
 600             return c >= '0' && c <= '9';
 601         }
 602     }
 603 
 604     private static Map<String, Object> extractLocaleNames(Map<String, Object> map, String id) {
 605         Map<String, Object> localeNames = new TreeMap<>(KeyComparator.INSTANCE);
 606         for (String key : map.keySet()) {
 607             if (key.startsWith(LOCALE_NAME_PREFIX)) {
 608                 switch (key) {
 609                     case LOCALE_SEPARATOR:
 610                         localeNames.put("ListCompositionPattern", map.get(key));
 611                         break;
 612                     case LOCALE_KEYTYPE:
 613                         localeNames.put("ListKeyTypePattern", map.get(key));
 614                         break;
 615                     default:
 616                         localeNames.put(key.substring(LOCALE_NAME_PREFIX.length()), map.get(key));
 617                         break;
 618                 }
 619             }
 620         }
 621 
 622         if (id.equals("root")) {
 623             // Add display name pattern, which is not in CLDR
 624             localeNames.put("DisplayNamePattern", "{0,choice,0#|1#{1}|2#{1} ({2})}");
 625         }
 626 
 627         return localeNames;
 628     }
 629 
 630     @SuppressWarnings("AssignmentToForLoopParameter")
 631     private static Map<String, Object> extractCurrencyNames(Map<String, Object> map, String id, String names)
 632             throws Exception {
 633         Map<String, Object> currencyNames = new TreeMap<>(KeyComparator.INSTANCE);
 634         for (String key : map.keySet()) {
 635             if (key.startsWith(CURRENCY_NAME_PREFIX)) {
 636                 currencyNames.put(key.substring(CURRENCY_NAME_PREFIX.length()), map.get(key));
 637             } else if (key.startsWith(CURRENCY_SYMBOL_PREFIX)) {
 638                 currencyNames.put(key.substring(CURRENCY_SYMBOL_PREFIX.length()), map.get(key));
 639             }
 640         }
 641         return currencyNames;
 642     }
 643 
 644     private static Map<String, Object> extractZoneNames(Map<String, Object> map, String id) {
 645         Map<String, Object> names = new HashMap<>();
 646 
 647         // Copy over missing time zone ids from JRE for English locale
 648         if (id.equals("en")) {
 649             Map<String[], String> jreMetaMap = new HashMap<>();
 650             jreTimeZoneNames.stream().forEach(e -> {
 651                 String tzid = (String)e[0];
 652                 String[] data = (String[])e[1];
 653 
 654                 if (map.get(TIMEZONE_ID_PREFIX + tzid) == null &&
 655                     handlerMetaZones.get(tzid) == null ||
 656                     handlerMetaZones.get(tzid) != null &&
 657                     map.get(METAZONE_ID_PREFIX + handlerMetaZones.get(tzid)) == null) {
 658                     // First, check the CLDR meta key
 659                     Optional<Map.Entry<String, String>> cldrMeta =
 660                         handlerMetaZones.getData().entrySet().stream()
 661                             .filter(me ->
 662                                 Arrays.deepEquals(data,
 663                                     (String[])map.get(METAZONE_ID_PREFIX + me.getValue())))
 664                             .findAny();
 665                     if (cldrMeta.isPresent()) {
 666                         names.put(tzid, cldrMeta.get().getValue());
 667                     } else {
 668                         // check the JRE meta key, add if there is not.
 669                         Optional<Map.Entry<String[], String>> jreMeta =
 670                             jreMetaMap.entrySet().stream()
 671                                 .filter(jm -> Arrays.deepEquals(data, jm.getKey()))
 672                                 .findAny();
 673                         if (jreMeta.isPresent()) {
 674                             names.put(tzid, jreMeta.get().getValue());
 675                         } else {
 676                             String metaName = "JRE_" + tzid.replaceAll("[/-]", "_");
 677                             names.put(METAZONE_ID_PREFIX + metaName, data);
 678                             names.put(tzid, metaName);
 679                             jreMetaMap.put(data, metaName);
 680                         }
 681                     }
 682                 }
 683             });
 684         }
 685 
 686         Arrays.stream(AVAILABLE_TZIDS).forEach(tzid -> {
 687             // If the tzid is deprecated, get the data for the replacement id
 688             String tzKey = Optional.ofNullable((String)handlerSupplMeta.get(tzid))
 689                                    .orElse(tzid);
 690             Object data = map.get(TIMEZONE_ID_PREFIX + tzKey);
 691 
 692             if (data instanceof String[]) {
 693                 names.put(tzid, data);
 694             } else {
 695                 String meta = handlerMetaZones.get(tzKey);
 696                 if (meta != null) {
 697                     String metaKey = METAZONE_ID_PREFIX + meta;
 698                     data = map.get(metaKey);
 699                     if (data instanceof String[]) {
 700                         // Keep the metazone prefix here.
 701                         names.put(metaKey, data);
 702                         names.put(tzid, meta);
 703                     }
 704                 }
 705             }
 706         });
 707 
 708         return names;
 709     }
 710 
 711     /**
 712      * Extracts the language independent calendar data. Each of the two keys,
 713      * "firstDayOfWeek" and "minimalDaysInFirstWeek" has a string value consists of
 714      * one or multiple occurrences of:
 715      *  i: rg1 rg2 ... rgn;
 716      * where "i" is the data for the following regions (delimited by a space) after
 717      * ":", and ends with a ";".
 718      */
 719     private static Map<String, Object> extractCalendarData(Map<String, Object> map, String id) {
 720         Map<String, Object> calendarData = new LinkedHashMap<>();
 721         if (id.equals("root")) {
 722             calendarData.put("firstDayOfWeek",
 723                 IntStream.range(1, 8)
 724                     .mapToObj(String::valueOf)
 725                     .filter(d -> map.keySet().contains(CALENDAR_FIRSTDAY_PREFIX + d))
 726                     .map(d -> d + ": " + map.get(CALENDAR_FIRSTDAY_PREFIX + d))
 727                     .collect(Collectors.joining(";")));
 728             calendarData.put("minimalDaysInFirstWeek",
 729                 IntStream.range(0, 7)
 730                     .mapToObj(String::valueOf)
 731                     .filter(d -> map.keySet().contains(CALENDAR_MINDAYS_PREFIX + d))
 732                     .map(d -> d + ": " + map.get(CALENDAR_MINDAYS_PREFIX + d))
 733                     .collect(Collectors.joining(";")));
 734         }
 735         return calendarData;
 736     }
 737 
 738     static final String[] FORMAT_DATA_ELEMENTS = {
 739         "MonthNames",
 740         "standalone.MonthNames",
 741         "MonthAbbreviations",
 742         "standalone.MonthAbbreviations",
 743         "MonthNarrows",
 744         "standalone.MonthNarrows",
 745         "DayNames",
 746         "standalone.DayNames",
 747         "DayAbbreviations",
 748         "standalone.DayAbbreviations",
 749         "DayNarrows",
 750         "standalone.DayNarrows",
 751         "QuarterNames",
 752         "standalone.QuarterNames",
 753         "QuarterAbbreviations",
 754         "standalone.QuarterAbbreviations",
 755         "QuarterNarrows",
 756         "standalone.QuarterNarrows",
 757         "AmPmMarkers",
 758         "narrow.AmPmMarkers",
 759         "abbreviated.AmPmMarkers",
 760         "long.Eras",
 761         "Eras",
 762         "narrow.Eras",
 763         "field.era",
 764         "field.year",
 765         "field.month",
 766         "field.week",
 767         "field.weekday",
 768         "field.dayperiod",
 769         "field.hour",
 770         "timezone.hourFormat",
 771         "timezone.gmtFormat",
 772         "field.minute",
 773         "field.second",
 774         "field.zone",
 775         "TimePatterns",
 776         "DatePatterns",
 777         "DateTimePatterns",
 778         "DateTimePatternChars"
 779     };
 780 
 781     private static Map<String, Object> extractFormatData(Map<String, Object> map, String id) {
 782         Map<String, Object> formatData = new LinkedHashMap<>();
 783         for (CalendarType calendarType : CalendarType.values()) {
 784             if (calendarType == CalendarType.GENERIC) {
 785                 continue;
 786             }
 787             String prefix = calendarType.keyElementName();
 788             for (String element : FORMAT_DATA_ELEMENTS) {
 789                 String key = prefix + element;
 790                 copyIfPresent(map, "java.time." + key, formatData);
 791                 copyIfPresent(map, key, formatData);
 792             }
 793         }
 794 
 795         for (String key : map.keySet()) {
 796         // Copy available calendar names
 797             if (key.startsWith(CLDRConverter.LOCALE_TYPE_PREFIX_CA)) {
 798                 String type = key.substring(CLDRConverter.LOCALE_TYPE_PREFIX_CA.length());
 799                 for (CalendarType calendarType : CalendarType.values()) {
 800                     if (calendarType == CalendarType.GENERIC) {
 801                         continue;
 802                     }
 803                     if (type.equals(calendarType.lname())) {
 804                         Object value = map.get(key);
 805                         String dataKey = key.replace(LOCALE_TYPE_PREFIX_CA,
 806                                 CALENDAR_NAME_PREFIX);
 807                         formatData.put(dataKey, value);
 808                         String ukey = CALENDAR_NAME_PREFIX + calendarType.uname();
 809                         if (!dataKey.equals(ukey)) {
 810                             formatData.put(ukey, value);
 811                         }
 812                     }
 813                 }
 814             }
 815         }
 816 
 817         copyIfPresent(map, "DefaultNumberingSystem", formatData);
 818 
 819         @SuppressWarnings("unchecked")
 820         List<String> numberingScripts = (List<String>) map.remove("numberingScripts");
 821         if (numberingScripts != null) {
 822             for (String script : numberingScripts) {
 823                 copyIfPresent(map, script + "." + "NumberElements", formatData);
 824             }
 825         } else {
 826             copyIfPresent(map, "NumberElements", formatData);
 827         }
 828         copyIfPresent(map, "NumberPatterns", formatData);
 829 
 830         // put extra number elements for available scripts into formatData, if it is "root"
 831         if (id.equals("root")) {
 832             handlerNumbering.keySet().stream()
 833                 .filter(k -> !numberingScripts.contains(k))
 834                 .forEach(k -> {
 835                     String[] ne = (String[])map.get("latn.NumberElements");
 836                     String[] neNew = Arrays.copyOf(ne, ne.length);
 837                     neNew[4] = handlerNumbering.get(k).substring(0, 1);
 838                     formatData.put(k + ".NumberElements", neNew);
 839                 });
 840         }
 841         return formatData;
 842     }
 843 
 844     private static void copyIfPresent(Map<String, Object> src, String key, Map<String, Object> dest) {
 845         Object value = src.get(key);
 846         if (value != null) {
 847             dest.put(key, value);
 848         }
 849     }
 850 
 851     // --- code below here is adapted from java.util.Properties ---
 852     private static final String specialSaveCharsJava = "\"";
 853     private static final String specialSaveCharsProperties = "=: \t\r\n\f#!";
 854 
 855     /*
 856      * Converts unicodes to encoded \uxxxx
 857      * and writes out any of the characters in specialSaveChars
 858      * with a preceding slash
 859      */
 860     static String saveConvert(String theString, boolean useJava) {
 861         if (theString == null) {
 862             return "";
 863         }
 864 
 865         String specialSaveChars;
 866         if (useJava) {
 867             specialSaveChars = specialSaveCharsJava;
 868         } else {
 869             specialSaveChars = specialSaveCharsProperties;
 870         }
 871         boolean escapeSpace = false;
 872 
 873         int len = theString.length();
 874         StringBuilder outBuffer = new StringBuilder(len * 2);
 875         Formatter formatter = new Formatter(outBuffer, Locale.ROOT);
 876 
 877         for (int x = 0; x < len; x++) {
 878             char aChar = theString.charAt(x);
 879             switch (aChar) {
 880             case ' ':
 881                 if (x == 0 || escapeSpace) {
 882                     outBuffer.append('\\');
 883                 }
 884                 outBuffer.append(' ');
 885                 break;
 886             case '\\':
 887                 outBuffer.append('\\');
 888                 outBuffer.append('\\');
 889                 break;
 890             case '\t':
 891                 outBuffer.append('\\');
 892                 outBuffer.append('t');
 893                 break;
 894             case '\n':
 895                 outBuffer.append('\\');
 896                 outBuffer.append('n');
 897                 break;
 898             case '\r':
 899                 outBuffer.append('\\');
 900                 outBuffer.append('r');
 901                 break;
 902             case '\f':
 903                 outBuffer.append('\\');
 904                 outBuffer.append('f');
 905                 break;
 906             default:
 907                 if (aChar < 0x0020 || (!USE_UTF8 && aChar > 0x007e)) {
 908                     formatter.format("\\u%04x", (int)aChar);
 909                 } else {
 910                     if (specialSaveChars.indexOf(aChar) != -1) {
 911                         outBuffer.append('\\');
 912                     }
 913                     outBuffer.append(aChar);
 914                 }
 915             }
 916         }
 917         return outBuffer.toString();
 918     }
 919 
 920     private static String toLanguageTag(String locName) {
 921         if (locName.indexOf('_') == -1) {
 922             return locName;
 923         }
 924         String tag = locName.replaceAll("_", "-");
 925         Locale loc = Locale.forLanguageTag(tag);
 926         return loc.toLanguageTag();
 927     }
 928 
 929     private static void addLikelySubtags(Map<String, SortedSet<String>> metaInfo, String category, String id) {
 930         String likelySubtag = handlerLikelySubtags.get(id);
 931         if (likelySubtag != null) {
 932             // Remove Script for now
 933             metaInfo.get(category).add(toLanguageTag(likelySubtag).replaceFirst("-[A-Z][a-z]{3}", ""));
 934         }
 935     }
 936 
 937     private static String toLocaleName(String tag) {
 938         if (tag.indexOf('-') == -1) {
 939             return tag;
 940         }
 941         return tag.replaceAll("-", "_");
 942     }
 943 
 944     private static void setupBaseLocales(String localeList) {
 945         Arrays.stream(localeList.split(","))
 946             .map(Locale::forLanguageTag)
 947             .map(l -> Control.getControl(Control.FORMAT_DEFAULT)
 948                              .getCandidateLocales("", l))
 949             .forEach(BASE_LOCALES::addAll);
 950     }
 951 
 952     // applying parent locale rules to the passed candidates list
 953     // This has to match with the one in sun.util.cldr.CLDRLocaleProviderAdapter
 954     private static Map<Locale, Locale> childToParentLocaleMap = null;
 955     private static List<Locale> applyParentLocales(String baseName, List<Locale> candidates) {
 956         if (Objects.isNull(childToParentLocaleMap)) {
 957             childToParentLocaleMap = new HashMap<>();
 958             parentLocalesMap.keySet().forEach(key -> {
 959                 String parent = key.substring(PARENT_LOCALE_PREFIX.length()).replaceAll("_", "-");
 960                 parentLocalesMap.get(key).stream().forEach(child -> {
 961                     childToParentLocaleMap.put(Locale.forLanguageTag(child),
 962                         "root".equals(parent) ? Locale.ROOT : Locale.forLanguageTag(parent));
 963                 });
 964             });
 965         }
 966 
 967         // check irregular parents
 968         for (int i = 0; i < candidates.size(); i++) {
 969             Locale l = candidates.get(i);
 970             Locale p = childToParentLocaleMap.get(l);
 971             if (!l.equals(Locale.ROOT) &&
 972                 Objects.nonNull(p) &&
 973                 !candidates.get(i+1).equals(p)) {
 974                 List<Locale> applied = candidates.subList(0, i+1);
 975                 applied.addAll(applyParentLocales(baseName, defCon.getCandidateLocales(baseName, p)));
 976                 return applied;
 977             }
 978         }
 979 
 980         return candidates;
 981     }
 982 
 983     private static void generateZoneName() throws Exception {
 984         Files.createDirectories(Paths.get(DESTINATION_DIR, "java", "time", "format"));
 985         Files.write(Paths.get(DESTINATION_DIR, "java", "time", "format", "ZoneName.java"),
 986             Files.lines(Paths.get(zoneNameTempFile))
 987                 .flatMap(l -> {
 988                     if (l.equals("%%%%ZIDMAP%%%%")) {
 989                         return zidMapEntry();
 990                     } else if (l.equals("%%%%MZONEMAP%%%%")) {
 991                         return handlerMetaZones.mzoneMapEntry();
 992                     } else if (l.equals("%%%%DEPRECATED%%%%")) {
 993                         return handlerSupplMeta.deprecatedMap();
 994                     } else if (l.equals("%%%%TZDATALINK%%%%")) {
 995                         return tzDataLinkEntry();
 996                     } else {
 997                         return Stream.of(l);
 998                     }
 999                 })
1000                 .collect(Collectors.toList()),
1001             StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
1002     }
1003 
1004     private static Stream<String> zidMapEntry() {
1005         Map<String, String> canonMap = new HashMap<>();
1006         handlerTimeZone.getData().entrySet().stream()
1007             .forEach(e -> {
1008                 String[] ids = ((String)e.getValue()).split("\\s");
1009                 for (int i = 1; i < ids.length; i++) {
1010                     canonMap.put(ids[i], ids[0]);
1011                 }});
1012         return ZoneId.getAvailableZoneIds().stream()
1013                 .map(id -> {
1014                     String canonId = canonMap.getOrDefault(id, id);
1015                     String meta = handlerMetaZones.get(canonId);
1016                     String zone001 = handlerMetaZones.zidMap().get(meta);
1017                     return zone001 == null ? "" :
1018                             String.format("        \"%s\", \"%s\", \"%s\",",
1019                                             id, meta, zone001);
1020                 })
1021                 .filter(s -> !s.isEmpty())
1022                 .sorted();
1023     }
1024 
1025     private static Stream<String> tzDataLinkEntry() {
1026         try {
1027             return Files.walk(Paths.get(tzDataDir), 1)
1028                 .filter(p -> !Files.isDirectory(p))
1029                 .flatMap(CLDRConverter::extractLinks)
1030                 .sorted();
1031         } catch (IOException e) {
1032             throw new UncheckedIOException(e);
1033         }
1034     }
1035 
1036     private static Stream<String> extractLinks(Path tzFile) {
1037         try {
1038             return Files.lines(tzFile)
1039                 .filter(l -> l.startsWith("Link"))
1040                 .map(l -> l.replaceFirst("^Link[\\s]+(\\S+)\\s+(\\S+).*",
1041                                          "        \"$2\", \"$1\","));
1042         } catch (IOException e) {
1043             throw new UncheckedIOException(e);
1044         }
1045     }
1046 }