1 /*
   2  * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 
  26 package build.tools.cldrconverter;
  27 
  28 import static build.tools.cldrconverter.Bundle.jreTimeZoneNames;
  29 import build.tools.cldrconverter.BundleGenerator.BundleType;
  30 import java.io.File;
  31 import java.io.IOException;
  32 import java.io.UncheckedIOException;
  33 import java.nio.file.*;
  34 import java.text.MessageFormat;
  35 import java.time.*;
  36 import java.util.*;
  37 import java.util.ResourceBundle.Control;
  38 import java.util.logging.Level;
  39 import java.util.logging.Logger;
  40 import java.util.stream.Collectors;
  41 import java.util.stream.IntStream;
  42 import java.util.stream.Stream;
  43 import javax.xml.parsers.SAXParser;
  44 import javax.xml.parsers.SAXParserFactory;
  45 import org.xml.sax.SAXNotRecognizedException;
  46 import org.xml.sax.SAXNotSupportedException;
  47 
  48 
  49 /**
  50  * Converts locale data from "Locale Data Markup Language" format to
  51  * JRE resource bundle format. LDML is the format used by the Common
  52  * Locale Data Repository maintained by the Unicode Consortium.
  53  */
  54 public class CLDRConverter {
  55 
  56     static final String LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldml.dtd";
  57     static final String SPPL_LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldmlSupplemental.dtd";
  58     static final String BCP47_LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldmlBCP47.dtd";
  59 
  60 
  61     private static String CLDR_BASE;
  62     static String LOCAL_LDML_DTD;
  63     static String LOCAL_SPPL_LDML_DTD;
  64     static String LOCAL_BCP47_LDML_DTD;
  65     private static String SOURCE_FILE_DIR;
  66     private static String SPPL_SOURCE_FILE;
  67     private static String SPPL_META_SOURCE_FILE;
  68     private static String NUMBERING_SOURCE_FILE;
  69     private static String METAZONES_SOURCE_FILE;
  70     private static String LIKELYSUBTAGS_SOURCE_FILE;
  71     private static String TIMEZONE_SOURCE_FILE;
  72     private static String WINZONES_SOURCE_FILE;
  73     static String DESTINATION_DIR = "build/gensrc";
  74 
  75     static final String LOCALE_NAME_PREFIX = "locale.displayname.";
  76     static final String LOCALE_SEPARATOR = LOCALE_NAME_PREFIX + "separator";
  77     static final String LOCALE_KEYTYPE = LOCALE_NAME_PREFIX + "keytype";
  78     static final String LOCALE_KEY_PREFIX = LOCALE_NAME_PREFIX + "key.";
  79     static final String LOCALE_TYPE_PREFIX = LOCALE_NAME_PREFIX + "type.";
  80     static final String LOCALE_TYPE_PREFIX_CA = LOCALE_TYPE_PREFIX + "ca.";
  81     static final String CURRENCY_SYMBOL_PREFIX = "currency.symbol.";
  82     static final String CURRENCY_NAME_PREFIX = "currency.displayname.";
  83     static final String CALENDAR_NAME_PREFIX = "calendarname.";
  84     static final String CALENDAR_FIRSTDAY_PREFIX = "firstDay.";
  85     static final String CALENDAR_MINDAYS_PREFIX = "minDays.";
  86     static final String TIMEZONE_ID_PREFIX = "timezone.id.";
  87     static final String EXEMPLAR_CITY_PREFIX = "timezone.excity.";
  88     static final String ZONE_NAME_PREFIX = "timezone.displayname.";
  89     static final String METAZONE_ID_PREFIX = "metazone.id.";
  90     static final String PARENT_LOCALE_PREFIX = "parentLocale.";
  91     static final String[] EMPTY_ZONE = {"", "", "", "", "", ""};
  92 
  93     private static SupplementDataParseHandler handlerSuppl;
  94     private static LikelySubtagsParseHandler handlerLikelySubtags;
  95     private static WinZonesParseHandler handlerWinZones;
  96     static SupplementalMetadataParseHandler handlerSupplMeta;
  97     static NumberingSystemsParseHandler handlerNumbering;
  98     static MetaZonesParseHandler handlerMetaZones;
  99     static TimeZoneParseHandler handlerTimeZone;
 100     private static BundleGenerator bundleGenerator;
 101 
 102     // java.base module related
 103     static boolean isBaseModule = false;
 104     static final Set<Locale> BASE_LOCALES = new HashSet<>();
 105 
 106     // "parentLocales" map
 107     private static final Map<String, SortedSet<String>> parentLocalesMap = new HashMap<>();
 108     private static final ResourceBundle.Control defCon =
 109         ResourceBundle.Control.getControl(ResourceBundle.Control.FORMAT_DEFAULT);
 110 
 111     private static Set<String> AVAILABLE_TZIDS;
 112     private static String zoneNameTempFile;
 113     private static String tzDataDir;
 114     private static final Map<String, String> canonicalTZMap = new HashMap<>();
 115 
 116     static enum DraftType {
 117         UNCONFIRMED,
 118         PROVISIONAL,
 119         CONTRIBUTED,
 120         APPROVED;
 121 
 122         private static final Map<String, DraftType> map = new HashMap<>();
 123         static {
 124             for (DraftType dt : values()) {
 125                 map.put(dt.getKeyword(), dt);
 126             }
 127         }
 128         static private DraftType defaultType = CONTRIBUTED;
 129 
 130         private final String keyword;
 131 
 132         private DraftType() {
 133             keyword = this.name().toLowerCase(Locale.ROOT);
 134 
 135         }
 136 
 137         static DraftType forKeyword(String keyword) {
 138             return map.get(keyword);
 139         }
 140 
 141         static DraftType getDefault() {
 142             return defaultType;
 143         }
 144 
 145         static void setDefault(String keyword) {
 146             defaultType = Objects.requireNonNull(forKeyword(keyword));
 147         }
 148 
 149         String getKeyword() {
 150             return keyword;
 151         }
 152     }
 153 
 154     static boolean USE_UTF8 = false;
 155     private static boolean verbose;
 156 
 157     private CLDRConverter() {
 158        // no instantiation
 159     }
 160 
 161     @SuppressWarnings("AssignmentToForLoopParameter")
 162     public static void main(String[] args) throws Exception {
 163         if (args.length != 0) {
 164             String currentArg = null;
 165             try {
 166                 for (int i = 0; i < args.length; i++) {
 167                     currentArg = args[i];
 168                     switch (currentArg) {
 169                     case "-draft":
 170                         String draftDataType = args[++i];
 171                         try {
 172                             DraftType.setDefault(draftDataType);
 173                         } catch (NullPointerException e) {
 174                             severe("Error: incorrect draft value: %s%n", draftDataType);
 175                             System.exit(1);
 176                         }
 177                         info("Using the specified data type: %s%n", draftDataType);
 178                         break;
 179 
 180                     case "-base":
 181                         // base directory for input files
 182                         CLDR_BASE = args[++i];
 183                         if (!CLDR_BASE.endsWith("/")) {
 184                             CLDR_BASE += "/";
 185                         }
 186                         break;
 187 
 188                     case "-baselocales":
 189                         // base locales
 190                         setupBaseLocales(args[++i]);
 191                         break;
 192 
 193                     case "-basemodule":
 194                         // indicates java.base module resource generation
 195                         isBaseModule = true;
 196                         break;
 197 
 198                     case "-o":
 199                         // output directory
 200                         DESTINATION_DIR = args[++i];
 201                         break;
 202 
 203                     case "-utf8":
 204                         USE_UTF8 = true;
 205                         break;
 206 
 207                     case "-verbose":
 208                         verbose = true;
 209                         break;
 210 
 211                     case "-zntempfile":
 212                         zoneNameTempFile = args[++i];
 213                         break;
 214 
 215                     case "-tzdatadir":
 216                         tzDataDir = args[++i];
 217                         break;
 218 
 219                     case "-help":
 220                         usage();
 221                         System.exit(0);
 222                         break;
 223 
 224                     default:
 225                         throw new RuntimeException();
 226                     }
 227                 }
 228             } catch (RuntimeException e) {
 229                 severe("unknown or imcomplete arg(s): " + currentArg);
 230                 usage();
 231                 System.exit(1);
 232             }
 233         }
 234 
 235         // Set up path names
 236         LOCAL_LDML_DTD = CLDR_BASE + "/dtd/ldml.dtd";
 237         LOCAL_SPPL_LDML_DTD = CLDR_BASE + "/dtd/ldmlSupplemental.dtd";
 238         LOCAL_BCP47_LDML_DTD = CLDR_BASE + "/dtd/ldmlBCP47.dtd";
 239         SOURCE_FILE_DIR = CLDR_BASE + "/main";
 240         SPPL_SOURCE_FILE = CLDR_BASE + "/supplemental/supplementalData.xml";
 241         LIKELYSUBTAGS_SOURCE_FILE = CLDR_BASE + "/supplemental/likelySubtags.xml";
 242         NUMBERING_SOURCE_FILE = CLDR_BASE + "/supplemental/numberingSystems.xml";
 243         METAZONES_SOURCE_FILE = CLDR_BASE + "/supplemental/metaZones.xml";
 244         TIMEZONE_SOURCE_FILE = CLDR_BASE + "/bcp47/timezone.xml";
 245         SPPL_META_SOURCE_FILE = CLDR_BASE + "/supplemental/supplementalMetadata.xml";
 246         WINZONES_SOURCE_FILE = CLDR_BASE + "/supplemental/windowsZones.xml";
 247 
 248         if (BASE_LOCALES.isEmpty()) {
 249             setupBaseLocales("en-US");
 250         }
 251 
 252         bundleGenerator = new ResourceBundleGenerator();
 253 
 254         // Parse data independent of locales
 255         parseSupplemental();
 256         parseBCP47();
 257 
 258         List<Bundle> bundles = readBundleList();
 259         convertBundles(bundles);
 260 
 261         if (isBaseModule) {
 262             // Generate java.time.format.ZoneName.java
 263             generateZoneName();
 264 
 265             // Generate Windows tzmappings
 266             generateWindowsTZMappings();
 267         }
 268     }
 269 
 270     private static void usage() {
 271         errout("Usage: java CLDRConverter [options]%n"
 272                 + "\t-help          output this usage message and exit%n"
 273                 + "\t-verbose       output information%n"
 274                 + "\t-draft [contributed | approved | provisional | unconfirmed]%n"
 275                 + "\t\t       draft level for using data (default: contributed)%n"
 276                 + "\t-base dir      base directory for CLDR input files%n"
 277                 + "\t-basemodule    generates bundles that go into java.base module%n"
 278                 + "\t-baselocales loc(,loc)*      locales that go into the base module%n"
 279                 + "\t-o dir         output directory (default: ./build/gensrc)%n"
 280                 + "\t-zntempfile    template file for java.time.format.ZoneName.java%n"
 281                 + "\t-tzdatadir     tzdata directory for java.time.format.ZoneName.java%n"
 282                 + "\t-utf8          use UTF-8 rather than \\uxxxx (for debug)%n");
 283     }
 284 
 285     static void info(String fmt, Object... args) {
 286         if (verbose) {
 287             System.out.printf(fmt, args);
 288         }
 289     }
 290 
 291     static void info(String msg) {
 292         if (verbose) {
 293             System.out.println(msg);
 294         }
 295     }
 296 
 297     static void warning(String fmt, Object... args) {
 298         System.err.print("Warning: ");
 299         System.err.printf(fmt, args);
 300     }
 301 
 302     static void warning(String msg) {
 303         System.err.print("Warning: ");
 304         errout(msg);
 305     }
 306 
 307     static void severe(String fmt, Object... args) {
 308         System.err.print("Error: ");
 309         System.err.printf(fmt, args);
 310     }
 311 
 312     static void severe(String msg) {
 313         System.err.print("Error: ");
 314         errout(msg);
 315     }
 316 
 317     private static void errout(String msg) {
 318         if (msg.contains("%n")) {
 319             System.err.printf(msg);
 320         } else {
 321             System.err.println(msg);
 322         }
 323     }
 324 
 325     /**
 326      * Configure the parser to allow access to DTDs on the file system.
 327      */
 328     private static void enableFileAccess(SAXParser parser) throws SAXNotSupportedException {
 329         try {
 330             parser.setProperty("http://javax.xml.XMLConstants/property/accessExternalDTD", "file");
 331         } catch (SAXNotRecognizedException ignore) {
 332             // property requires >= JAXP 1.5
 333         }
 334     }
 335 
 336     private static List<Bundle> readBundleList() throws Exception {
 337         List<Bundle> retList = new ArrayList<>();
 338         Path path = FileSystems.getDefault().getPath(SOURCE_FILE_DIR);
 339         try (DirectoryStream<Path> dirStr = Files.newDirectoryStream(path)) {
 340             for (Path entry : dirStr) {
 341                 String fileName = entry.getFileName().toString();
 342                 if (fileName.endsWith(".xml")) {
 343                     String id = fileName.substring(0, fileName.indexOf('.'));
 344                     Locale cldrLoc = Locale.forLanguageTag(toLanguageTag(id));
 345                     StringBuilder sb = getCandLocales(cldrLoc);
 346                     if (sb.indexOf("root") == -1) {
 347                         sb.append("root");
 348                     }
 349                     retList.add(new Bundle(id, sb.toString(), null, null));
 350                 }
 351             }
 352         }
 353 
 354         // Sort the bundles based on id. This will make sure all the parent bundles are
 355         // processed first, e.g., for en_GB bundle, en_001, and "root" comes before
 356         // en_GB. In order for "root" to come at the beginning, "root" is replaced with
 357         // empty string on comparison.
 358         retList.sort((o1, o2) -> {
 359             String id1 = o1.getID();
 360             String id2 = o2.getID();
 361             if(id1.equals("root")) {
 362                 id1 = "";
 363             }
 364             if(id2.equals("root")) {
 365                 id2 = "";
 366             }
 367             return id1.compareTo(id2);
 368         });
 369         return retList;
 370     }
 371 
 372     private static final Map<String, Map<String, Object>> cldrBundles = new HashMap<>();
 373 
 374     private static Map<String, SortedSet<String>> metaInfo = new HashMap<>();
 375 
 376     static {
 377         // For generating information on supported locales.
 378         metaInfo.put("AvailableLocales", new TreeSet<>());
 379     }
 380 
 381     static Map<String, Object> getCLDRBundle(String id) throws Exception {
 382         Map<String, Object> bundle = cldrBundles.get(id);
 383         if (bundle != null) {
 384             return bundle;
 385         }
 386         File file = new File(SOURCE_FILE_DIR + File.separator + id + ".xml");
 387         if (!file.exists()) {
 388             // Skip if the file doesn't exist.
 389             return Collections.emptyMap();
 390         }
 391 
 392         info("..... main directory .....");
 393         LDMLParseHandler handler = new LDMLParseHandler(id);
 394         parseLDMLFile(file, handler);
 395 
 396         bundle = handler.getData();
 397         cldrBundles.put(id, bundle);
 398 
 399         if (id.equals("root")) {
 400             // Calendar data (firstDayOfWeek & minDaysInFirstWeek)
 401             bundle = handlerSuppl.getData("root");
 402             if (bundle != null) {
 403                 //merge two maps into one map
 404                 Map<String, Object> temp = cldrBundles.remove(id);
 405                 bundle.putAll(temp);
 406                 cldrBundles.put(id, bundle);
 407             }
 408         }
 409         return bundle;
 410     }
 411 
 412     // Parsers for data in "supplemental" directory
 413     //
 414     private static void parseSupplemental() throws Exception {
 415         // Parse SupplementalData file and store the information in the HashMap
 416         // Calendar information such as firstDay and minDay are stored in
 417         // supplementalData.xml as of CLDR1.4. Individual territory is listed
 418         // with its ISO 3166 country code while default is listed using UNM49
 419         // region and composition numerical code (001 for World.)
 420         //
 421         // SupplementalData file also provides the "parent" locales which
 422         // are othrwise not to be fallen back. Process them here as well.
 423         //
 424         handlerSuppl = new SupplementDataParseHandler();
 425         parseLDMLFile(new File(SPPL_SOURCE_FILE), handlerSuppl);
 426         Map<String, Object> parentData = handlerSuppl.getData("root");
 427         parentData.keySet().stream()
 428                 .filter(key -> key.startsWith(PARENT_LOCALE_PREFIX))
 429                 .forEach(key -> {
 430                 parentLocalesMap.put(key, new TreeSet(
 431                     Arrays.asList(((String)parentData.get(key)).split(" "))));
 432             });
 433 
 434         // Parse numberingSystems to get digit zero character information.
 435         handlerNumbering = new NumberingSystemsParseHandler();
 436         parseLDMLFile(new File(NUMBERING_SOURCE_FILE), handlerNumbering);
 437 
 438         // Parse metaZones to create mappings between Olson tzids and CLDR meta zone names
 439         handlerMetaZones = new MetaZonesParseHandler();
 440         parseLDMLFile(new File(METAZONES_SOURCE_FILE), handlerMetaZones);
 441 
 442         // Parse likelySubtags
 443         handlerLikelySubtags = new LikelySubtagsParseHandler();
 444         parseLDMLFile(new File(LIKELYSUBTAGS_SOURCE_FILE), handlerLikelySubtags);
 445 
 446         // Parse supplementalMetadata
 447         // Currently interested in deprecated time zone ids and language aliases.
 448         handlerSupplMeta = new SupplementalMetadataParseHandler();
 449         parseLDMLFile(new File(SPPL_META_SOURCE_FILE), handlerSupplMeta);
 450 
 451         // Parse windowsZones
 452         handlerWinZones = new WinZonesParseHandler();
 453         parseLDMLFile(new File(WINZONES_SOURCE_FILE), handlerWinZones);
 454     }
 455 
 456     // Parsers for data in "bcp47" directory
 457     //
 458     private static void parseBCP47() throws Exception {
 459         // Parse timezone
 460         handlerTimeZone = new TimeZoneParseHandler();
 461         parseLDMLFile(new File(TIMEZONE_SOURCE_FILE), handlerTimeZone);
 462 
 463         // canonical tz name map
 464         // alias -> primary
 465         handlerTimeZone.getData().forEach((k, v) -> {
 466             String[] ids = ((String)v).split("\\s");
 467             for (int i = 1; i < ids.length; i++) {
 468                 canonicalTZMap.put(ids[i], ids[0]);
 469             }
 470         });
 471     }
 472 
 473     private static void parseLDMLFile(File srcfile, AbstractLDMLHandler handler) throws Exception {
 474         info("..... Parsing " + srcfile.getName() + " .....");
 475         SAXParserFactory pf = SAXParserFactory.newInstance();
 476         pf.setValidating(true);
 477         SAXParser parser = pf.newSAXParser();
 478         enableFileAccess(parser);
 479         parser.parse(srcfile, handler);
 480     }
 481 
 482     private static StringBuilder getCandLocales(Locale cldrLoc) {
 483         List<Locale> candList = getCandidateLocales(cldrLoc);
 484         StringBuilder sb = new StringBuilder();
 485         for (Locale loc : candList) {
 486             if (!loc.equals(Locale.ROOT)) {
 487                 sb.append(toLocaleName(loc.toLanguageTag()));
 488                 sb.append(",");
 489             }
 490         }
 491         return sb;
 492     }
 493 
 494     private static List<Locale> getCandidateLocales(Locale cldrLoc) {
 495         List<Locale> candList = new ArrayList<>();
 496         candList = applyParentLocales("", defCon.getCandidateLocales("",  cldrLoc));
 497         return candList;
 498     }
 499 
 500     private static void convertBundles(List<Bundle> bundles) throws Exception {
 501         // parent locales map. The mappings are put in base metaInfo file
 502         // for now.
 503         if (isBaseModule) {
 504             metaInfo.putAll(parentLocalesMap);
 505         }
 506 
 507         for (Bundle bundle : bundles) {
 508             // Get the target map, which contains all the data that should be
 509             // visible for the bundle's locale
 510 
 511             Map<String, Object> targetMap = bundle.getTargetMap();
 512 
 513             EnumSet<Bundle.Type> bundleTypes = bundle.getBundleTypes();
 514 
 515             if (bundle.isRoot()) {
 516                 // Add DateTimePatternChars because CLDR no longer supports localized patterns.
 517                 targetMap.put("DateTimePatternChars", "GyMdkHmsSEDFwWahKzZ");
 518             }
 519 
 520             // Now the map contains just the entries that need to be in the resources bundles.
 521             // Go ahead and generate them.
 522             if (bundleTypes.contains(Bundle.Type.LOCALENAMES)) {
 523                 Map<String, Object> localeNamesMap = extractLocaleNames(targetMap, bundle.getID());
 524                 if (!localeNamesMap.isEmpty() || bundle.isRoot()) {
 525                     bundleGenerator.generateBundle("util", "LocaleNames", bundle.getJavaID(), true, localeNamesMap, BundleType.OPEN);
 526                 }
 527             }
 528             if (bundleTypes.contains(Bundle.Type.CURRENCYNAMES)) {
 529                 Map<String, Object> currencyNamesMap = extractCurrencyNames(targetMap, bundle.getID(), bundle.getCurrencies());
 530                 if (!currencyNamesMap.isEmpty() || bundle.isRoot()) {
 531                     bundleGenerator.generateBundle("util", "CurrencyNames", bundle.getJavaID(), true, currencyNamesMap, BundleType.OPEN);
 532                 }
 533             }
 534             if (bundleTypes.contains(Bundle.Type.TIMEZONENAMES)) {
 535                 Map<String, Object> zoneNamesMap = extractZoneNames(targetMap, bundle.getID());
 536                 if (!zoneNamesMap.isEmpty() || bundle.isRoot()) {
 537                     bundleGenerator.generateBundle("util", "TimeZoneNames", bundle.getJavaID(), true, zoneNamesMap, BundleType.TIMEZONE);
 538                 }
 539             }
 540             if (bundleTypes.contains(Bundle.Type.CALENDARDATA)) {
 541                 Map<String, Object> calendarDataMap = extractCalendarData(targetMap, bundle.getID());
 542                 if (!calendarDataMap.isEmpty() || bundle.isRoot()) {
 543                     bundleGenerator.generateBundle("util", "CalendarData", bundle.getJavaID(), true, calendarDataMap, BundleType.PLAIN);
 544                 }
 545             }
 546             if (bundleTypes.contains(Bundle.Type.FORMATDATA)) {
 547                 Map<String, Object> formatDataMap = extractFormatData(targetMap, bundle.getID());
 548                 if (!formatDataMap.isEmpty() || bundle.isRoot()) {
 549                     bundleGenerator.generateBundle("text", "FormatData", bundle.getJavaID(), true, formatDataMap, BundleType.PLAIN);
 550                 }
 551             }
 552 
 553             // For AvailableLocales
 554             metaInfo.get("AvailableLocales").add(toLanguageTag(bundle.getID()));
 555             addLikelySubtags(metaInfo, "AvailableLocales", bundle.getID());
 556         }
 557         bundleGenerator.generateMetaInfo(metaInfo);
 558     }
 559 
 560     static final Map<String, String> aliases = new HashMap<>();
 561 
 562     /**
 563      * Translate the aliases into the real entries in the bundle map.
 564      */
 565     static void handleAliases(Map<String, Object> bundleMap) {
 566         Set bundleKeys = bundleMap.keySet();
 567         try {
 568             for (String key : aliases.keySet()) {
 569                 String targetKey = aliases.get(key);
 570                 if (bundleKeys.contains(targetKey)) {
 571                     bundleMap.putIfAbsent(key, bundleMap.get(targetKey));
 572                 }
 573             }
 574         } catch (Exception ex) {
 575             Logger.getLogger(CLDRConverter.class.getName()).log(Level.SEVERE, null, ex);
 576         }
 577     }
 578 
 579     /*
 580      * Returns the language portion of the given id.
 581      * If id is "root", "" is returned.
 582      */
 583     static String getLanguageCode(String id) {
 584         return "root".equals(id) ? "" : Locale.forLanguageTag(id.replaceAll("_", "-")).getLanguage();
 585     }
 586 
 587     /**
 588      * Examine if the id includes the country (territory) code. If it does, it returns
 589      * the country code.
 590      * Otherwise, it returns null. eg. when the id is "zh_Hans_SG", it return "SG".
 591      * It does NOT return UN M.49 code, e.g., '001', as those three digit numbers cannot
 592      * be translated into package names.
 593      */
 594     static String getCountryCode(String id) {
 595         String rgn = getRegionCode(id);
 596         return rgn.length() == 2 ? rgn: null;
 597     }
 598 
 599     /**
 600      * Examine if the id includes the region code. If it does, it returns
 601      * the region code.
 602      * Otherwise, it returns null. eg. when the id is "zh_Hans_SG", it return "SG".
 603      * It DOES return UN M.49 code, e.g., '001', as well as ISO 3166 two letter country codes.
 604      */
 605     static String getRegionCode(String id) {
 606         return Locale.forLanguageTag(id.replaceAll("_", "-")).getCountry();
 607     }
 608 
 609     private static class KeyComparator implements Comparator<String> {
 610         static KeyComparator INSTANCE = new KeyComparator();
 611 
 612         private KeyComparator() {
 613         }
 614 
 615         @Override
 616         public int compare(String o1, String o2) {
 617             int len1 = o1.length();
 618             int len2 = o2.length();
 619             if (!isDigit(o1.charAt(0)) && !isDigit(o2.charAt(0))) {
 620                 // Shorter string comes first unless either starts with a digit.
 621                 if (len1 < len2) {
 622                     return -1;
 623                 }
 624                 if (len1 > len2) {
 625                     return 1;
 626                 }
 627             }
 628             return o1.compareTo(o2);
 629         }
 630 
 631         private boolean isDigit(char c) {
 632             return c >= '0' && c <= '9';
 633         }
 634     }
 635 
 636     private static Map<String, Object> extractLocaleNames(Map<String, Object> map, String id) {
 637         Map<String, Object> localeNames = new TreeMap<>(KeyComparator.INSTANCE);
 638         for (String key : map.keySet()) {
 639             if (key.startsWith(LOCALE_NAME_PREFIX)) {
 640                 switch (key) {
 641                     case LOCALE_SEPARATOR:
 642                         localeNames.put("ListCompositionPattern", map.get(key));
 643                         break;
 644                     case LOCALE_KEYTYPE:
 645                         localeNames.put("ListKeyTypePattern", map.get(key));
 646                         break;
 647                     default:
 648                         localeNames.put(key.substring(LOCALE_NAME_PREFIX.length()), map.get(key));
 649                         break;
 650                 }
 651             }
 652         }
 653 
 654         if (id.equals("root")) {
 655             // Add display name pattern, which is not in CLDR
 656             localeNames.put("DisplayNamePattern", "{0,choice,0#|1#{1}|2#{1} ({2})}");
 657         }
 658 
 659         return localeNames;
 660     }
 661 
 662     @SuppressWarnings("AssignmentToForLoopParameter")
 663     private static Map<String, Object> extractCurrencyNames(Map<String, Object> map, String id, String names)
 664             throws Exception {
 665         Map<String, Object> currencyNames = new TreeMap<>(KeyComparator.INSTANCE);
 666         for (String key : map.keySet()) {
 667             if (key.startsWith(CURRENCY_NAME_PREFIX)) {
 668                 currencyNames.put(key.substring(CURRENCY_NAME_PREFIX.length()), map.get(key));
 669             } else if (key.startsWith(CURRENCY_SYMBOL_PREFIX)) {
 670                 currencyNames.put(key.substring(CURRENCY_SYMBOL_PREFIX.length()), map.get(key));
 671             }
 672         }
 673         return currencyNames;
 674     }
 675 
 676     private static Map<String, Object> extractZoneNames(Map<String, Object> map, String id) {
 677         Map<String, Object> names = new HashMap<>();
 678 
 679         // Copy over missing time zone ids from JRE for English locale
 680         if (id.equals("en")) {
 681             Map<String[], String> jreMetaMap = new HashMap<>();
 682             jreTimeZoneNames.stream().forEach(e -> {
 683                 String tzid = (String)e[0];
 684                 String[] data = (String[])e[1];
 685 
 686                 if (map.get(TIMEZONE_ID_PREFIX + tzid) == null &&
 687                     handlerMetaZones.get(tzid) == null ||
 688                     handlerMetaZones.get(tzid) != null &&
 689                     map.get(METAZONE_ID_PREFIX + handlerMetaZones.get(tzid)) == null) {
 690 
 691                     // First, check the alias
 692                     String canonID = canonicalTZMap.get(tzid);
 693                     if (canonID != null && !tzid.equals(canonID)) {
 694                         Object value = map.get(TIMEZONE_ID_PREFIX + canonID);
 695                         if (value != null) {
 696                             names.put(tzid, value);
 697                             return;
 698                         } else {
 699                             String meta = handlerMetaZones.get(canonID);
 700                             if (meta != null) {
 701                                 value = map.get(METAZONE_ID_PREFIX + meta);
 702                                 if (value != null) {
 703                                     names.put(tzid, meta);
 704                                     return;
 705                                 }
 706                             }
 707                         }
 708                     }
 709 
 710                     // Check the CLDR meta key
 711                     Optional<Map.Entry<String, String>> cldrMeta =
 712                         handlerMetaZones.getData().entrySet().stream()
 713                             .filter(me ->
 714                                 Arrays.deepEquals(data,
 715                                     (String[])map.get(METAZONE_ID_PREFIX + me.getValue())))
 716                             .findAny();
 717                     cldrMeta.ifPresentOrElse(meta -> names.put(tzid, meta.getValue()), () -> {
 718                         // Check the JRE meta key, add if there is not.
 719                         Optional<Map.Entry<String[], String>> jreMeta =
 720                             jreMetaMap.entrySet().stream()
 721                                 .filter(jm -> Arrays.deepEquals(data, jm.getKey()))
 722                                 .findAny();
 723                         jreMeta.ifPresentOrElse(meta -> names.put(tzid, meta.getValue()), () -> {
 724                                 String metaName = "JRE_" + tzid.replaceAll("[/-]", "_");
 725                                 names.put(METAZONE_ID_PREFIX + metaName, data);
 726                                 names.put(tzid, metaName);
 727                         });
 728                     });
 729                 }
 730             });
 731         }
 732 
 733         getAvailableZoneIds().stream().forEach(tzid -> {
 734             // If the tzid is deprecated, get the data for the replacement id
 735             String tzKey = Optional.ofNullable((String)handlerSupplMeta.get(tzid))
 736                                    .orElse(tzid);
 737             Object data = map.get(TIMEZONE_ID_PREFIX + tzKey);
 738 
 739             if (data instanceof String[]) {
 740                 names.put(tzid, data);
 741             } else {
 742                 String meta = handlerMetaZones.get(tzKey);
 743                 if (meta != null) {
 744                     String metaKey = METAZONE_ID_PREFIX + meta;
 745                     data = map.get(metaKey);
 746                     if (data instanceof String[]) {
 747                         // Keep the metazone prefix here.
 748                         names.put(metaKey, data);
 749                         names.put(tzid, meta);
 750                     }
 751                 }
 752             }
 753         });
 754 
 755         // exemplar cities.
 756         Map<String, Object> exCities = map.entrySet().stream()
 757                 .filter(e -> e.getKey().startsWith(CLDRConverter.EXEMPLAR_CITY_PREFIX))
 758                 .collect(Collectors
 759                         .toMap(Map.Entry::getKey, Map.Entry::getValue));
 760         names.putAll(exCities);
 761 
 762         if (!id.equals("en") &&
 763             !names.isEmpty()) {
 764             // CLDR does not have UTC entry, so add it here.
 765             names.put("UTC", EMPTY_ZONE);
 766 
 767             // no metazone zones
 768             Arrays.asList(handlerMetaZones.get(MetaZonesParseHandler.NO_METAZONE_KEY)
 769                 .split("\\s")).stream()
 770                 .forEach(tz -> {
 771                     names.put(tz, EMPTY_ZONE);
 772                 });
 773         }
 774 
 775         return names;
 776     }
 777 
 778     /**
 779      * Extracts the language independent calendar data. Each of the two keys,
 780      * "firstDayOfWeek" and "minimalDaysInFirstWeek" has a string value consists of
 781      * one or multiple occurrences of:
 782      *  i: rg1 rg2 ... rgn;
 783      * where "i" is the data for the following regions (delimited by a space) after
 784      * ":", and ends with a ";".
 785      */
 786     private static Map<String, Object> extractCalendarData(Map<String, Object> map, String id) {
 787         Map<String, Object> calendarData = new LinkedHashMap<>();
 788         if (id.equals("root")) {
 789             calendarData.put("firstDayOfWeek",
 790                 IntStream.range(1, 8)
 791                     .mapToObj(String::valueOf)
 792                     .filter(d -> map.keySet().contains(CALENDAR_FIRSTDAY_PREFIX + d))
 793                     .map(d -> d + ": " + map.get(CALENDAR_FIRSTDAY_PREFIX + d))
 794                     .collect(Collectors.joining(";")));
 795             calendarData.put("minimalDaysInFirstWeek",
 796                 IntStream.range(0, 7)
 797                     .mapToObj(String::valueOf)
 798                     .filter(d -> map.keySet().contains(CALENDAR_MINDAYS_PREFIX + d))
 799                     .map(d -> d + ": " + map.get(CALENDAR_MINDAYS_PREFIX + d))
 800                     .collect(Collectors.joining(";")));
 801         }
 802         return calendarData;
 803     }
 804 
 805     static final String[] FORMAT_DATA_ELEMENTS = {
 806         "MonthNames",
 807         "standalone.MonthNames",
 808         "MonthAbbreviations",
 809         "standalone.MonthAbbreviations",
 810         "MonthNarrows",
 811         "standalone.MonthNarrows",
 812         "DayNames",
 813         "standalone.DayNames",
 814         "DayAbbreviations",
 815         "standalone.DayAbbreviations",
 816         "DayNarrows",
 817         "standalone.DayNarrows",
 818         "QuarterNames",
 819         "standalone.QuarterNames",
 820         "QuarterAbbreviations",
 821         "standalone.QuarterAbbreviations",
 822         "QuarterNarrows",
 823         "standalone.QuarterNarrows",
 824         "AmPmMarkers",
 825         "narrow.AmPmMarkers",
 826         "abbreviated.AmPmMarkers",
 827         "long.Eras",
 828         "Eras",
 829         "narrow.Eras",
 830         "field.era",
 831         "field.year",
 832         "field.month",
 833         "field.week",
 834         "field.weekday",
 835         "field.dayperiod",
 836         "field.hour",
 837         "timezone.hourFormat",
 838         "timezone.gmtFormat",
 839         "timezone.gmtZeroFormat",
 840         "timezone.regionFormat",
 841         "timezone.regionFormat.daylight",
 842         "timezone.regionFormat.standard",
 843         "field.minute",
 844         "field.second",
 845         "field.zone",
 846         "TimePatterns",
 847         "DatePatterns",
 848         "DateTimePatterns",
 849         "DateTimePatternChars"
 850     };
 851 
 852     private static Map<String, Object> extractFormatData(Map<String, Object> map, String id) {
 853         Map<String, Object> formatData = new LinkedHashMap<>();
 854         for (CalendarType calendarType : CalendarType.values()) {
 855             if (calendarType == CalendarType.GENERIC) {
 856                 continue;
 857             }
 858             String prefix = calendarType.keyElementName();
 859             for (String element : FORMAT_DATA_ELEMENTS) {
 860                 String key = prefix + element;
 861                 copyIfPresent(map, "java.time." + key, formatData);
 862                 copyIfPresent(map, key, formatData);
 863             }
 864         }
 865 
 866         for (String key : map.keySet()) {
 867         // Copy available calendar names
 868             if (key.startsWith(CLDRConverter.LOCALE_TYPE_PREFIX_CA)) {
 869                 String type = key.substring(CLDRConverter.LOCALE_TYPE_PREFIX_CA.length());
 870                 for (CalendarType calendarType : CalendarType.values()) {
 871                     if (calendarType == CalendarType.GENERIC) {
 872                         continue;
 873                     }
 874                     if (type.equals(calendarType.lname())) {
 875                         Object value = map.get(key);
 876                         String dataKey = key.replace(LOCALE_TYPE_PREFIX_CA,
 877                                 CALENDAR_NAME_PREFIX);
 878                         formatData.put(dataKey, value);
 879                         String ukey = CALENDAR_NAME_PREFIX + calendarType.uname();
 880                         if (!dataKey.equals(ukey)) {
 881                             formatData.put(ukey, value);
 882                         }
 883                     }
 884                 }
 885             }
 886         }
 887 
 888         copyIfPresent(map, "DefaultNumberingSystem", formatData);
 889 
 890         @SuppressWarnings("unchecked")
 891         List<String> numberingScripts = (List<String>) map.remove("numberingScripts");
 892         if (numberingScripts != null) {
 893             for (String script : numberingScripts) {
 894                 copyIfPresent(map, script + "." + "NumberElements", formatData);
 895             }
 896         } else {
 897             copyIfPresent(map, "NumberElements", formatData);
 898         }
 899         copyIfPresent(map, "NumberPatterns", formatData);
 900         copyIfPresent(map, "short.CompactNumberPatterns", formatData);
 901         copyIfPresent(map, "long.CompactNumberPatterns", formatData);
 902 
 903         // put extra number elements for available scripts into formatData, if it is "root"
 904         if (id.equals("root")) {
 905             handlerNumbering.keySet().stream()
 906                 .filter(k -> !numberingScripts.contains(k))
 907                 .forEach(k -> {
 908                     String[] ne = (String[])map.get("latn.NumberElements");
 909                     String[] neNew = Arrays.copyOf(ne, ne.length);
 910                     neNew[4] = handlerNumbering.get(k).substring(0, 1);
 911                     formatData.put(k + ".NumberElements", neNew);
 912                 });
 913         }
 914         return formatData;
 915     }
 916 
 917     private static void copyIfPresent(Map<String, Object> src, String key, Map<String, Object> dest) {
 918         Object value = src.get(key);
 919         if (value != null) {
 920             dest.put(key, value);
 921         }
 922     }
 923 
 924     // --- code below here is adapted from java.util.Properties ---
 925     private static final String specialSaveCharsJava = "\"";
 926     private static final String specialSaveCharsProperties = "=: \t\r\n\f#!";
 927 
 928     /*
 929      * Converts unicodes to encoded \uxxxx
 930      * and writes out any of the characters in specialSaveChars
 931      * with a preceding slash
 932      */
 933     static String saveConvert(String theString, boolean useJava) {
 934         if (theString == null) {
 935             return "";
 936         }
 937 
 938         String specialSaveChars;
 939         if (useJava) {
 940             specialSaveChars = specialSaveCharsJava;
 941         } else {
 942             specialSaveChars = specialSaveCharsProperties;
 943         }
 944         boolean escapeSpace = false;
 945 
 946         int len = theString.length();
 947         StringBuilder outBuffer = new StringBuilder(len * 2);
 948         Formatter formatter = new Formatter(outBuffer, Locale.ROOT);
 949 
 950         for (int x = 0; x < len; x++) {
 951             char aChar = theString.charAt(x);
 952             switch (aChar) {
 953             case ' ':
 954                 if (x == 0 || escapeSpace) {
 955                     outBuffer.append('\\');
 956                 }
 957                 outBuffer.append(' ');
 958                 break;
 959             case '\\':
 960                 outBuffer.append('\\');
 961                 outBuffer.append('\\');
 962                 break;
 963             case '\t':
 964                 outBuffer.append('\\');
 965                 outBuffer.append('t');
 966                 break;
 967             case '\n':
 968                 outBuffer.append('\\');
 969                 outBuffer.append('n');
 970                 break;
 971             case '\r':
 972                 outBuffer.append('\\');
 973                 outBuffer.append('r');
 974                 break;
 975             case '\f':
 976                 outBuffer.append('\\');
 977                 outBuffer.append('f');
 978                 break;
 979             default:
 980                 if (aChar < 0x0020 || (!USE_UTF8 && aChar > 0x007e)) {
 981                     formatter.format("\\u%04x", (int)aChar);
 982                 } else {
 983                     if (specialSaveChars.indexOf(aChar) != -1) {
 984                         outBuffer.append('\\');
 985                     }
 986                     outBuffer.append(aChar);
 987                 }
 988             }
 989         }
 990         return outBuffer.toString();
 991     }
 992 
 993     private static String toLanguageTag(String locName) {
 994         if (locName.indexOf('_') == -1) {
 995             return locName;
 996         }
 997         String tag = locName.replaceAll("_", "-");
 998         Locale loc = Locale.forLanguageTag(tag);
 999         return loc.toLanguageTag();
1000     }
1001 
1002     private static void addLikelySubtags(Map<String, SortedSet<String>> metaInfo, String category, String id) {
1003         String likelySubtag = handlerLikelySubtags.get(id);
1004         if (likelySubtag != null) {
1005             // Remove Script for now
1006             metaInfo.get(category).add(toLanguageTag(likelySubtag).replaceFirst("-[A-Z][a-z]{3}", ""));
1007         }
1008     }
1009 
1010     private static String toLocaleName(String tag) {
1011         if (tag.indexOf('-') == -1) {
1012             return tag;
1013         }
1014         return tag.replaceAll("-", "_");
1015     }
1016 
1017     private static void setupBaseLocales(String localeList) {
1018         Arrays.stream(localeList.split(","))
1019             .map(Locale::forLanguageTag)
1020             .map(l -> Control.getControl(Control.FORMAT_DEFAULT)
1021                              .getCandidateLocales("", l))
1022             .forEach(BASE_LOCALES::addAll);
1023     }
1024 
1025     // applying parent locale rules to the passed candidates list
1026     // This has to match with the one in sun.util.cldr.CLDRLocaleProviderAdapter
1027     private static Map<Locale, Locale> childToParentLocaleMap = null;
1028     private static List<Locale> applyParentLocales(String baseName, List<Locale> candidates) {
1029         if (Objects.isNull(childToParentLocaleMap)) {
1030             childToParentLocaleMap = new HashMap<>();
1031             parentLocalesMap.keySet().forEach(key -> {
1032                 String parent = key.substring(PARENT_LOCALE_PREFIX.length()).replaceAll("_", "-");
1033                 parentLocalesMap.get(key).stream().forEach(child -> {
1034                     childToParentLocaleMap.put(Locale.forLanguageTag(child),
1035                         "root".equals(parent) ? Locale.ROOT : Locale.forLanguageTag(parent));
1036                 });
1037             });
1038         }
1039 
1040         // check irregular parents
1041         for (int i = 0; i < candidates.size(); i++) {
1042             Locale l = candidates.get(i);
1043             Locale p = childToParentLocaleMap.get(l);
1044             if (!l.equals(Locale.ROOT) &&
1045                 Objects.nonNull(p) &&
1046                 !candidates.get(i+1).equals(p)) {
1047                 List<Locale> applied = candidates.subList(0, i+1);
1048                 applied.addAll(applyParentLocales(baseName, defCon.getCandidateLocales(baseName, p)));
1049                 return applied;
1050             }
1051         }
1052 
1053         return candidates;
1054     }
1055 
1056     private static void generateZoneName() throws Exception {
1057         Files.createDirectories(Paths.get(DESTINATION_DIR, "java", "time", "format"));
1058         Files.write(Paths.get(DESTINATION_DIR, "java", "time", "format", "ZoneName.java"),
1059             Files.lines(Paths.get(zoneNameTempFile))
1060                 .flatMap(l -> {
1061                     if (l.equals("%%%%ZIDMAP%%%%")) {
1062                         return zidMapEntry();
1063                     } else if (l.equals("%%%%MZONEMAP%%%%")) {
1064                         return handlerMetaZones.mzoneMapEntry();
1065                     } else if (l.equals("%%%%DEPRECATED%%%%")) {
1066                         return handlerSupplMeta.deprecatedMap();
1067                     } else if (l.equals("%%%%TZDATALINK%%%%")) {
1068                         return tzDataLinkEntry();
1069                     } else {
1070                         return Stream.of(l);
1071                     }
1072                 })
1073                 .collect(Collectors.toList()),
1074             StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
1075     }
1076 
1077     // This method assumes handlerMetaZones is already initialized
1078     private static Set<String> getAvailableZoneIds() {
1079         assert handlerMetaZones != null;
1080         if (AVAILABLE_TZIDS == null) {
1081             AVAILABLE_TZIDS = new HashSet(ZoneId.getAvailableZoneIds());
1082             AVAILABLE_TZIDS.addAll(handlerMetaZones.keySet());
1083             AVAILABLE_TZIDS.remove(MetaZonesParseHandler.NO_METAZONE_KEY);
1084         }
1085 
1086         return AVAILABLE_TZIDS;
1087     }
1088 
1089     private static Stream<String> zidMapEntry() {
1090         return getAvailableZoneIds().stream()
1091                 .map(id -> {
1092                     String canonId = canonicalTZMap.getOrDefault(id, id);
1093                     String meta = handlerMetaZones.get(canonId);
1094                     String zone001 = handlerMetaZones.zidMap().get(meta);
1095                     return zone001 == null ? "" :
1096                             String.format("        \"%s\", \"%s\", \"%s\",",
1097                                             id, meta, zone001);
1098                 })
1099                 .filter(s -> !s.isEmpty())
1100                 .sorted();
1101     }
1102 
1103     private static Stream<String> tzDataLinkEntry() {
1104         try {
1105             return Files.walk(Paths.get(tzDataDir), 1)
1106                 .filter(p -> !Files.isDirectory(p))
1107                 .flatMap(CLDRConverter::extractLinks)
1108                 .sorted();
1109         } catch (IOException e) {
1110             throw new UncheckedIOException(e);
1111         }
1112     }
1113 
1114     private static Stream<String> extractLinks(Path tzFile) {
1115         try {
1116             return Files.lines(tzFile)
1117                 .filter(l -> l.startsWith("Link"))
1118                 .map(l -> l.replaceFirst("^Link[\\s]+(\\S+)\\s+(\\S+).*",
1119                                          "        \"$2\", \"$1\","));
1120         } catch (IOException e) {
1121             throw new UncheckedIOException(e);
1122         }
1123     }
1124 
1125     // Generate tzmappings for Windows. The format is:
1126     //
1127     // (Windows Zone Name):(REGION):(Java TZID)
1128     //
1129     // where:
1130     //   Windows Zone Name: arbitrary time zone name string used in Windows
1131     //   REGION: ISO3166 or UN M.49 code
1132     //   Java TZID: Java's time zone ID
1133     //
1134     // Note: the entries are alphabetically sorted, *except* the "world" region
1135     // code, i.e., "001". It should be the last entry for the same windows time
1136     // zone name entries. (cf. TimeZone_md.c)
1137     private static void generateWindowsTZMappings() throws Exception {
1138         Files.createDirectories(Paths.get(DESTINATION_DIR, "windows", "conf"));
1139         Files.write(Paths.get(DESTINATION_DIR, "windows", "conf", "tzmappings"),
1140             handlerWinZones.keySet().stream()
1141                 .map(k -> k + ":" + handlerWinZones.get(k) + ":")
1142                 .sorted(new Comparator<String>() {
1143                     public int compare(String t1, String t2) {
1144                         String[] s1 = t1.split(":");
1145                         String[] s2 = t2.split(":");
1146                         if (s1[0].equals(s2[0])) {
1147                             if (s1[1].equals("001")) {
1148                                 return 1;
1149                             } else if (s2[1].equals("001")) {
1150                                 return -1;
1151                             } else {
1152                                 return s1[1].compareTo(s2[1]);
1153                             }
1154                         } else {
1155                             return s1[0].compareTo(s2[0]);
1156                         }
1157                     }
1158                 })
1159                 .collect(Collectors.toList()),
1160             StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
1161     }
1162 }