1 /*
   2  * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 
  26 package build.tools.cldrconverter;
  27 
  28 import static build.tools.cldrconverter.Bundle.jreTimeZoneNames;
  29 import build.tools.cldrconverter.BundleGenerator.BundleType;
  30 import java.io.File;
  31 import java.io.IOException;
  32 import java.io.UncheckedIOException;
  33 import java.nio.file.*;
  34 import java.text.MessageFormat;
  35 import java.time.*;
  36 import java.util.*;
  37 import java.util.ResourceBundle.Control;
  38 import java.util.logging.Level;
  39 import java.util.logging.Logger;
  40 import java.util.stream.Collectors;
  41 import java.util.stream.IntStream;
  42 import java.util.stream.Stream;
  43 import javax.xml.parsers.SAXParser;
  44 import javax.xml.parsers.SAXParserFactory;
  45 import org.xml.sax.SAXNotRecognizedException;
  46 import org.xml.sax.SAXNotSupportedException;
  47 
  48 
  49 /**
  50  * Converts locale data from "Locale Data Markup Language" format to
  51  * JRE resource bundle format. LDML is the format used by the Common
  52  * Locale Data Repository maintained by the Unicode Consortium.
  53  */
  54 public class CLDRConverter {
  55 
  56     static final String LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldml.dtd";
  57     static final String SPPL_LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldmlSupplemental.dtd";
  58     static final String BCP47_LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldmlBCP47.dtd";
  59 
  60 
  61     private static String CLDR_BASE;
  62     static String LOCAL_LDML_DTD;
  63     static String LOCAL_SPPL_LDML_DTD;
  64     static String LOCAL_BCP47_LDML_DTD;
  65     private static String SOURCE_FILE_DIR;
  66     private static String SPPL_SOURCE_FILE;
  67     private static String SPPL_META_SOURCE_FILE;
  68     private static String NUMBERING_SOURCE_FILE;
  69     private static String METAZONES_SOURCE_FILE;
  70     private static String LIKELYSUBTAGS_SOURCE_FILE;
  71     private static String TIMEZONE_SOURCE_FILE;
  72     private static String WINZONES_SOURCE_FILE;
  73     static String DESTINATION_DIR = "build/gensrc";
  74 
  75     static final String LOCALE_NAME_PREFIX = "locale.displayname.";
  76     static final String LOCALE_SEPARATOR = LOCALE_NAME_PREFIX + "separator";
  77     static final String LOCALE_KEYTYPE = LOCALE_NAME_PREFIX + "keytype";
  78     static final String LOCALE_KEY_PREFIX = LOCALE_NAME_PREFIX + "key.";
  79     static final String LOCALE_TYPE_PREFIX = LOCALE_NAME_PREFIX + "type.";
  80     static final String LOCALE_TYPE_PREFIX_CA = LOCALE_TYPE_PREFIX + "ca.";
  81     static final String CURRENCY_SYMBOL_PREFIX = "currency.symbol.";
  82     static final String CURRENCY_NAME_PREFIX = "currency.displayname.";
  83     static final String CALENDAR_NAME_PREFIX = "calendarname.";
  84     static final String CALENDAR_FIRSTDAY_PREFIX = "firstDay.";
  85     static final String CALENDAR_MINDAYS_PREFIX = "minDays.";
  86     static final String TIMEZONE_ID_PREFIX = "timezone.id.";
  87     static final String EXEMPLAR_CITY_PREFIX = "timezone.excity.";
  88     static final String ZONE_NAME_PREFIX = "timezone.displayname.";
  89     static final String METAZONE_ID_PREFIX = "metazone.id.";
  90     static final String PARENT_LOCALE_PREFIX = "parentLocale.";
  91     static final String[] EMPTY_ZONE = {"", "", "", "", "", ""};
  92 
  93     private static SupplementDataParseHandler handlerSuppl;
  94     private static LikelySubtagsParseHandler handlerLikelySubtags;
  95     private static WinZonesParseHandler handlerWinZones;
  96     static SupplementalMetadataParseHandler handlerSupplMeta;
  97     static NumberingSystemsParseHandler handlerNumbering;
  98     static MetaZonesParseHandler handlerMetaZones;
  99     static TimeZoneParseHandler handlerTimeZone;
 100     private static BundleGenerator bundleGenerator;
 101 
 102     // java.base module related
 103     static boolean isBaseModule = false;
 104     static final Set<Locale> BASE_LOCALES = new HashSet<>();
 105 
 106     // "parentLocales" map
 107     private static final Map<String, SortedSet<String>> parentLocalesMap = new HashMap<>();
 108     private static final ResourceBundle.Control defCon =
 109         ResourceBundle.Control.getControl(ResourceBundle.Control.FORMAT_DEFAULT);
 110 
 111     private static final String[] AVAILABLE_TZIDS = TimeZone.getAvailableIDs();
 112     private static String zoneNameTempFile;
 113     private static String tzDataDir;
 114     private static final Map<String, String> canonicalTZMap = new HashMap<>();
 115 
 116     static enum DraftType {
 117         UNCONFIRMED,
 118         PROVISIONAL,
 119         CONTRIBUTED,
 120         APPROVED;
 121 
 122         private static final Map<String, DraftType> map = new HashMap<>();
 123         static {
 124             for (DraftType dt : values()) {
 125                 map.put(dt.getKeyword(), dt);
 126             }
 127         }
 128         static private DraftType defaultType = CONTRIBUTED;
 129 
 130         private final String keyword;
 131 
 132         private DraftType() {
 133             keyword = this.name().toLowerCase(Locale.ROOT);
 134 
 135         }
 136 
 137         static DraftType forKeyword(String keyword) {
 138             return map.get(keyword);
 139         }
 140 
 141         static DraftType getDefault() {
 142             return defaultType;
 143         }
 144 
 145         static void setDefault(String keyword) {
 146             defaultType = Objects.requireNonNull(forKeyword(keyword));
 147         }
 148 
 149         String getKeyword() {
 150             return keyword;
 151         }
 152     }
 153 
 154     static boolean USE_UTF8 = false;
 155     private static boolean verbose;
 156 
 157     private CLDRConverter() {
 158        // no instantiation
 159     }
 160 
 161     @SuppressWarnings("AssignmentToForLoopParameter")
 162     public static void main(String[] args) throws Exception {
 163         if (args.length != 0) {
 164             String currentArg = null;
 165             try {
 166                 for (int i = 0; i < args.length; i++) {
 167                     currentArg = args[i];
 168                     switch (currentArg) {
 169                     case "-draft":
 170                         String draftDataType = args[++i];
 171                         try {
 172                             DraftType.setDefault(draftDataType);
 173                         } catch (NullPointerException e) {
 174                             severe("Error: incorrect draft value: %s%n", draftDataType);
 175                             System.exit(1);
 176                         }
 177                         info("Using the specified data type: %s%n", draftDataType);
 178                         break;
 179 
 180                     case "-base":
 181                         // base directory for input files
 182                         CLDR_BASE = args[++i];
 183                         if (!CLDR_BASE.endsWith("/")) {
 184                             CLDR_BASE += "/";
 185                         }
 186                         break;
 187 
 188                     case "-baselocales":
 189                         // base locales
 190                         setupBaseLocales(args[++i]);
 191                         break;
 192 
 193                     case "-basemodule":
 194                         // indicates java.base module resource generation
 195                         isBaseModule = true;
 196                         break;
 197 
 198                     case "-o":
 199                         // output directory
 200                         DESTINATION_DIR = args[++i];
 201                         break;
 202 
 203                     case "-utf8":
 204                         USE_UTF8 = true;
 205                         break;
 206 
 207                     case "-verbose":
 208                         verbose = true;
 209                         break;
 210 
 211                     case "-zntempfile":
 212                         zoneNameTempFile = args[++i];
 213                         break;
 214 
 215                     case "-tzdatadir":
 216                         tzDataDir = args[++i];
 217                         break;
 218 
 219                     case "-help":
 220                         usage();
 221                         System.exit(0);
 222                         break;
 223 
 224                     default:
 225                         throw new RuntimeException();
 226                     }
 227                 }
 228             } catch (RuntimeException e) {
 229                 severe("unknown or imcomplete arg(s): " + currentArg);
 230                 usage();
 231                 System.exit(1);
 232             }
 233         }
 234 
 235         // Set up path names
 236         LOCAL_LDML_DTD = CLDR_BASE + "/dtd/ldml.dtd";
 237         LOCAL_SPPL_LDML_DTD = CLDR_BASE + "/dtd/ldmlSupplemental.dtd";
 238         LOCAL_BCP47_LDML_DTD = CLDR_BASE + "/dtd/ldmlBCP47.dtd";
 239         SOURCE_FILE_DIR = CLDR_BASE + "/main";
 240         SPPL_SOURCE_FILE = CLDR_BASE + "/supplemental/supplementalData.xml";
 241         LIKELYSUBTAGS_SOURCE_FILE = CLDR_BASE + "/supplemental/likelySubtags.xml";
 242         NUMBERING_SOURCE_FILE = CLDR_BASE + "/supplemental/numberingSystems.xml";
 243         METAZONES_SOURCE_FILE = CLDR_BASE + "/supplemental/metaZones.xml";
 244         TIMEZONE_SOURCE_FILE = CLDR_BASE + "/bcp47/timezone.xml";
 245         SPPL_META_SOURCE_FILE = CLDR_BASE + "/supplemental/supplementalMetadata.xml";
 246         WINZONES_SOURCE_FILE = CLDR_BASE + "/supplemental/windowsZones.xml";
 247 
 248         if (BASE_LOCALES.isEmpty()) {
 249             setupBaseLocales("en-US");
 250         }
 251 
 252         bundleGenerator = new ResourceBundleGenerator();
 253 
 254         // Parse data independent of locales
 255         parseSupplemental();
 256         parseBCP47();
 257 
 258         List<Bundle> bundles = readBundleList();
 259         convertBundles(bundles);
 260 
 261         if (isBaseModule) {
 262             // Generate java.time.format.ZoneName.java
 263             generateZoneName();
 264 
 265             // Generate Windows tzmappings
 266             generateWindowsTZMappings();
 267         }
 268     }
 269 
 270     private static void usage() {
 271         errout("Usage: java CLDRConverter [options]%n"
 272                 + "\t-help          output this usage message and exit%n"
 273                 + "\t-verbose       output information%n"
 274                 + "\t-draft [contributed | approved | provisional | unconfirmed]%n"
 275                 + "\t\t       draft level for using data (default: contributed)%n"
 276                 + "\t-base dir      base directory for CLDR input files%n"
 277                 + "\t-basemodule    generates bundles that go into java.base module%n"
 278                 + "\t-baselocales loc(,loc)*      locales that go into the base module%n"
 279                 + "\t-o dir         output directory (default: ./build/gensrc)%n"
 280                 + "\t-zntempfile    template file for java.time.format.ZoneName.java%n"
 281                 + "\t-tzdatadir     tzdata directory for java.time.format.ZoneName.java%n"
 282                 + "\t-utf8          use UTF-8 rather than \\uxxxx (for debug)%n");
 283     }
 284 
 285     static void info(String fmt, Object... args) {
 286         if (verbose) {
 287             System.out.printf(fmt, args);
 288         }
 289     }
 290 
 291     static void info(String msg) {
 292         if (verbose) {
 293             System.out.println(msg);
 294         }
 295     }
 296 
 297     static void warning(String fmt, Object... args) {
 298         System.err.print("Warning: ");
 299         System.err.printf(fmt, args);
 300     }
 301 
 302     static void warning(String msg) {
 303         System.err.print("Warning: ");
 304         errout(msg);
 305     }
 306 
 307     static void severe(String fmt, Object... args) {
 308         System.err.print("Error: ");
 309         System.err.printf(fmt, args);
 310     }
 311 
 312     static void severe(String msg) {
 313         System.err.print("Error: ");
 314         errout(msg);
 315     }
 316 
 317     private static void errout(String msg) {
 318         if (msg.contains("%n")) {
 319             System.err.printf(msg);
 320         } else {
 321             System.err.println(msg);
 322         }
 323     }
 324 
 325     /**
 326      * Configure the parser to allow access to DTDs on the file system.
 327      */
 328     private static void enableFileAccess(SAXParser parser) throws SAXNotSupportedException {
 329         try {
 330             parser.setProperty("http://javax.xml.XMLConstants/property/accessExternalDTD", "file");
 331         } catch (SAXNotRecognizedException ignore) {
 332             // property requires >= JAXP 1.5
 333         }
 334     }
 335 
 336     private static List<Bundle> readBundleList() throws Exception {
 337         List<Bundle> retList = new ArrayList<>();
 338         Path path = FileSystems.getDefault().getPath(SOURCE_FILE_DIR);
 339         try (DirectoryStream<Path> dirStr = Files.newDirectoryStream(path)) {
 340             for (Path entry : dirStr) {
 341                 String fileName = entry.getFileName().toString();
 342                 if (fileName.endsWith(".xml")) {
 343                     String id = fileName.substring(0, fileName.indexOf('.'));
 344                     Locale cldrLoc = Locale.forLanguageTag(toLanguageTag(id));
 345                     StringBuilder sb = getCandLocales(cldrLoc);
 346                     if (sb.indexOf("root") == -1) {
 347                         sb.append("root");
 348                     }
 349                     Bundle b = new Bundle(id, sb.toString(), null, null);
 350                     // Insert the bundle for root at the top so that it will get
 351                     // processed first.
 352                     if ("root".equals(id)) {
 353                         retList.add(0, b);
 354                     } else {
 355                         retList.add(b);
 356                     }
 357                 }
 358             }
 359         }
 360         return retList;
 361     }
 362 
 363     private static final Map<String, Map<String, Object>> cldrBundles = new HashMap<>();
 364 
 365     private static Map<String, SortedSet<String>> metaInfo = new HashMap<>();
 366 
 367     static {
 368         // For generating information on supported locales.
 369         metaInfo.put("AvailableLocales", new TreeSet<>());
 370     }
 371 
 372     static Map<String, Object> getCLDRBundle(String id) throws Exception {
 373         Map<String, Object> bundle = cldrBundles.get(id);
 374         if (bundle != null) {
 375             return bundle;
 376         }
 377         File file = new File(SOURCE_FILE_DIR + File.separator + id + ".xml");
 378         if (!file.exists()) {
 379             // Skip if the file doesn't exist.
 380             return Collections.emptyMap();
 381         }
 382 
 383         info("..... main directory .....");
 384         LDMLParseHandler handler = new LDMLParseHandler(id);
 385         parseLDMLFile(file, handler);
 386 
 387         bundle = handler.getData();
 388         cldrBundles.put(id, bundle);
 389 
 390         if (id.equals("root")) {
 391             // Calendar data (firstDayOfWeek & minDaysInFirstWeek)
 392             bundle = handlerSuppl.getData("root");
 393             if (bundle != null) {
 394                 //merge two maps into one map
 395                 Map<String, Object> temp = cldrBundles.remove(id);
 396                 bundle.putAll(temp);
 397                 cldrBundles.put(id, bundle);
 398             }
 399         }
 400         return bundle;
 401     }
 402 
 403     // Parsers for data in "supplemental" directory
 404     //
 405     private static void parseSupplemental() throws Exception {
 406         // Parse SupplementalData file and store the information in the HashMap
 407         // Calendar information such as firstDay and minDay are stored in
 408         // supplementalData.xml as of CLDR1.4. Individual territory is listed
 409         // with its ISO 3166 country code while default is listed using UNM49
 410         // region and composition numerical code (001 for World.)
 411         //
 412         // SupplementalData file also provides the "parent" locales which
 413         // are othrwise not to be fallen back. Process them here as well.
 414         //
 415         handlerSuppl = new SupplementDataParseHandler();
 416         parseLDMLFile(new File(SPPL_SOURCE_FILE), handlerSuppl);
 417         Map<String, Object> parentData = handlerSuppl.getData("root");
 418         parentData.keySet().stream()
 419                 .filter(key -> key.startsWith(PARENT_LOCALE_PREFIX))
 420                 .forEach(key -> {
 421                 parentLocalesMap.put(key, new TreeSet(
 422                     Arrays.asList(((String)parentData.get(key)).split(" "))));
 423             });
 424 
 425         // Parse numberingSystems to get digit zero character information.
 426         handlerNumbering = new NumberingSystemsParseHandler();
 427         parseLDMLFile(new File(NUMBERING_SOURCE_FILE), handlerNumbering);
 428 
 429         // Parse metaZones to create mappings between Olson tzids and CLDR meta zone names
 430         handlerMetaZones = new MetaZonesParseHandler();
 431         parseLDMLFile(new File(METAZONES_SOURCE_FILE), handlerMetaZones);
 432 
 433         // Parse likelySubtags
 434         handlerLikelySubtags = new LikelySubtagsParseHandler();
 435         parseLDMLFile(new File(LIKELYSUBTAGS_SOURCE_FILE), handlerLikelySubtags);
 436 
 437         // Parse supplementalMetadata
 438         // Currently interested in deprecated time zone ids and language aliases.
 439         handlerSupplMeta = new SupplementalMetadataParseHandler();
 440         parseLDMLFile(new File(SPPL_META_SOURCE_FILE), handlerSupplMeta);
 441 
 442         // Parse windowsZones
 443         handlerWinZones = new WinZonesParseHandler();
 444         parseLDMLFile(new File(WINZONES_SOURCE_FILE), handlerWinZones);
 445     }
 446 
 447     // Parsers for data in "bcp47" directory
 448     //
 449     private static void parseBCP47() throws Exception {
 450         // Parse timezone
 451         handlerTimeZone = new TimeZoneParseHandler();
 452         parseLDMLFile(new File(TIMEZONE_SOURCE_FILE), handlerTimeZone);
 453 
 454         // canonical tz name map
 455         // alias -> primary
 456         handlerTimeZone.getData().forEach((k, v) -> {
 457             String[] ids = ((String)v).split("\\s");
 458             for (int i = 1; i < ids.length; i++) {
 459                 canonicalTZMap.put(ids[i], ids[0]);
 460             }
 461         });
 462     }
 463 
 464     private static void parseLDMLFile(File srcfile, AbstractLDMLHandler handler) throws Exception {
 465         info("..... Parsing " + srcfile.getName() + " .....");
 466         SAXParserFactory pf = SAXParserFactory.newInstance();
 467         pf.setValidating(true);
 468         SAXParser parser = pf.newSAXParser();
 469         enableFileAccess(parser);
 470         parser.parse(srcfile, handler);
 471     }
 472 
 473     private static StringBuilder getCandLocales(Locale cldrLoc) {
 474         List<Locale> candList = getCandidateLocales(cldrLoc);
 475         StringBuilder sb = new StringBuilder();
 476         for (Locale loc : candList) {
 477             if (!loc.equals(Locale.ROOT)) {
 478                 sb.append(toLocaleName(loc.toLanguageTag()));
 479                 sb.append(",");
 480             }
 481         }
 482         return sb;
 483     }
 484 
 485     private static List<Locale> getCandidateLocales(Locale cldrLoc) {
 486         List<Locale> candList = new ArrayList<>();
 487         candList = applyParentLocales("", defCon.getCandidateLocales("",  cldrLoc));
 488         return candList;
 489     }
 490 
 491     private static void convertBundles(List<Bundle> bundles) throws Exception {
 492         // parent locales map. The mappings are put in base metaInfo file
 493         // for now.
 494         if (isBaseModule) {
 495             metaInfo.putAll(parentLocalesMap);
 496         }
 497 
 498         for (Bundle bundle : bundles) {
 499             // Get the target map, which contains all the data that should be
 500             // visible for the bundle's locale
 501 
 502             Map<String, Object> targetMap = bundle.getTargetMap();
 503 
 504             EnumSet<Bundle.Type> bundleTypes = bundle.getBundleTypes();
 505 
 506             if (bundle.isRoot()) {
 507                 // Add DateTimePatternChars because CLDR no longer supports localized patterns.
 508                 targetMap.put("DateTimePatternChars", "GyMdkHmsSEDFwWahKzZ");
 509             }
 510 
 511             // Now the map contains just the entries that need to be in the resources bundles.
 512             // Go ahead and generate them.
 513             if (bundleTypes.contains(Bundle.Type.LOCALENAMES)) {
 514                 Map<String, Object> localeNamesMap = extractLocaleNames(targetMap, bundle.getID());
 515                 if (!localeNamesMap.isEmpty() || bundle.isRoot()) {
 516                     bundleGenerator.generateBundle("util", "LocaleNames", bundle.getJavaID(), true, localeNamesMap, BundleType.OPEN);
 517                 }
 518             }
 519             if (bundleTypes.contains(Bundle.Type.CURRENCYNAMES)) {
 520                 Map<String, Object> currencyNamesMap = extractCurrencyNames(targetMap, bundle.getID(), bundle.getCurrencies());
 521                 if (!currencyNamesMap.isEmpty() || bundle.isRoot()) {
 522                     bundleGenerator.generateBundle("util", "CurrencyNames", bundle.getJavaID(), true, currencyNamesMap, BundleType.OPEN);
 523                 }
 524             }
 525             if (bundleTypes.contains(Bundle.Type.TIMEZONENAMES)) {
 526                 Map<String, Object> zoneNamesMap = extractZoneNames(targetMap, bundle.getID());
 527                 if (!zoneNamesMap.isEmpty() || bundle.isRoot()) {
 528                     bundleGenerator.generateBundle("util", "TimeZoneNames", bundle.getJavaID(), true, zoneNamesMap, BundleType.TIMEZONE);
 529                 }
 530             }
 531             if (bundleTypes.contains(Bundle.Type.CALENDARDATA)) {
 532                 Map<String, Object> calendarDataMap = extractCalendarData(targetMap, bundle.getID());
 533                 if (!calendarDataMap.isEmpty() || bundle.isRoot()) {
 534                     bundleGenerator.generateBundle("util", "CalendarData", bundle.getJavaID(), true, calendarDataMap, BundleType.PLAIN);
 535                 }
 536             }
 537             if (bundleTypes.contains(Bundle.Type.FORMATDATA)) {
 538                 Map<String, Object> formatDataMap = extractFormatData(targetMap, bundle.getID());
 539                 if (!formatDataMap.isEmpty() || bundle.isRoot()) {
 540                     bundleGenerator.generateBundle("text", "FormatData", bundle.getJavaID(), true, formatDataMap, BundleType.PLAIN);
 541                 }
 542             }
 543 
 544             // For AvailableLocales
 545             metaInfo.get("AvailableLocales").add(toLanguageTag(bundle.getID()));
 546             addLikelySubtags(metaInfo, "AvailableLocales", bundle.getID());
 547         }
 548         bundleGenerator.generateMetaInfo(metaInfo);
 549     }
 550 
 551     static final Map<String, String> aliases = new HashMap<>();
 552 
 553     /**
 554      * Translate the aliases into the real entries in the bundle map.
 555      */
 556     static void handleAliases(Map<String, Object> bundleMap) {
 557         Set bundleKeys = bundleMap.keySet();
 558         try {
 559             for (String key : aliases.keySet()) {
 560                 String targetKey = aliases.get(key);
 561                 if (bundleKeys.contains(targetKey)) {
 562                     bundleMap.putIfAbsent(key, bundleMap.get(targetKey));
 563                 }
 564             }
 565         } catch (Exception ex) {
 566             Logger.getLogger(CLDRConverter.class.getName()).log(Level.SEVERE, null, ex);
 567         }
 568     }
 569 
 570     /*
 571      * Returns the language portion of the given id.
 572      * If id is "root", "" is returned.
 573      */
 574     static String getLanguageCode(String id) {
 575         return "root".equals(id) ? "" : Locale.forLanguageTag(id.replaceAll("_", "-")).getLanguage();
 576     }
 577 
 578     /**
 579      * Examine if the id includes the country (territory) code. If it does, it returns
 580      * the country code.
 581      * Otherwise, it returns null. eg. when the id is "zh_Hans_SG", it return "SG".
 582      * It does NOT return UN M.49 code, e.g., '001', as those three digit numbers cannot
 583      * be translated into package names.
 584      */
 585     static String getCountryCode(String id) {
 586         String rgn = getRegionCode(id);
 587         return rgn.length() == 2 ? rgn: null;
 588     }
 589 
 590     /**
 591      * Examine if the id includes the region code. If it does, it returns
 592      * the region code.
 593      * Otherwise, it returns null. eg. when the id is "zh_Hans_SG", it return "SG".
 594      * It DOES return UN M.49 code, e.g., '001', as well as ISO 3166 two letter country codes.
 595      */
 596     static String getRegionCode(String id) {
 597         return Locale.forLanguageTag(id.replaceAll("_", "-")).getCountry();
 598     }
 599 
 600     private static class KeyComparator implements Comparator<String> {
 601         static KeyComparator INSTANCE = new KeyComparator();
 602 
 603         private KeyComparator() {
 604         }
 605 
 606         @Override
 607         public int compare(String o1, String o2) {
 608             int len1 = o1.length();
 609             int len2 = o2.length();
 610             if (!isDigit(o1.charAt(0)) && !isDigit(o2.charAt(0))) {
 611                 // Shorter string comes first unless either starts with a digit.
 612                 if (len1 < len2) {
 613                     return -1;
 614                 }
 615                 if (len1 > len2) {
 616                     return 1;
 617                 }
 618             }
 619             return o1.compareTo(o2);
 620         }
 621 
 622         private boolean isDigit(char c) {
 623             return c >= '0' && c <= '9';
 624         }
 625     }
 626 
 627     private static Map<String, Object> extractLocaleNames(Map<String, Object> map, String id) {
 628         Map<String, Object> localeNames = new TreeMap<>(KeyComparator.INSTANCE);
 629         for (String key : map.keySet()) {
 630             if (key.startsWith(LOCALE_NAME_PREFIX)) {
 631                 switch (key) {
 632                     case LOCALE_SEPARATOR:
 633                         localeNames.put("ListCompositionPattern", map.get(key));
 634                         break;
 635                     case LOCALE_KEYTYPE:
 636                         localeNames.put("ListKeyTypePattern", map.get(key));
 637                         break;
 638                     default:
 639                         localeNames.put(key.substring(LOCALE_NAME_PREFIX.length()), map.get(key));
 640                         break;
 641                 }
 642             }
 643         }
 644 
 645         if (id.equals("root")) {
 646             // Add display name pattern, which is not in CLDR
 647             localeNames.put("DisplayNamePattern", "{0,choice,0#|1#{1}|2#{1} ({2})}");
 648         }
 649 
 650         return localeNames;
 651     }
 652 
 653     @SuppressWarnings("AssignmentToForLoopParameter")
 654     private static Map<String, Object> extractCurrencyNames(Map<String, Object> map, String id, String names)
 655             throws Exception {
 656         Map<String, Object> currencyNames = new TreeMap<>(KeyComparator.INSTANCE);
 657         for (String key : map.keySet()) {
 658             if (key.startsWith(CURRENCY_NAME_PREFIX)) {
 659                 currencyNames.put(key.substring(CURRENCY_NAME_PREFIX.length()), map.get(key));
 660             } else if (key.startsWith(CURRENCY_SYMBOL_PREFIX)) {
 661                 currencyNames.put(key.substring(CURRENCY_SYMBOL_PREFIX.length()), map.get(key));
 662             }
 663         }
 664         return currencyNames;
 665     }
 666 
 667     private static Map<String, Object> extractZoneNames(Map<String, Object> map, String id) {
 668         Map<String, Object> names = new HashMap<>();
 669 
 670         // Copy over missing time zone ids from JRE for English locale
 671         if (id.equals("en")) {
 672             Map<String[], String> jreMetaMap = new HashMap<>();
 673             jreTimeZoneNames.stream().forEach(e -> {
 674                 String tzid = (String)e[0];
 675                 String[] data = (String[])e[1];
 676 
 677                 if (map.get(TIMEZONE_ID_PREFIX + tzid) == null &&
 678                     handlerMetaZones.get(tzid) == null ||
 679                     handlerMetaZones.get(tzid) != null &&
 680                     map.get(METAZONE_ID_PREFIX + handlerMetaZones.get(tzid)) == null) {
 681 
 682                     // First, check the alias
 683                     String canonID = canonicalTZMap.get(tzid);
 684                     if (canonID != null && !tzid.equals(canonID)) {
 685                         Object value = map.get(TIMEZONE_ID_PREFIX + canonID);
 686                         if (value != null) {
 687                             names.put(tzid, value);
 688                             return;
 689                         } else {
 690                             String meta = handlerMetaZones.get(canonID);
 691                             if (meta != null) {
 692                                 value = map.get(METAZONE_ID_PREFIX + meta);
 693                                 if (value != null) {
 694                                     names.put(tzid, meta);
 695                                     return;
 696                                 }
 697                             }
 698                         }
 699                     }
 700 
 701                     // Check the CLDR meta key
 702                     Optional<Map.Entry<String, String>> cldrMeta =
 703                         handlerMetaZones.getData().entrySet().stream()
 704                             .filter(me ->
 705                                 Arrays.deepEquals(data,
 706                                     (String[])map.get(METAZONE_ID_PREFIX + me.getValue())))
 707                             .findAny();
 708                     cldrMeta.ifPresentOrElse(meta -> names.put(tzid, meta.getValue()), () -> {
 709                         // Check the JRE meta key, add if there is not.
 710                         Optional<Map.Entry<String[], String>> jreMeta =
 711                             jreMetaMap.entrySet().stream()
 712                                 .filter(jm -> Arrays.deepEquals(data, jm.getKey()))
 713                                 .findAny();
 714                         jreMeta.ifPresentOrElse(meta -> names.put(tzid, meta.getValue()), () -> {
 715                                 String metaName = "JRE_" + tzid.replaceAll("[/-]", "_");
 716                                 names.put(METAZONE_ID_PREFIX + metaName, data);
 717                                 names.put(tzid, metaName);
 718                         });
 719                     });
 720                 }
 721             });
 722         }
 723 
 724         Arrays.stream(AVAILABLE_TZIDS).forEach(tzid -> {
 725             // If the tzid is deprecated, get the data for the replacement id
 726             String tzKey = Optional.ofNullable((String)handlerSupplMeta.get(tzid))
 727                                    .orElse(tzid);
 728             Object data = map.get(TIMEZONE_ID_PREFIX + tzKey);
 729 
 730             if (data instanceof String[]) {
 731                 names.put(tzid, data);
 732             } else {
 733                 String meta = handlerMetaZones.get(tzKey);
 734                 if (meta != null) {
 735                     String metaKey = METAZONE_ID_PREFIX + meta;
 736                     data = map.get(metaKey);
 737                     if (data instanceof String[]) {
 738                         // Keep the metazone prefix here.
 739                         names.put(metaKey, data);
 740                         names.put(tzid, meta);
 741                     }
 742                 }
 743             }
 744         });
 745 
 746         // exemplar cities.
 747         Map<String, Object> exCities = map.entrySet().stream()
 748                 .filter(e -> e.getKey().startsWith(CLDRConverter.EXEMPLAR_CITY_PREFIX))
 749                 .collect(Collectors
 750                         .toMap(Map.Entry::getKey, Map.Entry::getValue));
 751         names.putAll(exCities);
 752 
 753         if (!id.equals("en") &&
 754             !names.isEmpty()) {
 755             // CLDR does not have UTC entry, so add it here.
 756             names.put("UTC", EMPTY_ZONE);
 757 
 758             // no metazone zones
 759             Arrays.asList(handlerMetaZones.get(MetaZonesParseHandler.NO_METAZONE_KEY)
 760                 .split("\\s")).stream()
 761                 .forEach(tz -> {
 762                     names.put(tz, EMPTY_ZONE);
 763                 });
 764         }
 765 
 766         return names;
 767     }
 768 
 769     /**
 770      * Extracts the language independent calendar data. Each of the two keys,
 771      * "firstDayOfWeek" and "minimalDaysInFirstWeek" has a string value consists of
 772      * one or multiple occurrences of:
 773      *  i: rg1 rg2 ... rgn;
 774      * where "i" is the data for the following regions (delimited by a space) after
 775      * ":", and ends with a ";".
 776      */
 777     private static Map<String, Object> extractCalendarData(Map<String, Object> map, String id) {
 778         Map<String, Object> calendarData = new LinkedHashMap<>();
 779         if (id.equals("root")) {
 780             calendarData.put("firstDayOfWeek",
 781                 IntStream.range(1, 8)
 782                     .mapToObj(String::valueOf)
 783                     .filter(d -> map.keySet().contains(CALENDAR_FIRSTDAY_PREFIX + d))
 784                     .map(d -> d + ": " + map.get(CALENDAR_FIRSTDAY_PREFIX + d))
 785                     .collect(Collectors.joining(";")));
 786             calendarData.put("minimalDaysInFirstWeek",
 787                 IntStream.range(0, 7)
 788                     .mapToObj(String::valueOf)
 789                     .filter(d -> map.keySet().contains(CALENDAR_MINDAYS_PREFIX + d))
 790                     .map(d -> d + ": " + map.get(CALENDAR_MINDAYS_PREFIX + d))
 791                     .collect(Collectors.joining(";")));
 792         }
 793         return calendarData;
 794     }
 795 
 796     static final String[] FORMAT_DATA_ELEMENTS = {
 797         "MonthNames",
 798         "standalone.MonthNames",
 799         "MonthAbbreviations",
 800         "standalone.MonthAbbreviations",
 801         "MonthNarrows",
 802         "standalone.MonthNarrows",
 803         "DayNames",
 804         "standalone.DayNames",
 805         "DayAbbreviations",
 806         "standalone.DayAbbreviations",
 807         "DayNarrows",
 808         "standalone.DayNarrows",
 809         "QuarterNames",
 810         "standalone.QuarterNames",
 811         "QuarterAbbreviations",
 812         "standalone.QuarterAbbreviations",
 813         "QuarterNarrows",
 814         "standalone.QuarterNarrows",
 815         "AmPmMarkers",
 816         "narrow.AmPmMarkers",
 817         "abbreviated.AmPmMarkers",
 818         "long.Eras",
 819         "Eras",
 820         "narrow.Eras",
 821         "field.era",
 822         "field.year",
 823         "field.month",
 824         "field.week",
 825         "field.weekday",
 826         "field.dayperiod",
 827         "field.hour",
 828         "timezone.hourFormat",
 829         "timezone.gmtFormat",
 830         "timezone.gmtZeroFormat",
 831         "timezone.regionFormat",
 832         "timezone.regionFormat.daylight",
 833         "timezone.regionFormat.standard",
 834         "field.minute",
 835         "field.second",
 836         "field.zone",
 837         "TimePatterns",
 838         "DatePatterns",
 839         "DateTimePatterns",
 840         "DateTimePatternChars"
 841     };
 842 
 843     private static Map<String, Object> extractFormatData(Map<String, Object> map, String id) {
 844         Map<String, Object> formatData = new LinkedHashMap<>();
 845         for (CalendarType calendarType : CalendarType.values()) {
 846             if (calendarType == CalendarType.GENERIC) {
 847                 continue;
 848             }
 849             String prefix = calendarType.keyElementName();
 850             for (String element : FORMAT_DATA_ELEMENTS) {
 851                 String key = prefix + element;
 852                 copyIfPresent(map, "java.time." + key, formatData);
 853                 copyIfPresent(map, key, formatData);
 854             }
 855         }
 856 
 857         for (String key : map.keySet()) {
 858         // Copy available calendar names
 859             if (key.startsWith(CLDRConverter.LOCALE_TYPE_PREFIX_CA)) {
 860                 String type = key.substring(CLDRConverter.LOCALE_TYPE_PREFIX_CA.length());
 861                 for (CalendarType calendarType : CalendarType.values()) {
 862                     if (calendarType == CalendarType.GENERIC) {
 863                         continue;
 864                     }
 865                     if (type.equals(calendarType.lname())) {
 866                         Object value = map.get(key);
 867                         String dataKey = key.replace(LOCALE_TYPE_PREFIX_CA,
 868                                 CALENDAR_NAME_PREFIX);
 869                         formatData.put(dataKey, value);
 870                         String ukey = CALENDAR_NAME_PREFIX + calendarType.uname();
 871                         if (!dataKey.equals(ukey)) {
 872                             formatData.put(ukey, value);
 873                         }
 874                     }
 875                 }
 876             }
 877         }
 878 
 879         copyIfPresent(map, "DefaultNumberingSystem", formatData);
 880 
 881         @SuppressWarnings("unchecked")
 882         List<String> numberingScripts = (List<String>) map.remove("numberingScripts");
 883         if (numberingScripts != null) {
 884             for (String script : numberingScripts) {
 885                 copyIfPresent(map, script + "." + "NumberElements", formatData);
 886             }
 887         } else {
 888             copyIfPresent(map, "NumberElements", formatData);
 889         }
 890         copyIfPresent(map, "NumberPatterns", formatData);
 891 
 892         // put extra number elements for available scripts into formatData, if it is "root"
 893         if (id.equals("root")) {
 894             handlerNumbering.keySet().stream()
 895                 .filter(k -> !numberingScripts.contains(k))
 896                 .forEach(k -> {
 897                     String[] ne = (String[])map.get("latn.NumberElements");
 898                     String[] neNew = Arrays.copyOf(ne, ne.length);
 899                     neNew[4] = handlerNumbering.get(k).substring(0, 1);
 900                     formatData.put(k + ".NumberElements", neNew);
 901                 });
 902         }
 903         return formatData;
 904     }
 905 
 906     private static void copyIfPresent(Map<String, Object> src, String key, Map<String, Object> dest) {
 907         Object value = src.get(key);
 908         if (value != null) {
 909             dest.put(key, value);
 910         }
 911     }
 912 
 913     // --- code below here is adapted from java.util.Properties ---
 914     private static final String specialSaveCharsJava = "\"";
 915     private static final String specialSaveCharsProperties = "=: \t\r\n\f#!";
 916 
 917     /*
 918      * Converts unicodes to encoded \uxxxx
 919      * and writes out any of the characters in specialSaveChars
 920      * with a preceding slash
 921      */
 922     static String saveConvert(String theString, boolean useJava) {
 923         if (theString == null) {
 924             return "";
 925         }
 926 
 927         String specialSaveChars;
 928         if (useJava) {
 929             specialSaveChars = specialSaveCharsJava;
 930         } else {
 931             specialSaveChars = specialSaveCharsProperties;
 932         }
 933         boolean escapeSpace = false;
 934 
 935         int len = theString.length();
 936         StringBuilder outBuffer = new StringBuilder(len * 2);
 937         Formatter formatter = new Formatter(outBuffer, Locale.ROOT);
 938 
 939         for (int x = 0; x < len; x++) {
 940             char aChar = theString.charAt(x);
 941             switch (aChar) {
 942             case ' ':
 943                 if (x == 0 || escapeSpace) {
 944                     outBuffer.append('\\');
 945                 }
 946                 outBuffer.append(' ');
 947                 break;
 948             case '\\':
 949                 outBuffer.append('\\');
 950                 outBuffer.append('\\');
 951                 break;
 952             case '\t':
 953                 outBuffer.append('\\');
 954                 outBuffer.append('t');
 955                 break;
 956             case '\n':
 957                 outBuffer.append('\\');
 958                 outBuffer.append('n');
 959                 break;
 960             case '\r':
 961                 outBuffer.append('\\');
 962                 outBuffer.append('r');
 963                 break;
 964             case '\f':
 965                 outBuffer.append('\\');
 966                 outBuffer.append('f');
 967                 break;
 968             default:
 969                 if (aChar < 0x0020 || (!USE_UTF8 && aChar > 0x007e)) {
 970                     formatter.format("\\u%04x", (int)aChar);
 971                 } else {
 972                     if (specialSaveChars.indexOf(aChar) != -1) {
 973                         outBuffer.append('\\');
 974                     }
 975                     outBuffer.append(aChar);
 976                 }
 977             }
 978         }
 979         return outBuffer.toString();
 980     }
 981 
 982     private static String toLanguageTag(String locName) {
 983         if (locName.indexOf('_') == -1) {
 984             return locName;
 985         }
 986         String tag = locName.replaceAll("_", "-");
 987         Locale loc = Locale.forLanguageTag(tag);
 988         return loc.toLanguageTag();
 989     }
 990 
 991     private static void addLikelySubtags(Map<String, SortedSet<String>> metaInfo, String category, String id) {
 992         String likelySubtag = handlerLikelySubtags.get(id);
 993         if (likelySubtag != null) {
 994             // Remove Script for now
 995             metaInfo.get(category).add(toLanguageTag(likelySubtag).replaceFirst("-[A-Z][a-z]{3}", ""));
 996         }
 997     }
 998 
 999     private static String toLocaleName(String tag) {
1000         if (tag.indexOf('-') == -1) {
1001             return tag;
1002         }
1003         return tag.replaceAll("-", "_");
1004     }
1005 
1006     private static void setupBaseLocales(String localeList) {
1007         Arrays.stream(localeList.split(","))
1008             .map(Locale::forLanguageTag)
1009             .map(l -> Control.getControl(Control.FORMAT_DEFAULT)
1010                              .getCandidateLocales("", l))
1011             .forEach(BASE_LOCALES::addAll);
1012     }
1013 
1014     // applying parent locale rules to the passed candidates list
1015     // This has to match with the one in sun.util.cldr.CLDRLocaleProviderAdapter
1016     private static Map<Locale, Locale> childToParentLocaleMap = null;
1017     private static List<Locale> applyParentLocales(String baseName, List<Locale> candidates) {
1018         if (Objects.isNull(childToParentLocaleMap)) {
1019             childToParentLocaleMap = new HashMap<>();
1020             parentLocalesMap.keySet().forEach(key -> {
1021                 String parent = key.substring(PARENT_LOCALE_PREFIX.length()).replaceAll("_", "-");
1022                 parentLocalesMap.get(key).stream().forEach(child -> {
1023                     childToParentLocaleMap.put(Locale.forLanguageTag(child),
1024                         "root".equals(parent) ? Locale.ROOT : Locale.forLanguageTag(parent));
1025                 });
1026             });
1027         }
1028 
1029         // check irregular parents
1030         for (int i = 0; i < candidates.size(); i++) {
1031             Locale l = candidates.get(i);
1032             Locale p = childToParentLocaleMap.get(l);
1033             if (!l.equals(Locale.ROOT) &&
1034                 Objects.nonNull(p) &&
1035                 !candidates.get(i+1).equals(p)) {
1036                 List<Locale> applied = candidates.subList(0, i+1);
1037                 applied.addAll(applyParentLocales(baseName, defCon.getCandidateLocales(baseName, p)));
1038                 return applied;
1039             }
1040         }
1041 
1042         return candidates;
1043     }
1044 
1045     private static void generateZoneName() throws Exception {
1046         Files.createDirectories(Paths.get(DESTINATION_DIR, "java", "time", "format"));
1047         Files.write(Paths.get(DESTINATION_DIR, "java", "time", "format", "ZoneName.java"),
1048             Files.lines(Paths.get(zoneNameTempFile))
1049                 .flatMap(l -> {
1050                     if (l.equals("%%%%ZIDMAP%%%%")) {
1051                         return zidMapEntry();
1052                     } else if (l.equals("%%%%MZONEMAP%%%%")) {
1053                         return handlerMetaZones.mzoneMapEntry();
1054                     } else if (l.equals("%%%%DEPRECATED%%%%")) {
1055                         return handlerSupplMeta.deprecatedMap();
1056                     } else if (l.equals("%%%%TZDATALINK%%%%")) {
1057                         return tzDataLinkEntry();
1058                     } else {
1059                         return Stream.of(l);
1060                     }
1061                 })
1062                 .collect(Collectors.toList()),
1063             StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
1064     }
1065 
1066     private static Stream<String> zidMapEntry() {
1067         return ZoneId.getAvailableZoneIds().stream()
1068                 .map(id -> {
1069                     String canonId = canonicalTZMap.getOrDefault(id, id);
1070                     String meta = handlerMetaZones.get(canonId);
1071                     String zone001 = handlerMetaZones.zidMap().get(meta);
1072                     return zone001 == null ? "" :
1073                             String.format("        \"%s\", \"%s\", \"%s\",",
1074                                             id, meta, zone001);
1075                 })
1076                 .filter(s -> !s.isEmpty())
1077                 .sorted();
1078     }
1079 
1080     private static Stream<String> tzDataLinkEntry() {
1081         try {
1082             return Files.walk(Paths.get(tzDataDir), 1)
1083                 .filter(p -> !Files.isDirectory(p))
1084                 .flatMap(CLDRConverter::extractLinks)
1085                 .sorted();
1086         } catch (IOException e) {
1087             throw new UncheckedIOException(e);
1088         }
1089     }
1090 
1091     private static Stream<String> extractLinks(Path tzFile) {
1092         try {
1093             return Files.lines(tzFile)
1094                 .filter(l -> l.startsWith("Link"))
1095                 .map(l -> l.replaceFirst("^Link[\\s]+(\\S+)\\s+(\\S+).*",
1096                                          "        \"$2\", \"$1\","));
1097         } catch (IOException e) {
1098             throw new UncheckedIOException(e);
1099         }
1100     }
1101 
1102     // Generate tzmappings for Windows. The format is:
1103     //
1104     // (Windows Zone Name):(REGION):(Java TZID)
1105     //
1106     // where:
1107     //   Windows Zone Name: arbitrary time zone name string used in Windows
1108     //   REGION: ISO3166 or UN M.49 code
1109     //   Java TZID: Java's time zone ID
1110     //
1111     // Note: the entries are alphabetically sorted, *except* the "world" region
1112     // code, i.e., "001". It should be the last entry for the same windows time
1113     // zone name entries. (cf. TimeZone_md.c)
1114     private static void generateWindowsTZMappings() throws Exception {
1115         Files.createDirectories(Paths.get(DESTINATION_DIR, "windows", "conf"));
1116         Files.write(Paths.get(DESTINATION_DIR, "windows", "conf", "tzmappings"),
1117             handlerWinZones.keySet().stream()
1118                 .map(k -> k + ":" + handlerWinZones.get(k) + ":")
1119                 .sorted(new Comparator<String>() {
1120                     public int compare(String t1, String t2) {
1121                         String[] s1 = t1.split(":");
1122                         String[] s2 = t2.split(":");
1123                         if (s1[0].equals(s2[0])) {
1124                             if (s1[1].equals("001")) {
1125                                 return 1;
1126                             } else if (s2[1].equals("001")) {
1127                                 return -1;
1128                             } else {
1129                                 return s1[1].compareTo(s2[1]);
1130                             }
1131                         } else {
1132                             return s1[0].compareTo(s2[0]);
1133                         }
1134                     }
1135                 })
1136                 .collect(Collectors.toList()),
1137             StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
1138     }
1139 }