1 /*
   2  * Copyright (c) 2012, 2017, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 
  26 package build.tools.cldrconverter;
  27 
  28 import static build.tools.cldrconverter.Bundle.jreTimeZoneNames;
  29 import build.tools.cldrconverter.BundleGenerator.BundleType;
  30 import java.io.File;
  31 import java.nio.file.DirectoryStream;
  32 import java.nio.file.FileSystems;
  33 import java.nio.file.Files;
  34 import java.nio.file.Path;
  35 import java.util.*;
  36 import java.util.ResourceBundle.Control;
  37 import java.util.logging.Level;
  38 import java.util.logging.Logger;
  39 import java.util.stream.Collectors;
  40 import javax.xml.parsers.SAXParser;
  41 import javax.xml.parsers.SAXParserFactory;
  42 import org.xml.sax.SAXNotRecognizedException;
  43 import org.xml.sax.SAXNotSupportedException;
  44 
  45 
  46 /**
  47  * Converts locale data from "Locale Data Markup Language" format to
  48  * JRE resource bundle format. LDML is the format used by the Common
  49  * Locale Data Repository maintained by the Unicode Consortium.
  50  */
  51 public class CLDRConverter {
  52 
  53     static final String LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldml.dtd";
  54     static final String SPPL_LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldmlSupplemental.dtd";
  55     static final String BCP47_LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldmlBCP47.dtd";
  56   
  57 
  58     private static String CLDR_BASE = "../CLDR/21.0.1/";
  59     static String LOCAL_LDML_DTD;
  60     static String LOCAL_SPPL_LDML_DTD;
  61     static String LOCAL_BCP47_LDML_DTD;  
  62     private static String SOURCE_FILE_DIR;
  63     private static String SPPL_SOURCE_FILE;
  64     private static String NUMBERING_SOURCE_FILE;
  65     private static String METAZONES_SOURCE_FILE;
  66     private static String LIKELYSUBTAGS_SOURCE_FILE;
  67     private static String TIMEZONE_SOURCE_FILE;
  68     static String DESTINATION_DIR = "build/gensrc";
  69 
  70     static final String LOCALE_NAME_PREFIX = "locale.displayname.";
  71     static final String LOCALE_SEPARATOR = LOCALE_NAME_PREFIX + "separator";
  72     static final String LOCALE_KEYTYPE = LOCALE_NAME_PREFIX + "keytype";    
  73     static final String LOCALE_KEY_PREFIX = LOCALE_NAME_PREFIX + "key.";
  74     static final String LOCALE_TYPE_PREFIX = LOCALE_NAME_PREFIX + "type.";
  75     static final String LOCALE_TYPE_PREFIX_CA = LOCALE_TYPE_PREFIX + "ca.";
  76     static final String CURRENCY_SYMBOL_PREFIX = "currency.symbol.";
  77     static final String CURRENCY_NAME_PREFIX = "currency.displayname.";
  78     static final String CALENDAR_NAME_PREFIX = "calendarname.";
  79     static final String TIMEZONE_ID_PREFIX = "timezone.id.";
  80     static final String ZONE_NAME_PREFIX = "timezone.displayname.";
  81     static final String METAZONE_ID_PREFIX = "metazone.id.";
  82     static final String PARENT_LOCALE_PREFIX = "parentLocale.";
  83 
  84     private static SupplementDataParseHandler handlerSuppl;
  85     private static LikelySubtagsParseHandler handlerLikelySubtags;
  86     static NumberingSystemsParseHandler handlerNumbering;
  87     static MetaZonesParseHandler handlerMetaZones;
  88     static TimeZoneParseHandler handlerTimeZone;
  89     private static BundleGenerator bundleGenerator;
  90 
  91     // java.base module related
  92     static boolean isBaseModule = false;
  93     static final Set<Locale> BASE_LOCALES = new HashSet<>();
  94 
  95     // "parentLocales" map
  96     private static final Map<String, SortedSet<String>> parentLocalesMap = new HashMap<>();
  97     private static final ResourceBundle.Control defCon =
  98         ResourceBundle.Control.getControl(ResourceBundle.Control.FORMAT_DEFAULT);
  99 
 100     static enum DraftType {
 101         UNCONFIRMED,
 102         PROVISIONAL,
 103         CONTRIBUTED,
 104         APPROVED;
 105 
 106         private static final Map<String, DraftType> map = new HashMap<>();
 107         static {
 108             for (DraftType dt : values()) {
 109                 map.put(dt.getKeyword(), dt);
 110             }
 111         }
 112         static private DraftType defaultType = CONTRIBUTED;
 113 
 114         private final String keyword;
 115 
 116         private DraftType() {
 117             keyword = this.name().toLowerCase(Locale.ROOT);
 118 
 119         }
 120 
 121         static DraftType forKeyword(String keyword) {
 122             return map.get(keyword);
 123         }
 124 
 125         static DraftType getDefault() {
 126             return defaultType;
 127         }
 128 
 129         static void setDefault(String keyword) {
 130             defaultType = Objects.requireNonNull(forKeyword(keyword));
 131         }
 132 
 133         String getKeyword() {
 134             return keyword;
 135         }
 136     }
 137 
 138     static boolean USE_UTF8 = false;
 139     private static boolean verbose;
 140 
 141     private CLDRConverter() {
 142        // no instantiation
 143     }
 144 
 145     @SuppressWarnings("AssignmentToForLoopParameter")
 146     public static void main(String[] args) throws Exception {
 147         if (args.length != 0) {
 148             String currentArg = null;
 149             try {
 150                 for (int i = 0; i < args.length; i++) {
 151                     currentArg = args[i];
 152                     switch (currentArg) {
 153                     case "-draft":
 154                         String draftDataType = args[++i];
 155                         try {
 156                             DraftType.setDefault(draftDataType);
 157                         } catch (NullPointerException e) {
 158                             severe("Error: incorrect draft value: %s%n", draftDataType);
 159                             System.exit(1);
 160                         }
 161                         info("Using the specified data type: %s%n", draftDataType);
 162                         break;
 163 
 164                     case "-base":
 165                         // base directory for input files
 166                         CLDR_BASE = args[++i];
 167                         if (!CLDR_BASE.endsWith("/")) {
 168                             CLDR_BASE += "/";
 169                         }
 170                         break;
 171 
 172                     case "-baselocales":
 173                         // base locales
 174                         setupBaseLocales(args[++i]);
 175                         break;
 176 
 177                     case "-basemodule":
 178                         // indicates java.base module resource generation
 179                         isBaseModule = true;
 180                         break;
 181 
 182                     case "-o":
 183                         // output directory
 184                         DESTINATION_DIR = args[++i];
 185                         break;
 186 
 187                     case "-utf8":
 188                         USE_UTF8 = true;
 189                         break;
 190 
 191                     case "-verbose":
 192                         verbose = true;
 193                         break;
 194 
 195                     case "-help":
 196                         usage();
 197                         System.exit(0);
 198                         break;
 199 
 200                     default:
 201                         throw new RuntimeException();
 202                     }
 203                 }
 204             } catch (RuntimeException e) {
 205                 severe("unknown or imcomplete arg(s): " + currentArg);
 206                 usage();
 207                 System.exit(1);
 208             }
 209         }
 210 
 211         // Set up path names
 212         LOCAL_LDML_DTD = CLDR_BASE + "/dtd/ldml.dtd";
 213         LOCAL_SPPL_LDML_DTD = CLDR_BASE + "/dtd/ldmlSupplemental.dtd";
 214         LOCAL_BCP47_LDML_DTD = CLDR_BASE + "/dtd/ldmlBCP47.dtd";
 215         SOURCE_FILE_DIR = CLDR_BASE + "/main";
 216         SPPL_SOURCE_FILE = CLDR_BASE + "/supplemental/supplementalData.xml";
 217         LIKELYSUBTAGS_SOURCE_FILE = CLDR_BASE + "/supplemental/likelySubtags.xml";
 218         NUMBERING_SOURCE_FILE = CLDR_BASE + "/supplemental/numberingSystems.xml";
 219         METAZONES_SOURCE_FILE = CLDR_BASE + "/supplemental/metaZones.xml";
 220         TIMEZONE_SOURCE_FILE = CLDR_BASE + "/bcp47/timezone.xml";
 221 
 222         if (BASE_LOCALES.isEmpty()) {
 223             setupBaseLocales("en-US");
 224         }
 225 
 226         bundleGenerator = new ResourceBundleGenerator();
 227 
 228         // Parse data independent of locales
 229         parseSupplemental();
 230         parseBCP47();
 231 
 232         List<Bundle> bundles = readBundleList();
 233         convertBundles(bundles);
 234         convertBundles(addedBundles);
 235     }
 236 
 237     private static void usage() {
 238         errout("Usage: java CLDRConverter [options]%n"
 239                 + "\t-help          output this usage message and exit%n"
 240                 + "\t-verbose       output information%n"
 241                 + "\t-draft [contributed | approved | provisional | unconfirmed]%n"
 242                 + "\t\t       draft level for using data (default: contributed)%n"
 243                 + "\t-base dir      base directory for CLDR input files%n"
 244                 + "\t-basemodule    generates bundles that go into java.base module%n"
 245                 + "\t-baselocales loc(,loc)*      locales that go into the base module%n"
 246                 + "\t-o dir         output directory (default: ./build/gensrc)%n"
 247                 + "\t-o dir         output directory (defaut: ./build/gensrc)%n"
 248                 + "\t-utf8          use UTF-8 rather than \\uxxxx (for debug)%n");
 249     }
 250 
 251     static void info(String fmt, Object... args) {
 252         if (verbose) {
 253             System.out.printf(fmt, args);
 254         }
 255     }
 256 
 257     static void info(String msg) {
 258         if (verbose) {
 259             System.out.println(msg);
 260         }
 261     }
 262 
 263     static void warning(String fmt, Object... args) {
 264         System.err.print("Warning: ");
 265         System.err.printf(fmt, args);
 266     }
 267 
 268     static void warning(String msg) {
 269         System.err.print("Warning: ");
 270         errout(msg);
 271     }
 272 
 273     static void severe(String fmt, Object... args) {
 274         System.err.print("Error: ");
 275         System.err.printf(fmt, args);
 276     }
 277 
 278     static void severe(String msg) {
 279         System.err.print("Error: ");
 280         errout(msg);
 281     }
 282 
 283     private static void errout(String msg) {
 284         if (msg.contains("%n")) {
 285             System.err.printf(msg);
 286         } else {
 287             System.err.println(msg);
 288         }
 289     }
 290 
 291     /**
 292      * Configure the parser to allow access to DTDs on the file system.
 293      */
 294     private static void enableFileAccess(SAXParser parser) throws SAXNotSupportedException {
 295         try {
 296             parser.setProperty("http://javax.xml.XMLConstants/property/accessExternalDTD", "file");
 297         } catch (SAXNotRecognizedException ignore) {
 298             // property requires >= JAXP 1.5
 299         }
 300     }
 301 
 302     private static List<Bundle> readBundleList() throws Exception {
 303         List<Bundle> retList = new ArrayList<>();
 304         Path path = FileSystems.getDefault().getPath(SOURCE_FILE_DIR);
 305         try (DirectoryStream<Path> dirStr = Files.newDirectoryStream(path)) {
 306             for (Path entry : dirStr) {
 307                 String fileName = entry.getFileName().toString();
 308                 if (fileName.endsWith(".xml")) {
 309                     String id = fileName.substring(0, fileName.indexOf('.'));
 310                     Locale cldrLoc = Locale.forLanguageTag(toLanguageTag(id));
 311                     StringBuilder sb = getCandLocales(cldrLoc);
 312                     if (sb.indexOf("root") == -1) {
 313                         sb.append("root");
 314                     }
 315                     Bundle b = new Bundle(id, sb.toString(), null, null);
 316                     // Insert the bundle for root at the top so that it will get
 317                     // processed first.
 318                     if ("root".equals(id)) {
 319                         retList.add(0, b);
 320                     } else {
 321                         retList.add(b);
 322                     }
 323                 }
 324             }
 325         }
 326         return retList;
 327     }
 328 
 329     private static final Map<String, Map<String, Object>> cldrBundles = new HashMap<>();
 330     // this list will contain additional bundles to be generated for Region dependent Data.
 331     private static List<Bundle> addedBundles = new ArrayList<>();
 332 
 333     private static Map<String, SortedSet<String>> metaInfo = new HashMap<>();
 334 
 335     static {
 336         // For generating information on supported locales.
 337         metaInfo.put("AvailableLocales", new TreeSet<>());
 338     }
 339 
 340 
 341     private static Set<String> calendarDataFields = Set.of("firstDayOfWeek", "minimalDaysInFirstWeek");
 342 
 343     static Map<String, Object> getCLDRBundle(String id) throws Exception {
 344         Map<String, Object> bundle = cldrBundles.get(id);
 345         if (bundle != null) {
 346             return bundle;
 347         }
 348         File file = new File(SOURCE_FILE_DIR + File.separator + id + ".xml");
 349         if (!file.exists()) {
 350             // Skip if the file doesn't exist.
 351             return Collections.emptyMap();
 352         }
 353         
 354         info("..... main directory .....");
 355         LDMLParseHandler handler = new LDMLParseHandler(id);
 356         parseLDMLFile(file, handler);
 357 
 358         bundle = handler.getData();
 359         cldrBundles.put(id, bundle);
 360         String country = getCountryCode(id);
 361         if (country != null) {
 362             bundle = handlerSuppl.getData(country);
 363             if (bundle != null) {
 364                 //merge two maps into one map
 365                 Map<String, Object> temp = cldrBundles.remove(id);
 366                 bundle.putAll(temp);
 367                 cldrBundles.put(id, bundle);
 368             }
 369         }
 370         return bundle;
 371     }
 372 
 373     // Parsers for data in "supplemental" directory
 374     //
 375     private static void parseSupplemental() throws Exception {
 376         // Parse SupplementalData file and store the information in the HashMap
 377         // Calendar information such as firstDay and minDay are stored in
 378         // supplementalData.xml as of CLDR1.4. Individual territory is listed
 379         // with its ISO 3166 country code while default is listed using UNM49
 380         // region and composition numerical code (001 for World.)
 381         //
 382         // SupplementalData file also provides the "parent" locales which
 383         // are othrwise not to be fallen back. Process them here as well.
 384         //
 385         handlerSuppl = new SupplementDataParseHandler();
 386         parseLDMLFile(new File(SPPL_SOURCE_FILE), handlerSuppl);
 387         Map<String, Object> parentData = handlerSuppl.getData("root");
 388         parentData.keySet().forEach(key -> {
 389                 parentLocalesMap.put(key, new TreeSet(
 390                     Arrays.asList(((String)parentData.get(key)).split(" "))));
 391             });
 392 
 393         // Parse numberingSystems to get digit zero character information.
 394         handlerNumbering = new NumberingSystemsParseHandler();
 395         parseLDMLFile(new File(NUMBERING_SOURCE_FILE), handlerNumbering);
 396 
 397         // Parse metaZones to create mappings between Olson tzids and CLDR meta zone names
 398         handlerMetaZones = new MetaZonesParseHandler();
 399         parseLDMLFile(new File(METAZONES_SOURCE_FILE), handlerMetaZones);
 400 
 401         // Parse likelySubtags
 402         handlerLikelySubtags = new LikelySubtagsParseHandler();
 403         parseLDMLFile(new File(LIKELYSUBTAGS_SOURCE_FILE), handlerLikelySubtags);
 404     }
 405     
 406     // Parsers for data in "bcp47" directory
 407     //
 408     private static void parseBCP47() throws Exception {
 409         // Parse timezone
 410         handlerTimeZone = new TimeZoneParseHandler();
 411         parseLDMLFile(new File(TIMEZONE_SOURCE_FILE), handlerTimeZone);
 412     }
 413     
 414     private static void parseLDMLFile(File srcfile, AbstractLDMLHandler handler) throws Exception {
 415         info("..... Parsing " + srcfile.getName() + " .....");
 416         SAXParserFactory pf = SAXParserFactory.newInstance();
 417         pf.setValidating(true);
 418         SAXParser parser = pf.newSAXParser();
 419         enableFileAccess(parser);
 420         parser.parse(srcfile, handler);       
 421     }
 422 
 423     /**
 424      * This method will check if a new region dependent Bundle needs to be
 425      * generated for this Locale id and targetMap. New Bundle will be generated
 426      * when Locale id has non empty script and country code and targetMap
 427      * contains region dependent data. This method will also remove region
 428      * dependent data from this targetMap after candidate locales check. E.g. It
 429      * will call genRegionDependentBundle() in case of az_Latn_AZ locale and
 430      * remove region dependent data from this targetMap so that az_Latn_AZ
 431      * bundle will not be created. For az_Cyrl_AZ, new Bundle will be generated
 432      * but region dependent data will not be removed from targetMap as its candidate
 433      * locales are [az_Cyrl_AZ, az_Cyrl, root], which does not include az_AZ for
 434      * fallback.
 435      *
 436      */
 437 
 438     private static void checkRegionDependentBundle(Map<String, Object> targetMap, String id) {
 439         if ((CLDRConverter.getScript(id) != "")
 440                 && (CLDRConverter.getCountryCode(id) != "")) {
 441             Map<String, Object> regionDepDataMap = targetMap
 442                     .keySet()
 443                     .stream()
 444                     .filter(calendarDataFields::contains)
 445                     .collect(Collectors.toMap(k -> k, targetMap::get));
 446             if (!regionDepDataMap.isEmpty()) {
 447                 Locale cldrLoc = new Locale(CLDRConverter.getLanguageCode(id),
 448                                             CLDRConverter.getCountryCode(id));
 449                 genRegionDependentBundle(regionDepDataMap, cldrLoc);
 450                 if (checkCandidateLocales(id, cldrLoc)) {
 451                     // Remove matchedKeys from this targetMap only if checkCandidateLocales() returns true.
 452                     regionDepDataMap.keySet().forEach(targetMap::remove);
 453                 }
 454             }
 455         }
 456     }
 457     /**
 458      * This method will generate a new Bundle for region dependent data,
 459      * minimalDaysInFirstWeek and firstDayOfWeek. Newly generated Bundle will be added
 460      * to addedBundles list.
 461      */
 462     private static void genRegionDependentBundle(Map<String, Object> targetMap, Locale cldrLoc) {
 463         String localeId = cldrLoc.toString();
 464         StringBuilder sb = getCandLocales(cldrLoc);
 465         if (sb.indexOf(localeId) == -1) {
 466             sb.append(localeId);
 467         }
 468         Bundle bundle = new Bundle(localeId, sb.toString(), null, null);
 469         cldrBundles.put(localeId, targetMap);
 470         addedBundles.add(bundle);
 471     }
 472 
 473     private static StringBuilder getCandLocales(Locale cldrLoc) {
 474         List<Locale> candList = getCandidateLocales(cldrLoc);
 475         StringBuilder sb = new StringBuilder();
 476         for (Locale loc : candList) {
 477             if (!loc.equals(Locale.ROOT)) {
 478                 sb.append(toLocaleName(loc.toLanguageTag()));
 479                 sb.append(",");
 480             }
 481         }
 482         return sb;
 483     }
 484 
 485     private static List<Locale> getCandidateLocales(Locale cldrLoc) {
 486         List<Locale> candList = new ArrayList<>();
 487         candList = applyParentLocales("", defCon.getCandidateLocales("",  cldrLoc));
 488         return candList;
 489     }
 490 
 491     /**
 492      * This method will return true, if for a given locale, its language and
 493      * country specific locale will exist in runtime lookup path. E.g. it will
 494      * return true for bs_Latn_BA.
 495      */
 496     private static boolean checkCandidateLocales(String id, Locale cldrLoc) {
 497         return(getCandidateLocales(Locale.forLanguageTag(id.replaceAll("_", "-")))
 498                 .contains(cldrLoc));
 499     }
 500 
 501     private static void convertBundles(List<Bundle> bundles) throws Exception {
 502         // parent locales map. The mappings are put in base metaInfo file
 503         // for now.
 504         if (isBaseModule) {
 505             metaInfo.putAll(parentLocalesMap);
 506         }
 507 
 508         for (Bundle bundle : bundles) {
 509             // Get the target map, which contains all the data that should be
 510             // visible for the bundle's locale
 511 
 512             Map<String, Object> targetMap = bundle.getTargetMap();
 513 
 514             // check if new region DependentBundle needs to be generated for this Locale.
 515             checkRegionDependentBundle(targetMap, bundle.getID());
 516             EnumSet<Bundle.Type> bundleTypes = bundle.getBundleTypes();
 517 
 518             if (bundle.isRoot()) {
 519                 // Add DateTimePatternChars because CLDR no longer supports localized patterns.
 520                 targetMap.put("DateTimePatternChars", "GyMdkHmsSEDFwWahKzZ");
 521             }
 522 
 523             // Now the map contains just the entries that need to be in the resources bundles.
 524             // Go ahead and generate them.
 525             if (bundleTypes.contains(Bundle.Type.LOCALENAMES)) {
 526                 Map<String, Object> localeNamesMap = extractLocaleNames(targetMap, bundle.getID());
 527                 if (!localeNamesMap.isEmpty() || bundle.isRoot()) {
 528                     bundleGenerator.generateBundle("util", "LocaleNames", bundle.getJavaID(), true, localeNamesMap, BundleType.OPEN);
 529                 }
 530             }
 531             if (bundleTypes.contains(Bundle.Type.CURRENCYNAMES)) {
 532                 Map<String, Object> currencyNamesMap = extractCurrencyNames(targetMap, bundle.getID(), bundle.getCurrencies());
 533                 if (!currencyNamesMap.isEmpty() || bundle.isRoot()) {
 534                     bundleGenerator.generateBundle("util", "CurrencyNames", bundle.getJavaID(), true, currencyNamesMap, BundleType.OPEN);
 535                 }
 536             }
 537             if (bundleTypes.contains(Bundle.Type.TIMEZONENAMES)) {
 538                 Map<String, Object> zoneNamesMap = extractZoneNames(targetMap, bundle.getID());
 539                 if (!zoneNamesMap.isEmpty() || bundle.isRoot()) {
 540                     bundleGenerator.generateBundle("util", "TimeZoneNames", bundle.getJavaID(), true, zoneNamesMap, BundleType.TIMEZONE);
 541                 }
 542             }
 543             if (bundleTypes.contains(Bundle.Type.CALENDARDATA)) {
 544                 Map<String, Object> calendarDataMap = extractCalendarData(targetMap, bundle.getID());
 545                 if (!calendarDataMap.isEmpty() || bundle.isRoot()) {
 546                     bundleGenerator.generateBundle("util", "CalendarData", bundle.getJavaID(), true, calendarDataMap, BundleType.PLAIN);
 547                 }
 548             }
 549             if (bundleTypes.contains(Bundle.Type.FORMATDATA)) {
 550                 Map<String, Object> formatDataMap = extractFormatData(targetMap, bundle.getID());
 551                 if (!formatDataMap.isEmpty() || bundle.isRoot()) {
 552                     bundleGenerator.generateBundle("text", "FormatData", bundle.getJavaID(), true, formatDataMap, BundleType.PLAIN);
 553                 }
 554             }
 555 
 556             // For AvailableLocales
 557             metaInfo.get("AvailableLocales").add(toLanguageTag(bundle.getID()));
 558             addLikelySubtags(metaInfo, "AvailableLocales", bundle.getID());
 559         }
 560         bundleGenerator.generateMetaInfo(metaInfo);
 561     }
 562 
 563     static final Map<String, String> aliases = new HashMap<>();
 564 
 565     /**
 566      * Translate the aliases into the real entries in the bundle map.
 567      */
 568     static void handleAliases(Map<String, Object> bundleMap) {
 569         Set bundleKeys = bundleMap.keySet();
 570         try {
 571             for (String key : aliases.keySet()) {
 572                 String targetKey = aliases.get(key);
 573                 if (bundleKeys.contains(targetKey)) {
 574                     bundleMap.putIfAbsent(key, bundleMap.get(targetKey));
 575                 }
 576             }
 577         } catch (Exception ex) {
 578             Logger.getLogger(CLDRConverter.class.getName()).log(Level.SEVERE, null, ex);
 579         }
 580     }
 581 
 582     /*
 583      * Returns the language portion of the given id.
 584      * If id is "root", "" is returned.
 585      */
 586     static String getLanguageCode(String id) {
 587         return "root".equals(id) ? "" : Locale.forLanguageTag(id.replaceAll("_", "-")).getLanguage();
 588     }
 589 
 590     /**
 591      * Examine if the id includes the country (territory) code. If it does, it returns
 592      * the country code.
 593      * Otherwise, it returns null. eg. when the id is "zh_Hans_SG", it return "SG".
 594      * It does NOT return UN M.49 code, e.g., '001', as those three digit numbers cannot
 595      * be translated into package names.
 596      */
 597     static String getCountryCode(String id) {
 598         String rgn = getRegionCode(id);
 599         return rgn.length() == 2 ? rgn: null;
 600     }
 601 
 602     /**
 603      * Examine if the id includes the region code. If it does, it returns
 604      * the region code.
 605      * Otherwise, it returns null. eg. when the id is "zh_Hans_SG", it return "SG".
 606      * It DOES return UN M.49 code, e.g., '001', as well as ISO 3166 two letter country codes.
 607      */
 608     static String getRegionCode(String id) {
 609         return Locale.forLanguageTag(id.replaceAll("_", "-")).getCountry();
 610     }
 611 
 612     /*
 613      * Returns the script portion of the given id.
 614      * If id is "root", "" is returned.
 615      */
 616     static String getScript(String id) {
 617         return "root".equals(id) ? "" : Locale.forLanguageTag(id.replaceAll("_", "-")).getScript();
 618     }
 619 
 620     private static class KeyComparator implements Comparator<String> {
 621         static KeyComparator INSTANCE = new KeyComparator();
 622 
 623         private KeyComparator() {
 624         }
 625 
 626         @Override
 627         public int compare(String o1, String o2) {
 628             int len1 = o1.length();
 629             int len2 = o2.length();
 630             if (!isDigit(o1.charAt(0)) && !isDigit(o2.charAt(0))) {
 631                 // Shorter string comes first unless either starts with a digit.
 632                 if (len1 < len2) {
 633                     return -1;
 634                 }
 635                 if (len1 > len2) {
 636                     return 1;
 637                 }
 638             }
 639             return o1.compareTo(o2);
 640         }
 641 
 642         private boolean isDigit(char c) {
 643             return c >= '0' && c <= '9';
 644         }
 645     }
 646 
 647     private static Map<String, Object> extractLocaleNames(Map<String, Object> map, String id) {
 648         Map<String, Object> localeNames = new TreeMap<>(KeyComparator.INSTANCE);
 649         for (String key : map.keySet()) {
 650             if (key.startsWith(LOCALE_NAME_PREFIX)) {
 651                 switch (key) {
 652                     case LOCALE_SEPARATOR:
 653                         localeNames.put("ListCompositionPattern", map.get(key));
 654                         break;
 655                     case LOCALE_KEYTYPE:
 656                         localeNames.put("ListKeyTypePattern", map.get(key));
 657                         break;
 658                     default:
 659                         localeNames.put(key.substring(LOCALE_NAME_PREFIX.length()), map.get(key));
 660                         break;
 661                 }
 662             }
 663         }
 664 
 665         if (id.equals("root")) {
 666             // Add display name pattern, which is not in CLDR
 667             localeNames.put("DisplayNamePattern", "{0,choice,0#|1#{1}|2#{1} ({2})}");
 668         }
 669 
 670         return localeNames;
 671     }
 672 
 673     @SuppressWarnings("AssignmentToForLoopParameter")
 674     private static Map<String, Object> extractCurrencyNames(Map<String, Object> map, String id, String names)
 675             throws Exception {
 676         Map<String, Object> currencyNames = new TreeMap<>(KeyComparator.INSTANCE);
 677         for (String key : map.keySet()) {
 678             if (key.startsWith(CURRENCY_NAME_PREFIX)) {
 679                 currencyNames.put(key.substring(CURRENCY_NAME_PREFIX.length()), map.get(key));
 680             } else if (key.startsWith(CURRENCY_SYMBOL_PREFIX)) {
 681                 currencyNames.put(key.substring(CURRENCY_SYMBOL_PREFIX.length()), map.get(key));
 682             }
 683         }
 684         return currencyNames;
 685     }
 686 
 687     private static Map<String, Object> extractZoneNames(Map<String, Object> map, String id) {
 688         Map<String, Object> names = new HashMap<>();
 689 
 690         // Copy over missing time zone ids from JRE for English locale
 691         if (id.equals("en")) {
 692             Map<String[], String> jreMetaMap = new HashMap<>();
 693             jreTimeZoneNames.stream().forEach(e -> {
 694                 String tzid = (String)e[0];
 695                 String[] data = (String[])e[1];
 696 
 697                 if (map.get(TIMEZONE_ID_PREFIX + tzid) == null &&
 698                     handlerMetaZones.get(tzid) == null ||
 699                     handlerMetaZones.get(tzid) != null &&
 700                     map.get(METAZONE_ID_PREFIX + handlerMetaZones.get(tzid)) == null) {
 701                     // First, check the CLDR meta key
 702                     Optional<Map.Entry<String, String>> cldrMeta =
 703                         handlerMetaZones.getData().entrySet().stream()
 704                             .filter(me ->
 705                                 Arrays.deepEquals(data,
 706                                     (String[])map.get(METAZONE_ID_PREFIX + me.getValue())))
 707                             .findAny();
 708                     if (cldrMeta.isPresent()) {
 709                         names.put(tzid, cldrMeta.get().getValue());
 710                     } else {
 711                         // check the JRE meta key, add if there is not.
 712                         Optional<Map.Entry<String[], String>> jreMeta =
 713                             jreMetaMap.entrySet().stream()
 714                                 .filter(jm -> Arrays.deepEquals(data, jm.getKey()))
 715                                 .findAny();
 716                         if (jreMeta.isPresent()) {
 717                             names.put(tzid, jreMeta.get().getValue());
 718                         } else {
 719                             String metaName = "JRE_" + tzid.replaceAll("[/-]", "_");
 720                             names.put(METAZONE_ID_PREFIX + metaName, data);
 721                             names.put(tzid, metaName);
 722                             jreMetaMap.put(data, metaName);
 723                         }
 724                     }
 725                 }
 726             });
 727         }
 728 
 729         for (String tzid : handlerMetaZones.keySet()) {
 730             String tzKey = TIMEZONE_ID_PREFIX + tzid;
 731             Object data = map.get(tzKey);
 732             if (data instanceof String[]) {
 733                 names.put(tzid, data);
 734             } else {
 735                 String meta = handlerMetaZones.get(tzid);
 736                 if (meta != null) {
 737                     String metaKey = METAZONE_ID_PREFIX + meta;
 738                     data = map.get(metaKey);
 739                     if (data instanceof String[]) {
 740                         // Keep the metazone prefix here.
 741                         names.put(metaKey, data);
 742                         names.put(tzid, meta);
 743                     }
 744                 }
 745             }
 746         }
 747         return names;
 748     }
 749 
 750     private static Map<String, Object> extractCalendarData(Map<String, Object> map, String id) {
 751         Map<String, Object> calendarData = new LinkedHashMap<>();
 752         copyIfPresent(map, "firstDayOfWeek", calendarData);
 753         copyIfPresent(map, "minimalDaysInFirstWeek", calendarData);
 754         return calendarData;
 755     }
 756 
 757     static final String[] FORMAT_DATA_ELEMENTS = {
 758         "MonthNames",
 759         "standalone.MonthNames",
 760         "MonthAbbreviations",
 761         "standalone.MonthAbbreviations",
 762         "MonthNarrows",
 763         "standalone.MonthNarrows",
 764         "DayNames",
 765         "standalone.DayNames",
 766         "DayAbbreviations",
 767         "standalone.DayAbbreviations",
 768         "DayNarrows",
 769         "standalone.DayNarrows",
 770         "QuarterNames",
 771         "standalone.QuarterNames",
 772         "QuarterAbbreviations",
 773         "standalone.QuarterAbbreviations",
 774         "QuarterNarrows",
 775         "standalone.QuarterNarrows",
 776         "AmPmMarkers",
 777         "narrow.AmPmMarkers",
 778         "abbreviated.AmPmMarkers",
 779         "long.Eras",
 780         "Eras",
 781         "narrow.Eras",
 782         "field.era",
 783         "field.year",
 784         "field.month",
 785         "field.week",
 786         "field.weekday",
 787         "field.dayperiod",
 788         "field.hour",
 789         "timezone.hourFormat",
 790         "timezone.gmtFormat",
 791         "field.minute",
 792         "field.second",
 793         "field.zone",
 794         "TimePatterns",
 795         "DatePatterns",
 796         "DateTimePatterns",
 797         "DateTimePatternChars"
 798     };
 799 
 800     private static Map<String, Object> extractFormatData(Map<String, Object> map, String id) {
 801         Map<String, Object> formatData = new LinkedHashMap<>();
 802         for (CalendarType calendarType : CalendarType.values()) {
 803             if (calendarType == CalendarType.GENERIC) {
 804                 continue;
 805             }
 806             String prefix = calendarType.keyElementName();
 807             for (String element : FORMAT_DATA_ELEMENTS) {
 808                 String key = prefix + element;
 809                 copyIfPresent(map, "java.time." + key, formatData);
 810                 copyIfPresent(map, key, formatData);
 811             }
 812         }
 813 
 814         for (String key : map.keySet()) {
 815         // Copy available calendar names
 816             if (key.startsWith(CLDRConverter.LOCALE_TYPE_PREFIX_CA)) {
 817                 String type = key.substring(CLDRConverter.LOCALE_TYPE_PREFIX_CA.length());
 818                 for (CalendarType calendarType : CalendarType.values()) {
 819                     if (calendarType == CalendarType.GENERIC) {
 820                         continue;
 821                     }
 822                     if (type.equals(calendarType.lname())) {
 823                         Object value = map.get(key);
 824                         String dataKey = key.replace(LOCALE_TYPE_PREFIX_CA,
 825                                 CALENDAR_NAME_PREFIX);
 826                         formatData.put(dataKey, value);
 827                         String ukey = CALENDAR_NAME_PREFIX + calendarType.uname();
 828                         if (!dataKey.equals(ukey)) {
 829                             formatData.put(ukey, value);
 830                         }
 831                     }
 832                 }
 833             }
 834         }
 835 
 836         copyIfPresent(map, "DefaultNumberingSystem", formatData);
 837 
 838         @SuppressWarnings("unchecked")
 839         List<String> numberingScripts = (List<String>) map.remove("numberingScripts");
 840         if (numberingScripts != null) {
 841             for (String script : numberingScripts) {
 842                 copyIfPresent(map, script + "." + "NumberElements", formatData);
 843             }
 844         } else {
 845             copyIfPresent(map, "NumberElements", formatData);
 846         }
 847         copyIfPresent(map, "NumberPatterns", formatData);
 848 
 849         // put extra number elements for available scripts into formatData, if it is "root"
 850         if (id.equals("root")) {
 851             handlerNumbering.keySet().stream()
 852                 .filter(k -> !numberingScripts.contains(k))
 853                 .forEach(k -> {
 854                     String[] ne = (String[])map.get("latn.NumberElements");
 855                     String[] neNew = Arrays.copyOf(ne, ne.length);
 856                     neNew[4] = handlerNumbering.get(k).substring(0, 1);
 857                     formatData.put(k + ".NumberElements", neNew);
 858                 });
 859         }
 860         return formatData;
 861     }
 862 
 863     private static void copyIfPresent(Map<String, Object> src, String key, Map<String, Object> dest) {
 864         Object value = src.get(key);
 865         if (value != null) {
 866             dest.put(key, value);
 867         }
 868     }
 869 
 870     // --- code below here is adapted from java.util.Properties ---
 871     private static final String specialSaveCharsJava = "\"";
 872     private static final String specialSaveCharsProperties = "=: \t\r\n\f#!";
 873 
 874     /*
 875      * Converts unicodes to encoded \uxxxx
 876      * and writes out any of the characters in specialSaveChars
 877      * with a preceding slash
 878      */
 879     static String saveConvert(String theString, boolean useJava) {
 880         if (theString == null) {
 881             return "";
 882         }
 883 
 884         String specialSaveChars;
 885         if (useJava) {
 886             specialSaveChars = specialSaveCharsJava;
 887         } else {
 888             specialSaveChars = specialSaveCharsProperties;
 889         }
 890         boolean escapeSpace = false;
 891 
 892         int len = theString.length();
 893         StringBuilder outBuffer = new StringBuilder(len * 2);
 894         Formatter formatter = new Formatter(outBuffer, Locale.ROOT);
 895 
 896         for (int x = 0; x < len; x++) {
 897             char aChar = theString.charAt(x);
 898             switch (aChar) {
 899             case ' ':
 900                 if (x == 0 || escapeSpace) {
 901                     outBuffer.append('\\');
 902                 }
 903                 outBuffer.append(' ');
 904                 break;
 905             case '\\':
 906                 outBuffer.append('\\');
 907                 outBuffer.append('\\');
 908                 break;
 909             case '\t':
 910                 outBuffer.append('\\');
 911                 outBuffer.append('t');
 912                 break;
 913             case '\n':
 914                 outBuffer.append('\\');
 915                 outBuffer.append('n');
 916                 break;
 917             case '\r':
 918                 outBuffer.append('\\');
 919                 outBuffer.append('r');
 920                 break;
 921             case '\f':
 922                 outBuffer.append('\\');
 923                 outBuffer.append('f');
 924                 break;
 925             default:
 926                 if (aChar < 0x0020 || (!USE_UTF8 && aChar > 0x007e)) {
 927                     formatter.format("\\u%04x", (int)aChar);
 928                 } else {
 929                     if (specialSaveChars.indexOf(aChar) != -1) {
 930                         outBuffer.append('\\');
 931                     }
 932                     outBuffer.append(aChar);
 933                 }
 934             }
 935         }
 936         return outBuffer.toString();
 937     }
 938 
 939     private static String toLanguageTag(String locName) {
 940         if (locName.indexOf('_') == -1) {
 941             return locName;
 942         }
 943         String tag = locName.replaceAll("_", "-");
 944         Locale loc = Locale.forLanguageTag(tag);
 945         return loc.toLanguageTag();
 946     }
 947 
 948     private static void addLikelySubtags(Map<String, SortedSet<String>> metaInfo, String category, String id) {
 949         String likelySubtag = handlerLikelySubtags.get(id);
 950         if (likelySubtag != null) {
 951             // Remove Script for now
 952             metaInfo.get(category).add(toLanguageTag(likelySubtag).replaceFirst("-[A-Z][a-z]{3}", ""));
 953         }
 954     }
 955 
 956     private static String toLocaleName(String tag) {
 957         if (tag.indexOf('-') == -1) {
 958             return tag;
 959         }
 960         return tag.replaceAll("-", "_");
 961     }
 962 
 963     private static void setupBaseLocales(String localeList) {
 964         Arrays.stream(localeList.split(","))
 965             .map(Locale::forLanguageTag)
 966             .map(l -> Control.getControl(Control.FORMAT_DEFAULT)
 967                              .getCandidateLocales("", l))
 968             .forEach(BASE_LOCALES::addAll);
 969     }
 970 
 971     // applying parent locale rules to the passed candidates list
 972     // This has to match with the one in sun.util.cldr.CLDRLocaleProviderAdapter
 973     private static Map<Locale, Locale> childToParentLocaleMap = null;
 974     private static List<Locale> applyParentLocales(String baseName, List<Locale> candidates) {
 975         if (Objects.isNull(childToParentLocaleMap)) {
 976             childToParentLocaleMap = new HashMap<>();
 977             parentLocalesMap.keySet().forEach(key -> {
 978                 String parent = key.substring(PARENT_LOCALE_PREFIX.length()).replaceAll("_", "-");
 979                 parentLocalesMap.get(key).stream().forEach(child -> {
 980                     childToParentLocaleMap.put(Locale.forLanguageTag(child),
 981                         "root".equals(parent) ? Locale.ROOT : Locale.forLanguageTag(parent));
 982                 });
 983             });
 984         }
 985 
 986         // check irregular parents
 987         for (int i = 0; i < candidates.size(); i++) {
 988             Locale l = candidates.get(i);
 989             Locale p = childToParentLocaleMap.get(l);
 990             if (!l.equals(Locale.ROOT) &&
 991                 Objects.nonNull(p) &&
 992                 !candidates.get(i+1).equals(p)) {
 993                 List<Locale> applied = candidates.subList(0, i+1);
 994                 applied.addAll(applyParentLocales(baseName, defCon.getCandidateLocales(baseName, p)));
 995                 return applied;
 996             }
 997         }
 998 
 999         return candidates;
1000     }
1001 }