1 /* 2 * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. Oracle designates this 8 * particular file as subject to the "Classpath" exception as provided 9 * by Oracle in the LICENSE file that accompanied this code. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 */ 25 26 package build.tools.cldrconverter; 27 28 import static build.tools.cldrconverter.Bundle.jreTimeZoneNames; 29 import build.tools.cldrconverter.BundleGenerator.BundleType; 30 import java.io.File; 31 import java.io.IOException; 32 import java.io.UncheckedIOException; 33 import java.nio.file.*; 34 import java.text.MessageFormat; 35 import java.time.*; 36 import java.util.*; 37 import java.util.ResourceBundle.Control; 38 import java.util.logging.Level; 39 import java.util.logging.Logger; 40 import java.util.stream.Collectors; 41 import java.util.stream.IntStream; 42 import java.util.stream.Stream; 43 import javax.xml.parsers.SAXParser; 44 import javax.xml.parsers.SAXParserFactory; 45 import org.xml.sax.SAXNotRecognizedException; 46 import org.xml.sax.SAXNotSupportedException; 47 48 49 /** 50 * Converts locale data from "Locale Data Markup Language" format to 51 * JRE resource bundle format. LDML is the format used by the Common 52 * Locale Data Repository maintained by the Unicode Consortium. 53 */ 54 public class CLDRConverter { 55 56 static final String LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldml.dtd"; 57 static final String SPPL_LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldmlSupplemental.dtd"; 58 static final String BCP47_LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldmlBCP47.dtd"; 59 60 61 private static String CLDR_BASE; 62 static String LOCAL_LDML_DTD; 63 static String LOCAL_SPPL_LDML_DTD; 64 static String LOCAL_BCP47_LDML_DTD; 65 private static String SOURCE_FILE_DIR; 66 private static String SPPL_SOURCE_FILE; 67 private static String SPPL_META_SOURCE_FILE; 68 private static String NUMBERING_SOURCE_FILE; 69 private static String METAZONES_SOURCE_FILE; 70 private static String LIKELYSUBTAGS_SOURCE_FILE; 71 private static String TIMEZONE_SOURCE_FILE; 72 private static String WINZONES_SOURCE_FILE; 73 static String DESTINATION_DIR = "build/gensrc"; 74 75 static final String LOCALE_NAME_PREFIX = "locale.displayname."; 76 static final String LOCALE_SEPARATOR = LOCALE_NAME_PREFIX + "separator"; 77 static final String LOCALE_KEYTYPE = LOCALE_NAME_PREFIX + "keytype"; 78 static final String LOCALE_KEY_PREFIX = LOCALE_NAME_PREFIX + "key."; 79 static final String LOCALE_TYPE_PREFIX = LOCALE_NAME_PREFIX + "type."; 80 static final String LOCALE_TYPE_PREFIX_CA = LOCALE_TYPE_PREFIX + "ca."; 81 static final String CURRENCY_SYMBOL_PREFIX = "currency.symbol."; 82 static final String CURRENCY_NAME_PREFIX = "currency.displayname."; 83 static final String CALENDAR_NAME_PREFIX = "calendarname."; 84 static final String CALENDAR_FIRSTDAY_PREFIX = "firstDay."; 85 static final String CALENDAR_MINDAYS_PREFIX = "minDays."; 86 static final String TIMEZONE_ID_PREFIX = "timezone.id."; 87 static final String EXEMPLAR_CITY_PREFIX = "timezone.excity."; 88 static final String ZONE_NAME_PREFIX = "timezone.displayname."; 89 static final String METAZONE_ID_PREFIX = "metazone.id."; 90 static final String PARENT_LOCALE_PREFIX = "parentLocale."; 91 static final String[] EMPTY_ZONE = {"", "", "", "", "", ""}; 92 93 private static SupplementDataParseHandler handlerSuppl; 94 private static LikelySubtagsParseHandler handlerLikelySubtags; 95 private static WinZonesParseHandler handlerWinZones; 96 static SupplementalMetadataParseHandler handlerSupplMeta; 97 static NumberingSystemsParseHandler handlerNumbering; 98 static MetaZonesParseHandler handlerMetaZones; 99 static TimeZoneParseHandler handlerTimeZone; 100 private static BundleGenerator bundleGenerator; 101 102 // java.base module related 103 static boolean isBaseModule = false; 104 static final Set<Locale> BASE_LOCALES = new HashSet<>(); 105 106 // "parentLocales" map 107 private static final Map<String, SortedSet<String>> parentLocalesMap = new HashMap<>(); 108 private static final ResourceBundle.Control defCon = 109 ResourceBundle.Control.getControl(ResourceBundle.Control.FORMAT_DEFAULT); 110 111 private static final String[] AVAILABLE_TZIDS = TimeZone.getAvailableIDs(); 112 private static String zoneNameTempFile; 113 private static String tzDataDir; 114 private static final Map<String, String> canonicalTZMap = new HashMap<>(); 115 116 static enum DraftType { 117 UNCONFIRMED, 118 PROVISIONAL, 119 CONTRIBUTED, 120 APPROVED; 121 122 private static final Map<String, DraftType> map = new HashMap<>(); 123 static { 124 for (DraftType dt : values()) { 125 map.put(dt.getKeyword(), dt); 126 } 127 } 128 static private DraftType defaultType = CONTRIBUTED; 129 130 private final String keyword; 131 132 private DraftType() { 133 keyword = this.name().toLowerCase(Locale.ROOT); 134 135 } 136 137 static DraftType forKeyword(String keyword) { 138 return map.get(keyword); 139 } 140 141 static DraftType getDefault() { 142 return defaultType; 143 } 144 145 static void setDefault(String keyword) { 146 defaultType = Objects.requireNonNull(forKeyword(keyword)); 147 } 148 149 String getKeyword() { 150 return keyword; 151 } 152 } 153 154 static boolean USE_UTF8 = false; 155 private static boolean verbose; 156 157 private CLDRConverter() { 158 // no instantiation 159 } 160 161 @SuppressWarnings("AssignmentToForLoopParameter") 162 public static void main(String[] args) throws Exception { 163 if (args.length != 0) { 164 String currentArg = null; 165 try { 166 for (int i = 0; i < args.length; i++) { 167 currentArg = args[i]; 168 switch (currentArg) { 169 case "-draft": 170 String draftDataType = args[++i]; 171 try { 172 DraftType.setDefault(draftDataType); 173 } catch (NullPointerException e) { 174 severe("Error: incorrect draft value: %s%n", draftDataType); 175 System.exit(1); 176 } 177 info("Using the specified data type: %s%n", draftDataType); 178 break; 179 180 case "-base": 181 // base directory for input files 182 CLDR_BASE = args[++i]; 183 if (!CLDR_BASE.endsWith("/")) { 184 CLDR_BASE += "/"; 185 } 186 break; 187 188 case "-baselocales": 189 // base locales 190 setupBaseLocales(args[++i]); 191 break; 192 193 case "-basemodule": 194 // indicates java.base module resource generation 195 isBaseModule = true; 196 break; 197 198 case "-o": 199 // output directory 200 DESTINATION_DIR = args[++i]; 201 break; 202 203 case "-utf8": 204 USE_UTF8 = true; 205 break; 206 207 case "-verbose": 208 verbose = true; 209 break; 210 211 case "-zntempfile": 212 zoneNameTempFile = args[++i]; 213 break; 214 215 case "-tzdatadir": 216 tzDataDir = args[++i]; 217 break; 218 219 case "-help": 220 usage(); 221 System.exit(0); 222 break; 223 224 default: 225 throw new RuntimeException(); 226 } 227 } 228 } catch (RuntimeException e) { 229 severe("unknown or imcomplete arg(s): " + currentArg); 230 usage(); 231 System.exit(1); 232 } 233 } 234 235 // Set up path names 236 LOCAL_LDML_DTD = CLDR_BASE + "/dtd/ldml.dtd"; 237 LOCAL_SPPL_LDML_DTD = CLDR_BASE + "/dtd/ldmlSupplemental.dtd"; 238 LOCAL_BCP47_LDML_DTD = CLDR_BASE + "/dtd/ldmlBCP47.dtd"; 239 SOURCE_FILE_DIR = CLDR_BASE + "/main"; 240 SPPL_SOURCE_FILE = CLDR_BASE + "/supplemental/supplementalData.xml"; 241 LIKELYSUBTAGS_SOURCE_FILE = CLDR_BASE + "/supplemental/likelySubtags.xml"; 242 NUMBERING_SOURCE_FILE = CLDR_BASE + "/supplemental/numberingSystems.xml"; 243 METAZONES_SOURCE_FILE = CLDR_BASE + "/supplemental/metaZones.xml"; 244 TIMEZONE_SOURCE_FILE = CLDR_BASE + "/bcp47/timezone.xml"; 245 SPPL_META_SOURCE_FILE = CLDR_BASE + "/supplemental/supplementalMetadata.xml"; 246 WINZONES_SOURCE_FILE = CLDR_BASE + "/supplemental/windowsZones.xml"; 247 248 if (BASE_LOCALES.isEmpty()) { 249 setupBaseLocales("en-US"); 250 } 251 252 bundleGenerator = new ResourceBundleGenerator(); 253 254 // Parse data independent of locales 255 parseSupplemental(); 256 parseBCP47(); 257 258 List<Bundle> bundles = readBundleList(); 259 convertBundles(bundles); 260 261 if (isBaseModule) { 262 // Generate java.time.format.ZoneName.java 263 generateZoneName(); 264 265 // Generate Windows tzmappings 266 generateWindowsTZMappings(); 267 } 268 } 269 270 private static void usage() { 271 errout("Usage: java CLDRConverter [options]%n" 272 + "\t-help output this usage message and exit%n" 273 + "\t-verbose output information%n" 274 + "\t-draft [contributed | approved | provisional | unconfirmed]%n" 275 + "\t\t draft level for using data (default: contributed)%n" 276 + "\t-base dir base directory for CLDR input files%n" 277 + "\t-basemodule generates bundles that go into java.base module%n" 278 + "\t-baselocales loc(,loc)* locales that go into the base module%n" 279 + "\t-o dir output directory (default: ./build/gensrc)%n" 280 + "\t-zntempfile template file for java.time.format.ZoneName.java%n" 281 + "\t-tzdatadir tzdata directory for java.time.format.ZoneName.java%n" 282 + "\t-utf8 use UTF-8 rather than \\uxxxx (for debug)%n"); 283 } 284 285 static void info(String fmt, Object... args) { 286 if (verbose) { 287 System.out.printf(fmt, args); 288 } 289 } 290 291 static void info(String msg) { 292 if (verbose) { 293 System.out.println(msg); 294 } 295 } 296 297 static void warning(String fmt, Object... args) { 298 System.err.print("Warning: "); 299 System.err.printf(fmt, args); 300 } 301 302 static void warning(String msg) { 303 System.err.print("Warning: "); 304 errout(msg); 305 } 306 307 static void severe(String fmt, Object... args) { 308 System.err.print("Error: "); 309 System.err.printf(fmt, args); 310 } 311 312 static void severe(String msg) { 313 System.err.print("Error: "); 314 errout(msg); 315 } 316 317 private static void errout(String msg) { 318 if (msg.contains("%n")) { 319 System.err.printf(msg); 320 } else { 321 System.err.println(msg); 322 } 323 } 324 325 /** 326 * Configure the parser to allow access to DTDs on the file system. 327 */ 328 private static void enableFileAccess(SAXParser parser) throws SAXNotSupportedException { 329 try { 330 parser.setProperty("http://javax.xml.XMLConstants/property/accessExternalDTD", "file"); 331 } catch (SAXNotRecognizedException ignore) { 332 // property requires >= JAXP 1.5 333 } 334 } 335 336 private static List<Bundle> readBundleList() throws Exception { 337 List<Bundle> retList = new ArrayList<>(); 338 Path path = FileSystems.getDefault().getPath(SOURCE_FILE_DIR); 339 try (DirectoryStream<Path> dirStr = Files.newDirectoryStream(path)) { 340 for (Path entry : dirStr) { 341 String fileName = entry.getFileName().toString(); 342 if (fileName.endsWith(".xml")) { 343 String id = fileName.substring(0, fileName.indexOf('.')); 344 Locale cldrLoc = Locale.forLanguageTag(toLanguageTag(id)); 345 StringBuilder sb = getCandLocales(cldrLoc); 346 if (sb.indexOf("root") == -1) { 347 sb.append("root"); 348 } 349 Bundle b = new Bundle(id, sb.toString(), null, null); 350 // Insert the bundle for root at the top so that it will get 351 // processed first. 352 if ("root".equals(id)) { 353 retList.add(0, b); 354 } else { 355 retList.add(b); 356 } 357 } 358 } 359 } 360 return retList; 361 } 362 363 private static final Map<String, Map<String, Object>> cldrBundles = new HashMap<>(); 364 365 private static Map<String, SortedSet<String>> metaInfo = new HashMap<>(); 366 367 static { 368 // For generating information on supported locales. 369 metaInfo.put("AvailableLocales", new TreeSet<>()); 370 } 371 372 static Map<String, Object> getCLDRBundle(String id) throws Exception { 373 Map<String, Object> bundle = cldrBundles.get(id); 374 if (bundle != null) { 375 return bundle; 376 } 377 File file = new File(SOURCE_FILE_DIR + File.separator + id + ".xml"); 378 if (!file.exists()) { 379 // Skip if the file doesn't exist. 380 return Collections.emptyMap(); 381 } 382 383 info("..... main directory ....."); 384 LDMLParseHandler handler = new LDMLParseHandler(id); 385 parseLDMLFile(file, handler); 386 387 bundle = handler.getData(); 388 cldrBundles.put(id, bundle); 389 390 if (id.equals("root")) { 391 // Calendar data (firstDayOfWeek & minDaysInFirstWeek) 392 bundle = handlerSuppl.getData("root"); 393 if (bundle != null) { 394 //merge two maps into one map 395 Map<String, Object> temp = cldrBundles.remove(id); 396 bundle.putAll(temp); 397 cldrBundles.put(id, bundle); 398 } 399 } 400 return bundle; 401 } 402 403 // Parsers for data in "supplemental" directory 404 // 405 private static void parseSupplemental() throws Exception { 406 // Parse SupplementalData file and store the information in the HashMap 407 // Calendar information such as firstDay and minDay are stored in 408 // supplementalData.xml as of CLDR1.4. Individual territory is listed 409 // with its ISO 3166 country code while default is listed using UNM49 410 // region and composition numerical code (001 for World.) 411 // 412 // SupplementalData file also provides the "parent" locales which 413 // are othrwise not to be fallen back. Process them here as well. 414 // 415 handlerSuppl = new SupplementDataParseHandler(); 416 parseLDMLFile(new File(SPPL_SOURCE_FILE), handlerSuppl); 417 Map<String, Object> parentData = handlerSuppl.getData("root"); 418 parentData.keySet().stream() 419 .filter(key -> key.startsWith(PARENT_LOCALE_PREFIX)) 420 .forEach(key -> { 421 parentLocalesMap.put(key, new TreeSet( 422 Arrays.asList(((String)parentData.get(key)).split(" ")))); 423 }); 424 425 // Parse numberingSystems to get digit zero character information. 426 handlerNumbering = new NumberingSystemsParseHandler(); 427 parseLDMLFile(new File(NUMBERING_SOURCE_FILE), handlerNumbering); 428 429 // Parse metaZones to create mappings between Olson tzids and CLDR meta zone names 430 handlerMetaZones = new MetaZonesParseHandler(); 431 parseLDMLFile(new File(METAZONES_SOURCE_FILE), handlerMetaZones); 432 433 // Parse likelySubtags 434 handlerLikelySubtags = new LikelySubtagsParseHandler(); 435 parseLDMLFile(new File(LIKELYSUBTAGS_SOURCE_FILE), handlerLikelySubtags); 436 437 // Parse supplementalMetadata 438 // Currently interested in deprecated time zone ids and language aliases. 439 handlerSupplMeta = new SupplementalMetadataParseHandler(); 440 parseLDMLFile(new File(SPPL_META_SOURCE_FILE), handlerSupplMeta); 441 442 // Parse windowsZones 443 handlerWinZones = new WinZonesParseHandler(); 444 parseLDMLFile(new File(WINZONES_SOURCE_FILE), handlerWinZones); 445 } 446 447 // Parsers for data in "bcp47" directory 448 // 449 private static void parseBCP47() throws Exception { 450 // Parse timezone 451 handlerTimeZone = new TimeZoneParseHandler(); 452 parseLDMLFile(new File(TIMEZONE_SOURCE_FILE), handlerTimeZone); 453 454 // canonical tz name map 455 // alias -> primary 456 handlerTimeZone.getData().forEach((k, v) -> { 457 String[] ids = ((String)v).split("\\s"); 458 for (int i = 1; i < ids.length; i++) { 459 canonicalTZMap.put(ids[i], ids[0]); 460 } 461 }); 462 } 463 464 private static void parseLDMLFile(File srcfile, AbstractLDMLHandler handler) throws Exception { 465 info("..... Parsing " + srcfile.getName() + " ....."); 466 SAXParserFactory pf = SAXParserFactory.newInstance(); 467 pf.setValidating(true); 468 SAXParser parser = pf.newSAXParser(); 469 enableFileAccess(parser); 470 parser.parse(srcfile, handler); 471 } 472 473 private static StringBuilder getCandLocales(Locale cldrLoc) { 474 List<Locale> candList = getCandidateLocales(cldrLoc); 475 StringBuilder sb = new StringBuilder(); 476 for (Locale loc : candList) { 477 if (!loc.equals(Locale.ROOT)) { 478 sb.append(toLocaleName(loc.toLanguageTag())); 479 sb.append(","); 480 } 481 } 482 return sb; 483 } 484 485 private static List<Locale> getCandidateLocales(Locale cldrLoc) { 486 List<Locale> candList = new ArrayList<>(); 487 candList = applyParentLocales("", defCon.getCandidateLocales("", cldrLoc)); 488 return candList; 489 } 490 491 private static void convertBundles(List<Bundle> bundles) throws Exception { 492 // parent locales map. The mappings are put in base metaInfo file 493 // for now. 494 if (isBaseModule) { 495 metaInfo.putAll(parentLocalesMap); 496 } 497 498 for (Bundle bundle : bundles) { 499 // Get the target map, which contains all the data that should be 500 // visible for the bundle's locale 501 502 Map<String, Object> targetMap = bundle.getTargetMap(); 503 504 EnumSet<Bundle.Type> bundleTypes = bundle.getBundleTypes(); 505 506 if (bundle.isRoot()) { 507 // Add DateTimePatternChars because CLDR no longer supports localized patterns. 508 targetMap.put("DateTimePatternChars", "GyMdkHmsSEDFwWahKzZ"); 509 } 510 511 // Now the map contains just the entries that need to be in the resources bundles. 512 // Go ahead and generate them. 513 if (bundleTypes.contains(Bundle.Type.LOCALENAMES)) { 514 Map<String, Object> localeNamesMap = extractLocaleNames(targetMap, bundle.getID()); 515 if (!localeNamesMap.isEmpty() || bundle.isRoot()) { 516 bundleGenerator.generateBundle("util", "LocaleNames", bundle.getJavaID(), true, localeNamesMap, BundleType.OPEN); 517 } 518 } 519 if (bundleTypes.contains(Bundle.Type.CURRENCYNAMES)) { 520 Map<String, Object> currencyNamesMap = extractCurrencyNames(targetMap, bundle.getID(), bundle.getCurrencies()); 521 if (!currencyNamesMap.isEmpty() || bundle.isRoot()) { 522 bundleGenerator.generateBundle("util", "CurrencyNames", bundle.getJavaID(), true, currencyNamesMap, BundleType.OPEN); 523 } 524 } 525 if (bundleTypes.contains(Bundle.Type.TIMEZONENAMES)) { 526 Map<String, Object> zoneNamesMap = extractZoneNames(targetMap, bundle.getID()); 527 if (!zoneNamesMap.isEmpty() || bundle.isRoot()) { 528 bundleGenerator.generateBundle("util", "TimeZoneNames", bundle.getJavaID(), true, zoneNamesMap, BundleType.TIMEZONE); 529 } 530 } 531 if (bundleTypes.contains(Bundle.Type.CALENDARDATA)) { 532 Map<String, Object> calendarDataMap = extractCalendarData(targetMap, bundle.getID()); 533 if (!calendarDataMap.isEmpty() || bundle.isRoot()) { 534 bundleGenerator.generateBundle("util", "CalendarData", bundle.getJavaID(), true, calendarDataMap, BundleType.PLAIN); 535 } 536 } 537 if (bundleTypes.contains(Bundle.Type.FORMATDATA)) { 538 Map<String, Object> formatDataMap = extractFormatData(targetMap, bundle.getID()); 539 if (!formatDataMap.isEmpty() || bundle.isRoot()) { 540 bundleGenerator.generateBundle("text", "FormatData", bundle.getJavaID(), true, formatDataMap, BundleType.PLAIN); 541 } 542 } 543 544 // For AvailableLocales 545 metaInfo.get("AvailableLocales").add(toLanguageTag(bundle.getID())); 546 addLikelySubtags(metaInfo, "AvailableLocales", bundle.getID()); 547 } 548 bundleGenerator.generateMetaInfo(metaInfo); 549 } 550 551 static final Map<String, String> aliases = new HashMap<>(); 552 553 /** 554 * Translate the aliases into the real entries in the bundle map. 555 */ 556 static void handleAliases(Map<String, Object> bundleMap) { 557 Set bundleKeys = bundleMap.keySet(); 558 try { 559 for (String key : aliases.keySet()) { 560 String targetKey = aliases.get(key); 561 if (bundleKeys.contains(targetKey)) { 562 bundleMap.putIfAbsent(key, bundleMap.get(targetKey)); 563 } 564 } 565 } catch (Exception ex) { 566 Logger.getLogger(CLDRConverter.class.getName()).log(Level.SEVERE, null, ex); 567 } 568 } 569 570 /* 571 * Returns the language portion of the given id. 572 * If id is "root", "" is returned. 573 */ 574 static String getLanguageCode(String id) { 575 return "root".equals(id) ? "" : Locale.forLanguageTag(id.replaceAll("_", "-")).getLanguage(); 576 } 577 578 /** 579 * Examine if the id includes the country (territory) code. If it does, it returns 580 * the country code. 581 * Otherwise, it returns null. eg. when the id is "zh_Hans_SG", it return "SG". 582 * It does NOT return UN M.49 code, e.g., '001', as those three digit numbers cannot 583 * be translated into package names. 584 */ 585 static String getCountryCode(String id) { 586 String rgn = getRegionCode(id); 587 return rgn.length() == 2 ? rgn: null; 588 } 589 590 /** 591 * Examine if the id includes the region code. If it does, it returns 592 * the region code. 593 * Otherwise, it returns null. eg. when the id is "zh_Hans_SG", it return "SG". 594 * It DOES return UN M.49 code, e.g., '001', as well as ISO 3166 two letter country codes. 595 */ 596 static String getRegionCode(String id) { 597 return Locale.forLanguageTag(id.replaceAll("_", "-")).getCountry(); 598 } 599 600 private static class KeyComparator implements Comparator<String> { 601 static KeyComparator INSTANCE = new KeyComparator(); 602 603 private KeyComparator() { 604 } 605 606 @Override 607 public int compare(String o1, String o2) { 608 int len1 = o1.length(); 609 int len2 = o2.length(); 610 if (!isDigit(o1.charAt(0)) && !isDigit(o2.charAt(0))) { 611 // Shorter string comes first unless either starts with a digit. 612 if (len1 < len2) { 613 return -1; 614 } 615 if (len1 > len2) { 616 return 1; 617 } 618 } 619 return o1.compareTo(o2); 620 } 621 622 private boolean isDigit(char c) { 623 return c >= '0' && c <= '9'; 624 } 625 } 626 627 private static Map<String, Object> extractLocaleNames(Map<String, Object> map, String id) { 628 Map<String, Object> localeNames = new TreeMap<>(KeyComparator.INSTANCE); 629 for (String key : map.keySet()) { 630 if (key.startsWith(LOCALE_NAME_PREFIX)) { 631 switch (key) { 632 case LOCALE_SEPARATOR: 633 localeNames.put("ListCompositionPattern", map.get(key)); 634 break; 635 case LOCALE_KEYTYPE: 636 localeNames.put("ListKeyTypePattern", map.get(key)); 637 break; 638 default: 639 localeNames.put(key.substring(LOCALE_NAME_PREFIX.length()), map.get(key)); 640 break; 641 } 642 } 643 } 644 645 if (id.equals("root")) { 646 // Add display name pattern, which is not in CLDR 647 localeNames.put("DisplayNamePattern", "{0,choice,0#|1#{1}|2#{1} ({2})}"); 648 } 649 650 return localeNames; 651 } 652 653 @SuppressWarnings("AssignmentToForLoopParameter") 654 private static Map<String, Object> extractCurrencyNames(Map<String, Object> map, String id, String names) 655 throws Exception { 656 Map<String, Object> currencyNames = new TreeMap<>(KeyComparator.INSTANCE); 657 for (String key : map.keySet()) { 658 if (key.startsWith(CURRENCY_NAME_PREFIX)) { 659 currencyNames.put(key.substring(CURRENCY_NAME_PREFIX.length()), map.get(key)); 660 } else if (key.startsWith(CURRENCY_SYMBOL_PREFIX)) { 661 currencyNames.put(key.substring(CURRENCY_SYMBOL_PREFIX.length()), map.get(key)); 662 } 663 } 664 return currencyNames; 665 } 666 667 private static Map<String, Object> extractZoneNames(Map<String, Object> map, String id) { 668 Map<String, Object> names = new HashMap<>(); 669 670 // Copy over missing time zone ids from JRE for English locale 671 if (id.equals("en")) { 672 Map<String[], String> jreMetaMap = new HashMap<>(); 673 jreTimeZoneNames.stream().forEach(e -> { 674 String tzid = (String)e[0]; 675 String[] data = (String[])e[1]; 676 677 if (map.get(TIMEZONE_ID_PREFIX + tzid) == null && 678 handlerMetaZones.get(tzid) == null || 679 handlerMetaZones.get(tzid) != null && 680 map.get(METAZONE_ID_PREFIX + handlerMetaZones.get(tzid)) == null) { 681 682 // First, check the alias 683 String canonID = canonicalTZMap.get(tzid); 684 if (canonID != null && !tzid.equals(canonID)) { 685 Object value = map.get(TIMEZONE_ID_PREFIX + canonID); 686 if (value != null) { 687 names.put(tzid, value); 688 return; 689 } else { 690 String meta = handlerMetaZones.get(canonID); 691 if (meta != null) { 692 value = map.get(METAZONE_ID_PREFIX + meta); 693 if (value != null) { 694 names.put(tzid, meta); 695 return; 696 } 697 } 698 } 699 } 700 701 // Check the CLDR meta key 702 Optional<Map.Entry<String, String>> cldrMeta = 703 handlerMetaZones.getData().entrySet().stream() 704 .filter(me -> 705 Arrays.deepEquals(data, 706 (String[])map.get(METAZONE_ID_PREFIX + me.getValue()))) 707 .findAny(); 708 cldrMeta.ifPresentOrElse(meta -> names.put(tzid, meta.getValue()), () -> { 709 // Check the JRE meta key, add if there is not. 710 Optional<Map.Entry<String[], String>> jreMeta = 711 jreMetaMap.entrySet().stream() 712 .filter(jm -> Arrays.deepEquals(data, jm.getKey())) 713 .findAny(); 714 jreMeta.ifPresentOrElse(meta -> names.put(tzid, meta.getValue()), () -> { 715 String metaName = "JRE_" + tzid.replaceAll("[/-]", "_"); 716 names.put(METAZONE_ID_PREFIX + metaName, data); 717 names.put(tzid, metaName); 718 }); 719 }); 720 } 721 }); 722 } 723 724 Arrays.stream(AVAILABLE_TZIDS).forEach(tzid -> { 725 // If the tzid is deprecated, get the data for the replacement id 726 String tzKey = Optional.ofNullable((String)handlerSupplMeta.get(tzid)) 727 .orElse(tzid); 728 Object data = map.get(TIMEZONE_ID_PREFIX + tzKey); 729 730 if (data instanceof String[]) { 731 names.put(tzid, data); 732 } else { 733 String meta = handlerMetaZones.get(tzKey); 734 if (meta != null) { 735 String metaKey = METAZONE_ID_PREFIX + meta; 736 data = map.get(metaKey); 737 if (data instanceof String[]) { 738 // Keep the metazone prefix here. 739 names.put(metaKey, data); 740 names.put(tzid, meta); 741 } 742 } 743 } 744 }); 745 746 // exemplar cities. 747 Map<String, Object> exCities = map.entrySet().stream() 748 .filter(e -> e.getKey().startsWith(CLDRConverter.EXEMPLAR_CITY_PREFIX)) 749 .collect(Collectors 750 .toMap(Map.Entry::getKey, Map.Entry::getValue)); 751 names.putAll(exCities); 752 753 if (!id.equals("en") && 754 !names.isEmpty()) { 755 // CLDR does not have UTC entry, so add it here. 756 names.put("UTC", EMPTY_ZONE); 757 758 // no metazone zones 759 Arrays.asList(handlerMetaZones.get(MetaZonesParseHandler.NO_METAZONE_KEY) 760 .split("\\s")).stream() 761 .forEach(tz -> { 762 names.put(tz, EMPTY_ZONE); 763 }); 764 } 765 766 return names; 767 } 768 769 /** 770 * Extracts the language independent calendar data. Each of the two keys, 771 * "firstDayOfWeek" and "minimalDaysInFirstWeek" has a string value consists of 772 * one or multiple occurrences of: 773 * i: rg1 rg2 ... rgn; 774 * where "i" is the data for the following regions (delimited by a space) after 775 * ":", and ends with a ";". 776 */ 777 private static Map<String, Object> extractCalendarData(Map<String, Object> map, String id) { 778 Map<String, Object> calendarData = new LinkedHashMap<>(); 779 if (id.equals("root")) { 780 calendarData.put("firstDayOfWeek", 781 IntStream.range(1, 8) 782 .mapToObj(String::valueOf) 783 .filter(d -> map.keySet().contains(CALENDAR_FIRSTDAY_PREFIX + d)) 784 .map(d -> d + ": " + map.get(CALENDAR_FIRSTDAY_PREFIX + d)) 785 .collect(Collectors.joining(";"))); 786 calendarData.put("minimalDaysInFirstWeek", 787 IntStream.range(0, 7) 788 .mapToObj(String::valueOf) 789 .filter(d -> map.keySet().contains(CALENDAR_MINDAYS_PREFIX + d)) 790 .map(d -> d + ": " + map.get(CALENDAR_MINDAYS_PREFIX + d)) 791 .collect(Collectors.joining(";"))); 792 } 793 return calendarData; 794 } 795 796 static final String[] FORMAT_DATA_ELEMENTS = { 797 "MonthNames", 798 "standalone.MonthNames", 799 "MonthAbbreviations", 800 "standalone.MonthAbbreviations", 801 "MonthNarrows", 802 "standalone.MonthNarrows", 803 "DayNames", 804 "standalone.DayNames", 805 "DayAbbreviations", 806 "standalone.DayAbbreviations", 807 "DayNarrows", 808 "standalone.DayNarrows", 809 "QuarterNames", 810 "standalone.QuarterNames", 811 "QuarterAbbreviations", 812 "standalone.QuarterAbbreviations", 813 "QuarterNarrows", 814 "standalone.QuarterNarrows", 815 "AmPmMarkers", 816 "narrow.AmPmMarkers", 817 "abbreviated.AmPmMarkers", 818 "long.Eras", 819 "Eras", 820 "narrow.Eras", 821 "field.era", 822 "field.year", 823 "field.month", 824 "field.week", 825 "field.weekday", 826 "field.dayperiod", 827 "field.hour", 828 "timezone.hourFormat", 829 "timezone.gmtFormat", 830 "timezone.gmtZeroFormat", 831 "timezone.regionFormat", 832 "timezone.regionFormat.daylight", 833 "timezone.regionFormat.standard", 834 "field.minute", 835 "field.second", 836 "field.zone", 837 "TimePatterns", 838 "DatePatterns", 839 "DateTimePatterns", 840 "DateTimePatternChars" 841 }; 842 843 private static Map<String, Object> extractFormatData(Map<String, Object> map, String id) { 844 Map<String, Object> formatData = new LinkedHashMap<>(); 845 for (CalendarType calendarType : CalendarType.values()) { 846 if (calendarType == CalendarType.GENERIC) { 847 continue; 848 } 849 String prefix = calendarType.keyElementName(); 850 for (String element : FORMAT_DATA_ELEMENTS) { 851 String key = prefix + element; 852 copyIfPresent(map, "java.time." + key, formatData); 853 copyIfPresent(map, key, formatData); 854 } 855 } 856 857 for (String key : map.keySet()) { 858 // Copy available calendar names 859 if (key.startsWith(CLDRConverter.LOCALE_TYPE_PREFIX_CA)) { 860 String type = key.substring(CLDRConverter.LOCALE_TYPE_PREFIX_CA.length()); 861 for (CalendarType calendarType : CalendarType.values()) { 862 if (calendarType == CalendarType.GENERIC) { 863 continue; 864 } 865 if (type.equals(calendarType.lname())) { 866 Object value = map.get(key); 867 String dataKey = key.replace(LOCALE_TYPE_PREFIX_CA, 868 CALENDAR_NAME_PREFIX); 869 formatData.put(dataKey, value); 870 String ukey = CALENDAR_NAME_PREFIX + calendarType.uname(); 871 if (!dataKey.equals(ukey)) { 872 formatData.put(ukey, value); 873 } 874 } 875 } 876 } 877 } 878 879 copyIfPresent(map, "DefaultNumberingSystem", formatData); 880 881 @SuppressWarnings("unchecked") 882 List<String> numberingScripts = (List<String>) map.remove("numberingScripts"); 883 if (numberingScripts != null) { 884 for (String script : numberingScripts) { 885 copyIfPresent(map, script + "." + "NumberElements", formatData); 886 } 887 } else { 888 copyIfPresent(map, "NumberElements", formatData); 889 } 890 copyIfPresent(map, "NumberPatterns", formatData); 891 892 // put extra number elements for available scripts into formatData, if it is "root" 893 if (id.equals("root")) { 894 handlerNumbering.keySet().stream() 895 .filter(k -> !numberingScripts.contains(k)) 896 .forEach(k -> { 897 String[] ne = (String[])map.get("latn.NumberElements"); 898 String[] neNew = Arrays.copyOf(ne, ne.length); 899 neNew[4] = handlerNumbering.get(k).substring(0, 1); 900 formatData.put(k + ".NumberElements", neNew); 901 }); 902 } 903 return formatData; 904 } 905 906 private static void copyIfPresent(Map<String, Object> src, String key, Map<String, Object> dest) { 907 Object value = src.get(key); 908 if (value != null) { 909 dest.put(key, value); 910 } 911 } 912 913 // --- code below here is adapted from java.util.Properties --- 914 private static final String specialSaveCharsJava = "\""; 915 private static final String specialSaveCharsProperties = "=: \t\r\n\f#!"; 916 917 /* 918 * Converts unicodes to encoded \uxxxx 919 * and writes out any of the characters in specialSaveChars 920 * with a preceding slash 921 */ 922 static String saveConvert(String theString, boolean useJava) { 923 if (theString == null) { 924 return ""; 925 } 926 927 String specialSaveChars; 928 if (useJava) { 929 specialSaveChars = specialSaveCharsJava; 930 } else { 931 specialSaveChars = specialSaveCharsProperties; 932 } 933 boolean escapeSpace = false; 934 935 int len = theString.length(); 936 StringBuilder outBuffer = new StringBuilder(len * 2); 937 Formatter formatter = new Formatter(outBuffer, Locale.ROOT); 938 939 for (int x = 0; x < len; x++) { 940 char aChar = theString.charAt(x); 941 switch (aChar) { 942 case ' ': 943 if (x == 0 || escapeSpace) { 944 outBuffer.append('\\'); 945 } 946 outBuffer.append(' '); 947 break; 948 case '\\': 949 outBuffer.append('\\'); 950 outBuffer.append('\\'); 951 break; 952 case '\t': 953 outBuffer.append('\\'); 954 outBuffer.append('t'); 955 break; 956 case '\n': 957 outBuffer.append('\\'); 958 outBuffer.append('n'); 959 break; 960 case '\r': 961 outBuffer.append('\\'); 962 outBuffer.append('r'); 963 break; 964 case '\f': 965 outBuffer.append('\\'); 966 outBuffer.append('f'); 967 break; 968 default: 969 if (aChar < 0x0020 || (!USE_UTF8 && aChar > 0x007e)) { 970 formatter.format("\\u%04x", (int)aChar); 971 } else { 972 if (specialSaveChars.indexOf(aChar) != -1) { 973 outBuffer.append('\\'); 974 } 975 outBuffer.append(aChar); 976 } 977 } 978 } 979 return outBuffer.toString(); 980 } 981 982 private static String toLanguageTag(String locName) { 983 if (locName.indexOf('_') == -1) { 984 return locName; 985 } 986 String tag = locName.replaceAll("_", "-"); 987 Locale loc = Locale.forLanguageTag(tag); 988 return loc.toLanguageTag(); 989 } 990 991 private static void addLikelySubtags(Map<String, SortedSet<String>> metaInfo, String category, String id) { 992 String likelySubtag = handlerLikelySubtags.get(id); 993 if (likelySubtag != null) { 994 // Remove Script for now 995 metaInfo.get(category).add(toLanguageTag(likelySubtag).replaceFirst("-[A-Z][a-z]{3}", "")); 996 } 997 } 998 999 private static String toLocaleName(String tag) { 1000 if (tag.indexOf('-') == -1) { 1001 return tag; 1002 } 1003 return tag.replaceAll("-", "_"); 1004 } 1005 1006 private static void setupBaseLocales(String localeList) { 1007 Arrays.stream(localeList.split(",")) 1008 .map(Locale::forLanguageTag) 1009 .map(l -> Control.getControl(Control.FORMAT_DEFAULT) 1010 .getCandidateLocales("", l)) 1011 .forEach(BASE_LOCALES::addAll); 1012 } 1013 1014 // applying parent locale rules to the passed candidates list 1015 // This has to match with the one in sun.util.cldr.CLDRLocaleProviderAdapter 1016 private static Map<Locale, Locale> childToParentLocaleMap = null; 1017 private static List<Locale> applyParentLocales(String baseName, List<Locale> candidates) { 1018 if (Objects.isNull(childToParentLocaleMap)) { 1019 childToParentLocaleMap = new HashMap<>(); 1020 parentLocalesMap.keySet().forEach(key -> { 1021 String parent = key.substring(PARENT_LOCALE_PREFIX.length()).replaceAll("_", "-"); 1022 parentLocalesMap.get(key).stream().forEach(child -> { 1023 childToParentLocaleMap.put(Locale.forLanguageTag(child), 1024 "root".equals(parent) ? Locale.ROOT : Locale.forLanguageTag(parent)); 1025 }); 1026 }); 1027 } 1028 1029 // check irregular parents 1030 for (int i = 0; i < candidates.size(); i++) { 1031 Locale l = candidates.get(i); 1032 Locale p = childToParentLocaleMap.get(l); 1033 if (!l.equals(Locale.ROOT) && 1034 Objects.nonNull(p) && 1035 !candidates.get(i+1).equals(p)) { 1036 List<Locale> applied = candidates.subList(0, i+1); 1037 applied.addAll(applyParentLocales(baseName, defCon.getCandidateLocales(baseName, p))); 1038 return applied; 1039 } 1040 } 1041 1042 return candidates; 1043 } 1044 1045 private static void generateZoneName() throws Exception { 1046 Files.createDirectories(Paths.get(DESTINATION_DIR, "java", "time", "format")); 1047 Files.write(Paths.get(DESTINATION_DIR, "java", "time", "format", "ZoneName.java"), 1048 Files.lines(Paths.get(zoneNameTempFile)) 1049 .flatMap(l -> { 1050 if (l.equals("%%%%ZIDMAP%%%%")) { 1051 return zidMapEntry(); 1052 } else if (l.equals("%%%%MZONEMAP%%%%")) { 1053 return handlerMetaZones.mzoneMapEntry(); 1054 } else if (l.equals("%%%%DEPRECATED%%%%")) { 1055 return handlerSupplMeta.deprecatedMap(); 1056 } else if (l.equals("%%%%TZDATALINK%%%%")) { 1057 return tzDataLinkEntry(); 1058 } else { 1059 return Stream.of(l); 1060 } 1061 }) 1062 .collect(Collectors.toList()), 1063 StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); 1064 } 1065 1066 private static Stream<String> zidMapEntry() { 1067 return ZoneId.getAvailableZoneIds().stream() 1068 .map(id -> { 1069 String canonId = canonicalTZMap.getOrDefault(id, id); 1070 String meta = handlerMetaZones.get(canonId); 1071 String zone001 = handlerMetaZones.zidMap().get(meta); 1072 return zone001 == null ? "" : 1073 String.format(" \"%s\", \"%s\", \"%s\",", 1074 id, meta, zone001); 1075 }) 1076 .filter(s -> !s.isEmpty()) 1077 .sorted(); 1078 } 1079 1080 private static Stream<String> tzDataLinkEntry() { 1081 try { 1082 return Files.walk(Paths.get(tzDataDir), 1) 1083 .filter(p -> !Files.isDirectory(p)) 1084 .flatMap(CLDRConverter::extractLinks) 1085 .sorted(); 1086 } catch (IOException e) { 1087 throw new UncheckedIOException(e); 1088 } 1089 } 1090 1091 private static Stream<String> extractLinks(Path tzFile) { 1092 try { 1093 return Files.lines(tzFile) 1094 .filter(l -> l.startsWith("Link")) 1095 .map(l -> l.replaceFirst("^Link[\\s]+(\\S+)\\s+(\\S+).*", 1096 " \"$2\", \"$1\",")); 1097 } catch (IOException e) { 1098 throw new UncheckedIOException(e); 1099 } 1100 } 1101 1102 // Generate tzmappings for Windows. The format is: 1103 // 1104 // (Windows Zone Name):(REGION):(Java TZID) 1105 // 1106 // where: 1107 // Windows Zone Name: arbitrary time zone name string used in Windows 1108 // REGION: ISO3166 or UN M.49 code 1109 // Java TZID: Java's time zone ID 1110 // 1111 // Note: the entries are alphabetically soreted, *except* the "world" region 1112 // code, i.e., "001". It should be the last entry for the same windows time 1113 // zone name entries. (cf. TimeZone_md.c) 1114 private static void generateWindowsTZMappings() throws Exception { 1115 Files.createDirectories(Paths.get(DESTINATION_DIR, "windows", "conf")); 1116 Files.write(Paths.get(DESTINATION_DIR, "windows", "conf", "tzmappings"), 1117 handlerWinZones.keySet().stream() 1118 .map(k -> k + ":" + handlerWinZones.get(k) + ":") 1119 .sorted(new Comparator<String>() { 1120 public int compare(String t1, String t2) { 1121 String[] s1 = t1.split(":"); 1122 String[] s2 = t2.split(":"); 1123 if (s1[0].equals(s2[0])) { 1124 if (s1[1].equals("001")) { 1125 return 1; 1126 } else if (s2[1].equals("001")) { 1127 return -1; 1128 } else { 1129 return s1[1].compareTo(s2[1]); 1130 } 1131 } else { 1132 return s1[0].compareTo(s2[0]); 1133 } 1134 } 1135 }) 1136 .collect(Collectors.toList()), 1137 StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); 1138 } 1139 }