1 /* 2 * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. Oracle designates this 8 * particular file as subject to the "Classpath" exception as provided 9 * by Oracle in the LICENSE file that accompanied this code. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 */ 25 26 package build.tools.cldrconverter; 27 28 import static build.tools.cldrconverter.Bundle.jreTimeZoneNames; 29 import build.tools.cldrconverter.BundleGenerator.BundleType; 30 import java.io.File; 31 import java.io.IOException; 32 import java.io.UncheckedIOException; 33 import java.nio.file.*; 34 import java.time.*; 35 import java.util.*; 36 import java.util.ResourceBundle.Control; 37 import java.util.logging.Level; 38 import java.util.logging.Logger; 39 import java.util.stream.Collectors; 40 import java.util.stream.IntStream; 41 import java.util.stream.Stream; 42 import javax.xml.parsers.SAXParser; 43 import javax.xml.parsers.SAXParserFactory; 44 import org.xml.sax.SAXNotRecognizedException; 45 import org.xml.sax.SAXNotSupportedException; 46 47 48 /** 49 * Converts locale data from "Locale Data Markup Language" format to 50 * JRE resource bundle format. LDML is the format used by the Common 51 * Locale Data Repository maintained by the Unicode Consortium. 52 */ 53 public class CLDRConverter { 54 55 static final String LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldml.dtd"; 56 static final String SPPL_LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldmlSupplemental.dtd"; 57 static final String BCP47_LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldmlBCP47.dtd"; 58 59 60 private static String CLDR_BASE; 61 static String LOCAL_LDML_DTD; 62 static String LOCAL_SPPL_LDML_DTD; 63 static String LOCAL_BCP47_LDML_DTD; 64 private static String SOURCE_FILE_DIR; 65 private static String SPPL_SOURCE_FILE; 66 private static String SPPL_META_SOURCE_FILE; 67 private static String NUMBERING_SOURCE_FILE; 68 private static String METAZONES_SOURCE_FILE; 69 private static String LIKELYSUBTAGS_SOURCE_FILE; 70 private static String TIMEZONE_SOURCE_FILE; 71 static String DESTINATION_DIR = "build/gensrc"; 72 73 static final String LOCALE_NAME_PREFIX = "locale.displayname."; 74 static final String LOCALE_SEPARATOR = LOCALE_NAME_PREFIX + "separator"; 75 static final String LOCALE_KEYTYPE = LOCALE_NAME_PREFIX + "keytype"; 76 static final String LOCALE_KEY_PREFIX = LOCALE_NAME_PREFIX + "key."; 77 static final String LOCALE_TYPE_PREFIX = LOCALE_NAME_PREFIX + "type."; 78 static final String LOCALE_TYPE_PREFIX_CA = LOCALE_TYPE_PREFIX + "ca."; 79 static final String CURRENCY_SYMBOL_PREFIX = "currency.symbol."; 80 static final String CURRENCY_NAME_PREFIX = "currency.displayname."; 81 static final String CALENDAR_NAME_PREFIX = "calendarname."; 82 static final String CALENDAR_FIRSTDAY_PREFIX = "firstDay."; 83 static final String CALENDAR_MINDAYS_PREFIX = "minDays."; 84 static final String TIMEZONE_ID_PREFIX = "timezone.id."; 85 static final String ZONE_NAME_PREFIX = "timezone.displayname."; 86 static final String METAZONE_ID_PREFIX = "metazone.id."; 87 static final String PARENT_LOCALE_PREFIX = "parentLocale."; 88 89 private static SupplementDataParseHandler handlerSuppl; 90 private static SupplementalMetadataParseHandler handlerSupplMeta; 91 private static LikelySubtagsParseHandler handlerLikelySubtags; 92 static NumberingSystemsParseHandler handlerNumbering; 93 static MetaZonesParseHandler handlerMetaZones; 94 static TimeZoneParseHandler handlerTimeZone; 95 private static BundleGenerator bundleGenerator; 96 97 // java.base module related 98 static boolean isBaseModule = false; 99 static final Set<Locale> BASE_LOCALES = new HashSet<>(); 100 101 // "parentLocales" map 102 private static final Map<String, SortedSet<String>> parentLocalesMap = new HashMap<>(); 103 private static final ResourceBundle.Control defCon = 104 ResourceBundle.Control.getControl(ResourceBundle.Control.FORMAT_DEFAULT); 105 106 private static final String[] AVAILABLE_TZIDS = TimeZone.getAvailableIDs(); 107 private static String zoneNameTempFile; 108 private static String tzDataDir; 109 110 static enum DraftType { 111 UNCONFIRMED, 112 PROVISIONAL, 113 CONTRIBUTED, 114 APPROVED; 115 116 private static final Map<String, DraftType> map = new HashMap<>(); 117 static { 118 for (DraftType dt : values()) { 119 map.put(dt.getKeyword(), dt); 120 } 121 } 122 static private DraftType defaultType = CONTRIBUTED; 123 124 private final String keyword; 125 126 private DraftType() { 127 keyword = this.name().toLowerCase(Locale.ROOT); 128 129 } 130 131 static DraftType forKeyword(String keyword) { 132 return map.get(keyword); 133 } 134 135 static DraftType getDefault() { 136 return defaultType; 137 } 138 139 static void setDefault(String keyword) { 140 defaultType = Objects.requireNonNull(forKeyword(keyword)); 141 } 142 143 String getKeyword() { 144 return keyword; 145 } 146 } 147 148 static boolean USE_UTF8 = false; 149 private static boolean verbose; 150 151 private CLDRConverter() { 152 // no instantiation 153 } 154 155 @SuppressWarnings("AssignmentToForLoopParameter") 156 public static void main(String[] args) throws Exception { 157 if (args.length != 0) { 158 String currentArg = null; 159 try { 160 for (int i = 0; i < args.length; i++) { 161 currentArg = args[i]; 162 switch (currentArg) { 163 case "-draft": 164 String draftDataType = args[++i]; 165 try { 166 DraftType.setDefault(draftDataType); 167 } catch (NullPointerException e) { 168 severe("Error: incorrect draft value: %s%n", draftDataType); 169 System.exit(1); 170 } 171 info("Using the specified data type: %s%n", draftDataType); 172 break; 173 174 case "-base": 175 // base directory for input files 176 CLDR_BASE = args[++i]; 177 if (!CLDR_BASE.endsWith("/")) { 178 CLDR_BASE += "/"; 179 } 180 break; 181 182 case "-baselocales": 183 // base locales 184 setupBaseLocales(args[++i]); 185 break; 186 187 case "-basemodule": 188 // indicates java.base module resource generation 189 isBaseModule = true; 190 break; 191 192 case "-o": 193 // output directory 194 DESTINATION_DIR = args[++i]; 195 break; 196 197 case "-utf8": 198 USE_UTF8 = true; 199 break; 200 201 case "-verbose": 202 verbose = true; 203 break; 204 205 case "-zntempfile": 206 zoneNameTempFile = args[++i]; 207 break; 208 209 case "-tzdatadir": 210 tzDataDir = args[++i]; 211 break; 212 213 case "-help": 214 usage(); 215 System.exit(0); 216 break; 217 218 default: 219 throw new RuntimeException(); 220 } 221 } 222 } catch (RuntimeException e) { 223 severe("unknown or imcomplete arg(s): " + currentArg); 224 usage(); 225 System.exit(1); 226 } 227 } 228 229 // Set up path names 230 LOCAL_LDML_DTD = CLDR_BASE + "/dtd/ldml.dtd"; 231 LOCAL_SPPL_LDML_DTD = CLDR_BASE + "/dtd/ldmlSupplemental.dtd"; 232 LOCAL_BCP47_LDML_DTD = CLDR_BASE + "/dtd/ldmlBCP47.dtd"; 233 SOURCE_FILE_DIR = CLDR_BASE + "/main"; 234 SPPL_SOURCE_FILE = CLDR_BASE + "/supplemental/supplementalData.xml"; 235 LIKELYSUBTAGS_SOURCE_FILE = CLDR_BASE + "/supplemental/likelySubtags.xml"; 236 NUMBERING_SOURCE_FILE = CLDR_BASE + "/supplemental/numberingSystems.xml"; 237 METAZONES_SOURCE_FILE = CLDR_BASE + "/supplemental/metaZones.xml"; 238 TIMEZONE_SOURCE_FILE = CLDR_BASE + "/bcp47/timezone.xml"; 239 SPPL_META_SOURCE_FILE = CLDR_BASE + "/supplemental/supplementalMetadata.xml"; 240 241 if (BASE_LOCALES.isEmpty()) { 242 setupBaseLocales("en-US"); 243 } 244 245 bundleGenerator = new ResourceBundleGenerator(); 246 247 // Parse data independent of locales 248 parseSupplemental(); 249 parseBCP47(); 250 251 List<Bundle> bundles = readBundleList(); 252 convertBundles(bundles); 253 254 // Generate java.time.format.ZoneName.java 255 if (isBaseModule) { 256 generateZoneName(); 257 } 258 } 259 260 private static void usage() { 261 errout("Usage: java CLDRConverter [options]%n" 262 + "\t-help output this usage message and exit%n" 263 + "\t-verbose output information%n" 264 + "\t-draft [contributed | approved | provisional | unconfirmed]%n" 265 + "\t\t draft level for using data (default: contributed)%n" 266 + "\t-base dir base directory for CLDR input files%n" 267 + "\t-basemodule generates bundles that go into java.base module%n" 268 + "\t-baselocales loc(,loc)* locales that go into the base module%n" 269 + "\t-o dir output directory (default: ./build/gensrc)%n" 270 + "\t-zntempfile template file for java.time.format.ZoneName.java%n" 271 + "\t-tzdatadir tzdata directory for java.time.format.ZoneName.java%n" 272 + "\t-utf8 use UTF-8 rather than \\uxxxx (for debug)%n"); 273 } 274 275 static void info(String fmt, Object... args) { 276 if (verbose) { 277 System.out.printf(fmt, args); 278 } 279 } 280 281 static void info(String msg) { 282 if (verbose) { 283 System.out.println(msg); 284 } 285 } 286 287 static void warning(String fmt, Object... args) { 288 System.err.print("Warning: "); 289 System.err.printf(fmt, args); 290 } 291 292 static void warning(String msg) { 293 System.err.print("Warning: "); 294 errout(msg); 295 } 296 297 static void severe(String fmt, Object... args) { 298 System.err.print("Error: "); 299 System.err.printf(fmt, args); 300 } 301 302 static void severe(String msg) { 303 System.err.print("Error: "); 304 errout(msg); 305 } 306 307 private static void errout(String msg) { 308 if (msg.contains("%n")) { 309 System.err.printf(msg); 310 } else { 311 System.err.println(msg); 312 } 313 } 314 315 /** 316 * Configure the parser to allow access to DTDs on the file system. 317 */ 318 private static void enableFileAccess(SAXParser parser) throws SAXNotSupportedException { 319 try { 320 parser.setProperty("http://javax.xml.XMLConstants/property/accessExternalDTD", "file"); 321 } catch (SAXNotRecognizedException ignore) { 322 // property requires >= JAXP 1.5 323 } 324 } 325 326 private static List<Bundle> readBundleList() throws Exception { 327 List<Bundle> retList = new ArrayList<>(); 328 Path path = FileSystems.getDefault().getPath(SOURCE_FILE_DIR); 329 try (DirectoryStream<Path> dirStr = Files.newDirectoryStream(path)) { 330 for (Path entry : dirStr) { 331 String fileName = entry.getFileName().toString(); 332 if (fileName.endsWith(".xml")) { 333 String id = fileName.substring(0, fileName.indexOf('.')); 334 Locale cldrLoc = Locale.forLanguageTag(toLanguageTag(id)); 335 StringBuilder sb = getCandLocales(cldrLoc); 336 if (sb.indexOf("root") == -1) { 337 sb.append("root"); 338 } 339 Bundle b = new Bundle(id, sb.toString(), null, null); 340 // Insert the bundle for root at the top so that it will get 341 // processed first. 342 if ("root".equals(id)) { 343 retList.add(0, b); 344 } else { 345 retList.add(b); 346 } 347 } 348 } 349 } 350 return retList; 351 } 352 353 private static final Map<String, Map<String, Object>> cldrBundles = new HashMap<>(); 354 355 private static Map<String, SortedSet<String>> metaInfo = new HashMap<>(); 356 357 static { 358 // For generating information on supported locales. 359 metaInfo.put("AvailableLocales", new TreeSet<>()); 360 } 361 362 static Map<String, Object> getCLDRBundle(String id) throws Exception { 363 Map<String, Object> bundle = cldrBundles.get(id); 364 if (bundle != null) { 365 return bundle; 366 } 367 File file = new File(SOURCE_FILE_DIR + File.separator + id + ".xml"); 368 if (!file.exists()) { 369 // Skip if the file doesn't exist. 370 return Collections.emptyMap(); 371 } 372 373 info("..... main directory ....."); 374 LDMLParseHandler handler = new LDMLParseHandler(id); 375 parseLDMLFile(file, handler); 376 377 bundle = handler.getData(); 378 cldrBundles.put(id, bundle); 379 380 if (id.equals("root")) { 381 // Calendar data (firstDayOfWeek & minDaysInFirstWeek) 382 bundle = handlerSuppl.getData("root"); 383 if (bundle != null) { 384 //merge two maps into one map 385 Map<String, Object> temp = cldrBundles.remove(id); 386 bundle.putAll(temp); 387 cldrBundles.put(id, bundle); 388 } 389 } 390 return bundle; 391 } 392 393 // Parsers for data in "supplemental" directory 394 // 395 private static void parseSupplemental() throws Exception { 396 // Parse SupplementalData file and store the information in the HashMap 397 // Calendar information such as firstDay and minDay are stored in 398 // supplementalData.xml as of CLDR1.4. Individual territory is listed 399 // with its ISO 3166 country code while default is listed using UNM49 400 // region and composition numerical code (001 for World.) 401 // 402 // SupplementalData file also provides the "parent" locales which 403 // are othrwise not to be fallen back. Process them here as well. 404 // 405 handlerSuppl = new SupplementDataParseHandler(); 406 parseLDMLFile(new File(SPPL_SOURCE_FILE), handlerSuppl); 407 Map<String, Object> parentData = handlerSuppl.getData("root"); 408 parentData.keySet().stream() 409 .filter(key -> key.startsWith(PARENT_LOCALE_PREFIX)) 410 .forEach(key -> { 411 parentLocalesMap.put(key, new TreeSet( 412 Arrays.asList(((String)parentData.get(key)).split(" ")))); 413 }); 414 415 // Parse numberingSystems to get digit zero character information. 416 handlerNumbering = new NumberingSystemsParseHandler(); 417 parseLDMLFile(new File(NUMBERING_SOURCE_FILE), handlerNumbering); 418 419 // Parse metaZones to create mappings between Olson tzids and CLDR meta zone names 420 handlerMetaZones = new MetaZonesParseHandler(); 421 parseLDMLFile(new File(METAZONES_SOURCE_FILE), handlerMetaZones); 422 423 // Parse likelySubtags 424 handlerLikelySubtags = new LikelySubtagsParseHandler(); 425 parseLDMLFile(new File(LIKELYSUBTAGS_SOURCE_FILE), handlerLikelySubtags); 426 427 // Parse supplementalMetadata 428 // Currently only interested in deprecated time zone ids. 429 handlerSupplMeta = new SupplementalMetadataParseHandler(); 430 parseLDMLFile(new File(SPPL_META_SOURCE_FILE), handlerSupplMeta); 431 } 432 433 // Parsers for data in "bcp47" directory 434 // 435 private static void parseBCP47() throws Exception { 436 // Parse timezone 437 handlerTimeZone = new TimeZoneParseHandler(); 438 parseLDMLFile(new File(TIMEZONE_SOURCE_FILE), handlerTimeZone); 439 } 440 441 private static void parseLDMLFile(File srcfile, AbstractLDMLHandler handler) throws Exception { 442 info("..... Parsing " + srcfile.getName() + " ....."); 443 SAXParserFactory pf = SAXParserFactory.newInstance(); 444 pf.setValidating(true); 445 SAXParser parser = pf.newSAXParser(); 446 enableFileAccess(parser); 447 parser.parse(srcfile, handler); 448 } 449 450 private static StringBuilder getCandLocales(Locale cldrLoc) { 451 List<Locale> candList = getCandidateLocales(cldrLoc); 452 StringBuilder sb = new StringBuilder(); 453 for (Locale loc : candList) { 454 if (!loc.equals(Locale.ROOT)) { 455 sb.append(toLocaleName(loc.toLanguageTag())); 456 sb.append(","); 457 } 458 } 459 return sb; 460 } 461 462 private static List<Locale> getCandidateLocales(Locale cldrLoc) { 463 List<Locale> candList = new ArrayList<>(); 464 candList = applyParentLocales("", defCon.getCandidateLocales("", cldrLoc)); 465 return candList; 466 } 467 468 private static void convertBundles(List<Bundle> bundles) throws Exception { 469 // parent locales map. The mappings are put in base metaInfo file 470 // for now. 471 if (isBaseModule) { 472 metaInfo.putAll(parentLocalesMap); 473 } 474 475 for (Bundle bundle : bundles) { 476 // Get the target map, which contains all the data that should be 477 // visible for the bundle's locale 478 479 Map<String, Object> targetMap = bundle.getTargetMap(); 480 481 EnumSet<Bundle.Type> bundleTypes = bundle.getBundleTypes(); 482 483 if (bundle.isRoot()) { 484 // Add DateTimePatternChars because CLDR no longer supports localized patterns. 485 targetMap.put("DateTimePatternChars", "GyMdkHmsSEDFwWahKzZ"); 486 } 487 488 // Now the map contains just the entries that need to be in the resources bundles. 489 // Go ahead and generate them. 490 if (bundleTypes.contains(Bundle.Type.LOCALENAMES)) { 491 Map<String, Object> localeNamesMap = extractLocaleNames(targetMap, bundle.getID()); 492 if (!localeNamesMap.isEmpty() || bundle.isRoot()) { 493 bundleGenerator.generateBundle("util", "LocaleNames", bundle.getJavaID(), true, localeNamesMap, BundleType.OPEN); 494 } 495 } 496 if (bundleTypes.contains(Bundle.Type.CURRENCYNAMES)) { 497 Map<String, Object> currencyNamesMap = extractCurrencyNames(targetMap, bundle.getID(), bundle.getCurrencies()); 498 if (!currencyNamesMap.isEmpty() || bundle.isRoot()) { 499 bundleGenerator.generateBundle("util", "CurrencyNames", bundle.getJavaID(), true, currencyNamesMap, BundleType.OPEN); 500 } 501 } 502 if (bundleTypes.contains(Bundle.Type.TIMEZONENAMES)) { 503 Map<String, Object> zoneNamesMap = extractZoneNames(targetMap, bundle.getID()); 504 if (!zoneNamesMap.isEmpty() || bundle.isRoot()) { 505 bundleGenerator.generateBundle("util", "TimeZoneNames", bundle.getJavaID(), true, zoneNamesMap, BundleType.TIMEZONE); 506 } 507 } 508 if (bundleTypes.contains(Bundle.Type.CALENDARDATA)) { 509 Map<String, Object> calendarDataMap = extractCalendarData(targetMap, bundle.getID()); 510 if (!calendarDataMap.isEmpty() || bundle.isRoot()) { 511 bundleGenerator.generateBundle("util", "CalendarData", bundle.getJavaID(), true, calendarDataMap, BundleType.PLAIN); 512 } 513 } 514 if (bundleTypes.contains(Bundle.Type.FORMATDATA)) { 515 Map<String, Object> formatDataMap = extractFormatData(targetMap, bundle.getID()); 516 if (!formatDataMap.isEmpty() || bundle.isRoot()) { 517 bundleGenerator.generateBundle("text", "FormatData", bundle.getJavaID(), true, formatDataMap, BundleType.PLAIN); 518 } 519 } 520 521 // For AvailableLocales 522 metaInfo.get("AvailableLocales").add(toLanguageTag(bundle.getID())); 523 addLikelySubtags(metaInfo, "AvailableLocales", bundle.getID()); 524 } 525 bundleGenerator.generateMetaInfo(metaInfo); 526 } 527 528 static final Map<String, String> aliases = new HashMap<>(); 529 530 /** 531 * Translate the aliases into the real entries in the bundle map. 532 */ 533 static void handleAliases(Map<String, Object> bundleMap) { 534 Set bundleKeys = bundleMap.keySet(); 535 try { 536 for (String key : aliases.keySet()) { 537 String targetKey = aliases.get(key); 538 if (bundleKeys.contains(targetKey)) { 539 bundleMap.putIfAbsent(key, bundleMap.get(targetKey)); 540 } 541 } 542 } catch (Exception ex) { 543 Logger.getLogger(CLDRConverter.class.getName()).log(Level.SEVERE, null, ex); 544 } 545 } 546 547 /* 548 * Returns the language portion of the given id. 549 * If id is "root", "" is returned. 550 */ 551 static String getLanguageCode(String id) { 552 return "root".equals(id) ? "" : Locale.forLanguageTag(id.replaceAll("_", "-")).getLanguage(); 553 } 554 555 /** 556 * Examine if the id includes the country (territory) code. If it does, it returns 557 * the country code. 558 * Otherwise, it returns null. eg. when the id is "zh_Hans_SG", it return "SG". 559 * It does NOT return UN M.49 code, e.g., '001', as those three digit numbers cannot 560 * be translated into package names. 561 */ 562 static String getCountryCode(String id) { 563 String rgn = getRegionCode(id); 564 return rgn.length() == 2 ? rgn: null; 565 } 566 567 /** 568 * Examine if the id includes the region code. If it does, it returns 569 * the region code. 570 * Otherwise, it returns null. eg. when the id is "zh_Hans_SG", it return "SG". 571 * It DOES return UN M.49 code, e.g., '001', as well as ISO 3166 two letter country codes. 572 */ 573 static String getRegionCode(String id) { 574 return Locale.forLanguageTag(id.replaceAll("_", "-")).getCountry(); 575 } 576 577 private static class KeyComparator implements Comparator<String> { 578 static KeyComparator INSTANCE = new KeyComparator(); 579 580 private KeyComparator() { 581 } 582 583 @Override 584 public int compare(String o1, String o2) { 585 int len1 = o1.length(); 586 int len2 = o2.length(); 587 if (!isDigit(o1.charAt(0)) && !isDigit(o2.charAt(0))) { 588 // Shorter string comes first unless either starts with a digit. 589 if (len1 < len2) { 590 return -1; 591 } 592 if (len1 > len2) { 593 return 1; 594 } 595 } 596 return o1.compareTo(o2); 597 } 598 599 private boolean isDigit(char c) { 600 return c >= '0' && c <= '9'; 601 } 602 } 603 604 private static Map<String, Object> extractLocaleNames(Map<String, Object> map, String id) { 605 Map<String, Object> localeNames = new TreeMap<>(KeyComparator.INSTANCE); 606 for (String key : map.keySet()) { 607 if (key.startsWith(LOCALE_NAME_PREFIX)) { 608 switch (key) { 609 case LOCALE_SEPARATOR: 610 localeNames.put("ListCompositionPattern", map.get(key)); 611 break; 612 case LOCALE_KEYTYPE: 613 localeNames.put("ListKeyTypePattern", map.get(key)); 614 break; 615 default: 616 localeNames.put(key.substring(LOCALE_NAME_PREFIX.length()), map.get(key)); 617 break; 618 } 619 } 620 } 621 622 if (id.equals("root")) { 623 // Add display name pattern, which is not in CLDR 624 localeNames.put("DisplayNamePattern", "{0,choice,0#|1#{1}|2#{1} ({2})}"); 625 } 626 627 return localeNames; 628 } 629 630 @SuppressWarnings("AssignmentToForLoopParameter") 631 private static Map<String, Object> extractCurrencyNames(Map<String, Object> map, String id, String names) 632 throws Exception { 633 Map<String, Object> currencyNames = new TreeMap<>(KeyComparator.INSTANCE); 634 for (String key : map.keySet()) { 635 if (key.startsWith(CURRENCY_NAME_PREFIX)) { 636 currencyNames.put(key.substring(CURRENCY_NAME_PREFIX.length()), map.get(key)); 637 } else if (key.startsWith(CURRENCY_SYMBOL_PREFIX)) { 638 currencyNames.put(key.substring(CURRENCY_SYMBOL_PREFIX.length()), map.get(key)); 639 } 640 } 641 return currencyNames; 642 } 643 644 private static Map<String, Object> extractZoneNames(Map<String, Object> map, String id) { 645 Map<String, Object> names = new HashMap<>(); 646 647 // Copy over missing time zone ids from JRE for English locale 648 if (id.equals("en")) { 649 Map<String[], String> jreMetaMap = new HashMap<>(); 650 jreTimeZoneNames.stream().forEach(e -> { 651 String tzid = (String)e[0]; 652 String[] data = (String[])e[1]; 653 654 if (map.get(TIMEZONE_ID_PREFIX + tzid) == null && 655 handlerMetaZones.get(tzid) == null || 656 handlerMetaZones.get(tzid) != null && 657 map.get(METAZONE_ID_PREFIX + handlerMetaZones.get(tzid)) == null) { 658 // First, check the CLDR meta key 659 Optional<Map.Entry<String, String>> cldrMeta = 660 handlerMetaZones.getData().entrySet().stream() 661 .filter(me -> 662 Arrays.deepEquals(data, 663 (String[])map.get(METAZONE_ID_PREFIX + me.getValue()))) 664 .findAny(); 665 if (cldrMeta.isPresent()) { 666 names.put(tzid, cldrMeta.get().getValue()); 667 } else { 668 // check the JRE meta key, add if there is not. 669 Optional<Map.Entry<String[], String>> jreMeta = 670 jreMetaMap.entrySet().stream() 671 .filter(jm -> Arrays.deepEquals(data, jm.getKey())) 672 .findAny(); 673 if (jreMeta.isPresent()) { 674 names.put(tzid, jreMeta.get().getValue()); 675 } else { 676 String metaName = "JRE_" + tzid.replaceAll("[/-]", "_"); 677 names.put(METAZONE_ID_PREFIX + metaName, data); 678 names.put(tzid, metaName); 679 jreMetaMap.put(data, metaName); 680 } 681 } 682 } 683 }); 684 } 685 686 Arrays.stream(AVAILABLE_TZIDS).forEach(tzid -> { 687 // If the tzid is deprecated, get the data for the replacement id 688 String tzKey = Optional.ofNullable((String)handlerSupplMeta.get(tzid)) 689 .orElse(tzid); 690 Object data = map.get(TIMEZONE_ID_PREFIX + tzKey); 691 692 if (data instanceof String[]) { 693 names.put(tzid, data); 694 } else { 695 String meta = handlerMetaZones.get(tzKey); 696 if (meta != null) { 697 String metaKey = METAZONE_ID_PREFIX + meta; 698 data = map.get(metaKey); 699 if (data instanceof String[]) { 700 // Keep the metazone prefix here. 701 names.put(metaKey, data); 702 names.put(tzid, meta); 703 } 704 } 705 } 706 }); 707 708 return names; 709 } 710 711 /** 712 * Extracts the language independent calendar data. Each of the two keys, 713 * "firstDayOfWeek" and "minimalDaysInFirstWeek" has a string value consists of 714 * one or multiple occurrences of: 715 * i: rg1 rg2 ... rgn; 716 * where "i" is the data for the following regions (delimited by a space) after 717 * ":", and ends with a ";". 718 */ 719 private static Map<String, Object> extractCalendarData(Map<String, Object> map, String id) { 720 Map<String, Object> calendarData = new LinkedHashMap<>(); 721 if (id.equals("root")) { 722 calendarData.put("firstDayOfWeek", 723 IntStream.range(1, 8) 724 .mapToObj(String::valueOf) 725 .filter(d -> map.keySet().contains(CALENDAR_FIRSTDAY_PREFIX + d)) 726 .map(d -> d + ": " + map.get(CALENDAR_FIRSTDAY_PREFIX + d)) 727 .collect(Collectors.joining(";"))); 728 calendarData.put("minimalDaysInFirstWeek", 729 IntStream.range(0, 7) 730 .mapToObj(String::valueOf) 731 .filter(d -> map.keySet().contains(CALENDAR_MINDAYS_PREFIX + d)) 732 .map(d -> d + ": " + map.get(CALENDAR_MINDAYS_PREFIX + d)) 733 .collect(Collectors.joining(";"))); 734 } 735 return calendarData; 736 } 737 738 static final String[] FORMAT_DATA_ELEMENTS = { 739 "MonthNames", 740 "standalone.MonthNames", 741 "MonthAbbreviations", 742 "standalone.MonthAbbreviations", 743 "MonthNarrows", 744 "standalone.MonthNarrows", 745 "DayNames", 746 "standalone.DayNames", 747 "DayAbbreviations", 748 "standalone.DayAbbreviations", 749 "DayNarrows", 750 "standalone.DayNarrows", 751 "QuarterNames", 752 "standalone.QuarterNames", 753 "QuarterAbbreviations", 754 "standalone.QuarterAbbreviations", 755 "QuarterNarrows", 756 "standalone.QuarterNarrows", 757 "AmPmMarkers", 758 "narrow.AmPmMarkers", 759 "abbreviated.AmPmMarkers", 760 "long.Eras", 761 "Eras", 762 "narrow.Eras", 763 "field.era", 764 "field.year", 765 "field.month", 766 "field.week", 767 "field.weekday", 768 "field.dayperiod", 769 "field.hour", 770 "timezone.hourFormat", 771 "timezone.gmtFormat", 772 "field.minute", 773 "field.second", 774 "field.zone", 775 "TimePatterns", 776 "DatePatterns", 777 "DateTimePatterns", 778 "DateTimePatternChars" 779 }; 780 781 private static Map<String, Object> extractFormatData(Map<String, Object> map, String id) { 782 Map<String, Object> formatData = new LinkedHashMap<>(); 783 for (CalendarType calendarType : CalendarType.values()) { 784 if (calendarType == CalendarType.GENERIC) { 785 continue; 786 } 787 String prefix = calendarType.keyElementName(); 788 for (String element : FORMAT_DATA_ELEMENTS) { 789 String key = prefix + element; 790 copyIfPresent(map, "java.time." + key, formatData); 791 copyIfPresent(map, key, formatData); 792 } 793 } 794 795 for (String key : map.keySet()) { 796 // Copy available calendar names 797 if (key.startsWith(CLDRConverter.LOCALE_TYPE_PREFIX_CA)) { 798 String type = key.substring(CLDRConverter.LOCALE_TYPE_PREFIX_CA.length()); 799 for (CalendarType calendarType : CalendarType.values()) { 800 if (calendarType == CalendarType.GENERIC) { 801 continue; 802 } 803 if (type.equals(calendarType.lname())) { 804 Object value = map.get(key); 805 String dataKey = key.replace(LOCALE_TYPE_PREFIX_CA, 806 CALENDAR_NAME_PREFIX); 807 formatData.put(dataKey, value); 808 String ukey = CALENDAR_NAME_PREFIX + calendarType.uname(); 809 if (!dataKey.equals(ukey)) { 810 formatData.put(ukey, value); 811 } 812 } 813 } 814 } 815 } 816 817 copyIfPresent(map, "DefaultNumberingSystem", formatData); 818 819 @SuppressWarnings("unchecked") 820 List<String> numberingScripts = (List<String>) map.remove("numberingScripts"); 821 if (numberingScripts != null) { 822 for (String script : numberingScripts) { 823 copyIfPresent(map, script + "." + "NumberElements", formatData); 824 } 825 } else { 826 copyIfPresent(map, "NumberElements", formatData); 827 } 828 copyIfPresent(map, "NumberPatterns", formatData); 829 830 // put extra number elements for available scripts into formatData, if it is "root" 831 if (id.equals("root")) { 832 handlerNumbering.keySet().stream() 833 .filter(k -> !numberingScripts.contains(k)) 834 .forEach(k -> { 835 String[] ne = (String[])map.get("latn.NumberElements"); 836 String[] neNew = Arrays.copyOf(ne, ne.length); 837 neNew[4] = handlerNumbering.get(k).substring(0, 1); 838 formatData.put(k + ".NumberElements", neNew); 839 }); 840 } 841 return formatData; 842 } 843 844 private static void copyIfPresent(Map<String, Object> src, String key, Map<String, Object> dest) { 845 Object value = src.get(key); 846 if (value != null) { 847 dest.put(key, value); 848 } 849 } 850 851 // --- code below here is adapted from java.util.Properties --- 852 private static final String specialSaveCharsJava = "\""; 853 private static final String specialSaveCharsProperties = "=: \t\r\n\f#!"; 854 855 /* 856 * Converts unicodes to encoded \uxxxx 857 * and writes out any of the characters in specialSaveChars 858 * with a preceding slash 859 */ 860 static String saveConvert(String theString, boolean useJava) { 861 if (theString == null) { 862 return ""; 863 } 864 865 String specialSaveChars; 866 if (useJava) { 867 specialSaveChars = specialSaveCharsJava; 868 } else { 869 specialSaveChars = specialSaveCharsProperties; 870 } 871 boolean escapeSpace = false; 872 873 int len = theString.length(); 874 StringBuilder outBuffer = new StringBuilder(len * 2); 875 Formatter formatter = new Formatter(outBuffer, Locale.ROOT); 876 877 for (int x = 0; x < len; x++) { 878 char aChar = theString.charAt(x); 879 switch (aChar) { 880 case ' ': 881 if (x == 0 || escapeSpace) { 882 outBuffer.append('\\'); 883 } 884 outBuffer.append(' '); 885 break; 886 case '\\': 887 outBuffer.append('\\'); 888 outBuffer.append('\\'); 889 break; 890 case '\t': 891 outBuffer.append('\\'); 892 outBuffer.append('t'); 893 break; 894 case '\n': 895 outBuffer.append('\\'); 896 outBuffer.append('n'); 897 break; 898 case '\r': 899 outBuffer.append('\\'); 900 outBuffer.append('r'); 901 break; 902 case '\f': 903 outBuffer.append('\\'); 904 outBuffer.append('f'); 905 break; 906 default: 907 if (aChar < 0x0020 || (!USE_UTF8 && aChar > 0x007e)) { 908 formatter.format("\\u%04x", (int)aChar); 909 } else { 910 if (specialSaveChars.indexOf(aChar) != -1) { 911 outBuffer.append('\\'); 912 } 913 outBuffer.append(aChar); 914 } 915 } 916 } 917 return outBuffer.toString(); 918 } 919 920 private static String toLanguageTag(String locName) { 921 if (locName.indexOf('_') == -1) { 922 return locName; 923 } 924 String tag = locName.replaceAll("_", "-"); 925 Locale loc = Locale.forLanguageTag(tag); 926 return loc.toLanguageTag(); 927 } 928 929 private static void addLikelySubtags(Map<String, SortedSet<String>> metaInfo, String category, String id) { 930 String likelySubtag = handlerLikelySubtags.get(id); 931 if (likelySubtag != null) { 932 // Remove Script for now 933 metaInfo.get(category).add(toLanguageTag(likelySubtag).replaceFirst("-[A-Z][a-z]{3}", "")); 934 } 935 } 936 937 private static String toLocaleName(String tag) { 938 if (tag.indexOf('-') == -1) { 939 return tag; 940 } 941 return tag.replaceAll("-", "_"); 942 } 943 944 private static void setupBaseLocales(String localeList) { 945 Arrays.stream(localeList.split(",")) 946 .map(Locale::forLanguageTag) 947 .map(l -> Control.getControl(Control.FORMAT_DEFAULT) 948 .getCandidateLocales("", l)) 949 .forEach(BASE_LOCALES::addAll); 950 } 951 952 // applying parent locale rules to the passed candidates list 953 // This has to match with the one in sun.util.cldr.CLDRLocaleProviderAdapter 954 private static Map<Locale, Locale> childToParentLocaleMap = null; 955 private static List<Locale> applyParentLocales(String baseName, List<Locale> candidates) { 956 if (Objects.isNull(childToParentLocaleMap)) { 957 childToParentLocaleMap = new HashMap<>(); 958 parentLocalesMap.keySet().forEach(key -> { 959 String parent = key.substring(PARENT_LOCALE_PREFIX.length()).replaceAll("_", "-"); 960 parentLocalesMap.get(key).stream().forEach(child -> { 961 childToParentLocaleMap.put(Locale.forLanguageTag(child), 962 "root".equals(parent) ? Locale.ROOT : Locale.forLanguageTag(parent)); 963 }); 964 }); 965 } 966 967 // check irregular parents 968 for (int i = 0; i < candidates.size(); i++) { 969 Locale l = candidates.get(i); 970 Locale p = childToParentLocaleMap.get(l); 971 if (!l.equals(Locale.ROOT) && 972 Objects.nonNull(p) && 973 !candidates.get(i+1).equals(p)) { 974 List<Locale> applied = candidates.subList(0, i+1); 975 applied.addAll(applyParentLocales(baseName, defCon.getCandidateLocales(baseName, p))); 976 return applied; 977 } 978 } 979 980 return candidates; 981 } 982 983 private static void generateZoneName() throws Exception { 984 Files.createDirectories(Paths.get(DESTINATION_DIR, "java", "time", "format")); 985 Files.write(Paths.get(DESTINATION_DIR, "java", "time", "format", "ZoneName.java"), 986 Files.lines(Paths.get(zoneNameTempFile)) 987 .flatMap(l -> { 988 if (l.equals("%%%%ZIDMAP%%%%")) { 989 return zidMapEntry(); 990 } else if (l.equals("%%%%MZONEMAP%%%%")) { 991 return handlerMetaZones.mzoneMapEntry(); 992 } else if (l.equals("%%%%DEPRECATED%%%%")) { 993 return handlerSupplMeta.deprecatedMap(); 994 } else if (l.equals("%%%%TZDATALINK%%%%")) { 995 return tzDataLinkEntry(); 996 } else { 997 return Stream.of(l); 998 } 999 }) 1000 .collect(Collectors.toList()), 1001 StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); 1002 } 1003 1004 private static Stream<String> zidMapEntry() { 1005 Map<String, String> canonMap = new HashMap<>(); 1006 handlerTimeZone.getData().entrySet().stream() 1007 .forEach(e -> { 1008 String[] ids = ((String)e.getValue()).split("\\s"); 1009 for (int i = 1; i < ids.length; i++) { 1010 canonMap.put(ids[i], ids[0]); 1011 }}); 1012 return ZoneId.getAvailableZoneIds().stream() 1013 .map(id -> { 1014 String canonId = canonMap.getOrDefault(id, id); 1015 String meta = handlerMetaZones.get(canonId); 1016 String zone001 = handlerMetaZones.zidMap().get(meta); 1017 return zone001 == null ? "" : 1018 String.format(" \"%s\", \"%s\", \"%s\",", 1019 id, meta, zone001); 1020 }) 1021 .filter(s -> !s.isEmpty()) 1022 .sorted(); 1023 } 1024 1025 private static Stream<String> tzDataLinkEntry() { 1026 try { 1027 return Files.walk(Paths.get(tzDataDir), 1) 1028 .filter(p -> !Files.isDirectory(p)) 1029 .flatMap(CLDRConverter::extractLinks) 1030 .sorted(); 1031 } catch (IOException e) { 1032 throw new UncheckedIOException(e); 1033 } 1034 } 1035 1036 private static Stream<String> extractLinks(Path tzFile) { 1037 try { 1038 return Files.lines(tzFile) 1039 .filter(l -> l.startsWith("Link")) 1040 .map(l -> l.replaceFirst("^Link[\\s]+(\\S+)\\s+(\\S+).*", 1041 " \"$2\", \"$1\",")); 1042 } catch (IOException e) { 1043 throw new UncheckedIOException(e); 1044 } 1045 } 1046 }