1 /*
2 * Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25
26 package build.tools.cldrconverter;
27
28 import static build.tools.cldrconverter.Bundle.jreTimeZoneNames;
29 import build.tools.cldrconverter.BundleGenerator.BundleType;
30 import java.io.File;
31 import java.io.IOException;
32 import java.io.UncheckedIOException;
33 import java.nio.file.*;
34 import java.text.MessageFormat;
35 import java.time.*;
36 import java.util.*;
37 import java.util.ResourceBundle.Control;
38 import java.util.logging.Level;
39 import java.util.logging.Logger;
40 import java.util.stream.Collectors;
41 import java.util.stream.IntStream;
42 import java.util.stream.Stream;
43 import javax.xml.parsers.SAXParser;
44 import javax.xml.parsers.SAXParserFactory;
45 import org.xml.sax.SAXNotRecognizedException;
46 import org.xml.sax.SAXNotSupportedException;
47
48
49 /**
50 * Converts locale data from "Locale Data Markup Language" format to
51 * JRE resource bundle format. LDML is the format used by the Common
52 * Locale Data Repository maintained by the Unicode Consortium.
53 */
54 public class CLDRConverter {
55
56 static final String LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldml.dtd";
57 static final String SPPL_LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldmlSupplemental.dtd";
58 static final String BCP47_LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldmlBCP47.dtd";
59
60
61 private static String CLDR_BASE;
62 static String LOCAL_LDML_DTD;
63 static String LOCAL_SPPL_LDML_DTD;
64 static String LOCAL_BCP47_LDML_DTD;
65 private static String SOURCE_FILE_DIR;
66 private static String SPPL_SOURCE_FILE;
67 private static String SPPL_META_SOURCE_FILE;
68 private static String NUMBERING_SOURCE_FILE;
69 private static String METAZONES_SOURCE_FILE;
70 private static String LIKELYSUBTAGS_SOURCE_FILE;
71 private static String TIMEZONE_SOURCE_FILE;
72 private static String WINZONES_SOURCE_FILE;
73 static String DESTINATION_DIR = "build/gensrc";
74
75 static final String LOCALE_NAME_PREFIX = "locale.displayname.";
76 static final String LOCALE_SEPARATOR = LOCALE_NAME_PREFIX + "separator";
77 static final String LOCALE_KEYTYPE = LOCALE_NAME_PREFIX + "keytype";
78 static final String LOCALE_KEY_PREFIX = LOCALE_NAME_PREFIX + "key.";
79 static final String LOCALE_TYPE_PREFIX = LOCALE_NAME_PREFIX + "type.";
80 static final String LOCALE_TYPE_PREFIX_CA = LOCALE_TYPE_PREFIX + "ca.";
81 static final String CURRENCY_SYMBOL_PREFIX = "currency.symbol.";
82 static final String CURRENCY_NAME_PREFIX = "currency.displayname.";
83 static final String CALENDAR_NAME_PREFIX = "calendarname.";
84 static final String CALENDAR_FIRSTDAY_PREFIX = "firstDay.";
85 static final String CALENDAR_MINDAYS_PREFIX = "minDays.";
86 static final String TIMEZONE_ID_PREFIX = "timezone.id.";
87 static final String EXEMPLAR_CITY_PREFIX = "timezone.excity.";
88 static final String ZONE_NAME_PREFIX = "timezone.displayname.";
89 static final String METAZONE_ID_PREFIX = "metazone.id.";
90 static final String PARENT_LOCALE_PREFIX = "parentLocale.";
91 static final String[] EMPTY_ZONE = {"", "", "", "", "", ""};
92
93 private static SupplementDataParseHandler handlerSuppl;
94 private static LikelySubtagsParseHandler handlerLikelySubtags;
95 private static WinZonesParseHandler handlerWinZones;
96 static SupplementalMetadataParseHandler handlerSupplMeta;
97 static NumberingSystemsParseHandler handlerNumbering;
98 static MetaZonesParseHandler handlerMetaZones;
99 static TimeZoneParseHandler handlerTimeZone;
100 private static BundleGenerator bundleGenerator;
101
102 // java.base module related
103 static boolean isBaseModule = false;
104 static final Set<Locale> BASE_LOCALES = new HashSet<>();
105
106 // "parentLocales" map
107 private static final Map<String, SortedSet<String>> parentLocalesMap = new HashMap<>();
108 private static final ResourceBundle.Control defCon =
109 ResourceBundle.Control.getControl(ResourceBundle.Control.FORMAT_DEFAULT);
110
111 private static final String[] AVAILABLE_TZIDS = TimeZone.getAvailableIDs();
112 private static String zoneNameTempFile;
113 private static String tzDataDir;
114 private static final Map<String, String> canonicalTZMap = new HashMap<>();
115
116 static enum DraftType {
117 UNCONFIRMED,
118 PROVISIONAL,
119 CONTRIBUTED,
120 APPROVED;
121
122 private static final Map<String, DraftType> map = new HashMap<>();
123 static {
124 for (DraftType dt : values()) {
125 map.put(dt.getKeyword(), dt);
126 }
127 }
128 static private DraftType defaultType = CONTRIBUTED;
129
130 private final String keyword;
131
132 private DraftType() {
133 keyword = this.name().toLowerCase(Locale.ROOT);
134
135 }
136
137 static DraftType forKeyword(String keyword) {
138 return map.get(keyword);
139 }
140
141 static DraftType getDefault() {
142 return defaultType;
143 }
144
145 static void setDefault(String keyword) {
146 defaultType = Objects.requireNonNull(forKeyword(keyword));
147 }
148
149 String getKeyword() {
150 return keyword;
151 }
152 }
153
154 static boolean USE_UTF8 = false;
155 private static boolean verbose;
156
157 private CLDRConverter() {
158 // no instantiation
159 }
160
161 @SuppressWarnings("AssignmentToForLoopParameter")
162 public static void main(String[] args) throws Exception {
163 if (args.length != 0) {
164 String currentArg = null;
165 try {
166 for (int i = 0; i < args.length; i++) {
167 currentArg = args[i];
168 switch (currentArg) {
169 case "-draft":
170 String draftDataType = args[++i];
171 try {
172 DraftType.setDefault(draftDataType);
173 } catch (NullPointerException e) {
174 severe("Error: incorrect draft value: %s%n", draftDataType);
175 System.exit(1);
176 }
177 info("Using the specified data type: %s%n", draftDataType);
178 break;
179
180 case "-base":
181 // base directory for input files
182 CLDR_BASE = args[++i];
183 if (!CLDR_BASE.endsWith("/")) {
184 CLDR_BASE += "/";
185 }
186 break;
187
188 case "-baselocales":
189 // base locales
190 setupBaseLocales(args[++i]);
191 break;
192
193 case "-basemodule":
194 // indicates java.base module resource generation
195 isBaseModule = true;
196 break;
197
198 case "-o":
199 // output directory
200 DESTINATION_DIR = args[++i];
201 break;
202
203 case "-utf8":
204 USE_UTF8 = true;
205 break;
206
207 case "-verbose":
208 verbose = true;
209 break;
210
211 case "-zntempfile":
212 zoneNameTempFile = args[++i];
213 break;
214
215 case "-tzdatadir":
216 tzDataDir = args[++i];
217 break;
218
219 case "-help":
220 usage();
221 System.exit(0);
222 break;
223
224 default:
225 throw new RuntimeException();
226 }
227 }
228 } catch (RuntimeException e) {
229 severe("unknown or imcomplete arg(s): " + currentArg);
230 usage();
231 System.exit(1);
232 }
233 }
234
235 // Set up path names
236 LOCAL_LDML_DTD = CLDR_BASE + "/dtd/ldml.dtd";
237 LOCAL_SPPL_LDML_DTD = CLDR_BASE + "/dtd/ldmlSupplemental.dtd";
238 LOCAL_BCP47_LDML_DTD = CLDR_BASE + "/dtd/ldmlBCP47.dtd";
239 SOURCE_FILE_DIR = CLDR_BASE + "/main";
240 SPPL_SOURCE_FILE = CLDR_BASE + "/supplemental/supplementalData.xml";
241 LIKELYSUBTAGS_SOURCE_FILE = CLDR_BASE + "/supplemental/likelySubtags.xml";
242 NUMBERING_SOURCE_FILE = CLDR_BASE + "/supplemental/numberingSystems.xml";
243 METAZONES_SOURCE_FILE = CLDR_BASE + "/supplemental/metaZones.xml";
244 TIMEZONE_SOURCE_FILE = CLDR_BASE + "/bcp47/timezone.xml";
245 SPPL_META_SOURCE_FILE = CLDR_BASE + "/supplemental/supplementalMetadata.xml";
246 WINZONES_SOURCE_FILE = CLDR_BASE + "/supplemental/windowsZones.xml";
247
248 if (BASE_LOCALES.isEmpty()) {
249 setupBaseLocales("en-US");
250 }
251
252 bundleGenerator = new ResourceBundleGenerator();
253
254 // Parse data independent of locales
255 parseSupplemental();
256 parseBCP47();
257
258 List<Bundle> bundles = readBundleList();
259 convertBundles(bundles);
260
261 if (isBaseModule) {
262 // Generate java.time.format.ZoneName.java
263 generateZoneName();
264
265 // Generate Windows tzmappings
266 generateWindowsTZMappings();
267 }
268 }
269
270 private static void usage() {
271 errout("Usage: java CLDRConverter [options]%n"
272 + "\t-help output this usage message and exit%n"
273 + "\t-verbose output information%n"
274 + "\t-draft [contributed | approved | provisional | unconfirmed]%n"
275 + "\t\t draft level for using data (default: contributed)%n"
276 + "\t-base dir base directory for CLDR input files%n"
277 + "\t-basemodule generates bundles that go into java.base module%n"
278 + "\t-baselocales loc(,loc)* locales that go into the base module%n"
279 + "\t-o dir output directory (default: ./build/gensrc)%n"
280 + "\t-zntempfile template file for java.time.format.ZoneName.java%n"
281 + "\t-tzdatadir tzdata directory for java.time.format.ZoneName.java%n"
282 + "\t-utf8 use UTF-8 rather than \\uxxxx (for debug)%n");
283 }
284
285 static void info(String fmt, Object... args) {
286 if (verbose) {
287 System.out.printf(fmt, args);
288 }
289 }
290
291 static void info(String msg) {
292 if (verbose) {
293 System.out.println(msg);
294 }
295 }
296
297 static void warning(String fmt, Object... args) {
298 System.err.print("Warning: ");
299 System.err.printf(fmt, args);
300 }
301
302 static void warning(String msg) {
303 System.err.print("Warning: ");
304 errout(msg);
305 }
306
307 static void severe(String fmt, Object... args) {
308 System.err.print("Error: ");
309 System.err.printf(fmt, args);
310 }
311
312 static void severe(String msg) {
313 System.err.print("Error: ");
314 errout(msg);
315 }
316
317 private static void errout(String msg) {
318 if (msg.contains("%n")) {
319 System.err.printf(msg);
320 } else {
321 System.err.println(msg);
322 }
323 }
324
325 /**
326 * Configure the parser to allow access to DTDs on the file system.
327 */
328 private static void enableFileAccess(SAXParser parser) throws SAXNotSupportedException {
329 try {
330 parser.setProperty("http://javax.xml.XMLConstants/property/accessExternalDTD", "file");
331 } catch (SAXNotRecognizedException ignore) {
332 // property requires >= JAXP 1.5
333 }
334 }
335
336 private static List<Bundle> readBundleList() throws Exception {
337 List<Bundle> retList = new ArrayList<>();
338 Path path = FileSystems.getDefault().getPath(SOURCE_FILE_DIR);
339 try (DirectoryStream<Path> dirStr = Files.newDirectoryStream(path)) {
340 for (Path entry : dirStr) {
341 String fileName = entry.getFileName().toString();
342 if (fileName.endsWith(".xml")) {
343 String id = fileName.substring(0, fileName.indexOf('.'));
344 Locale cldrLoc = Locale.forLanguageTag(toLanguageTag(id));
345 StringBuilder sb = getCandLocales(cldrLoc);
346 if (sb.indexOf("root") == -1) {
347 sb.append("root");
348 }
349 retList.add(new Bundle(id, sb.toString(), null, null));
350 }
351 }
352 }
353
354 // Sort the bundles based on id. This will make sure all the parent bundles are
355 // processed first, e.g., for en_GB bundle, en_001, and "root" comes before
356 // en_GB. In order for "root" to come at the beginning, "root" is replaced with
357 // empty string on comparison.
358 retList.sort((o1, o2) -> {
359 String id1 = o1.getID();
360 String id2 = o2.getID();
361 if(id1.equals("root")) {
362 id1 = "";
363 }
364 if(id2.equals("root")) {
365 id2 = "";
366 }
367 return id1.compareTo(id2);
368 });
369 return retList;
370 }
371
372 private static final Map<String, Map<String, Object>> cldrBundles = new HashMap<>();
373
374 private static Map<String, SortedSet<String>> metaInfo = new HashMap<>();
375
376 static {
377 // For generating information on supported locales.
378 metaInfo.put("AvailableLocales", new TreeSet<>());
379 }
380
381 static Map<String, Object> getCLDRBundle(String id) throws Exception {
382 Map<String, Object> bundle = cldrBundles.get(id);
383 if (bundle != null) {
384 return bundle;
385 }
386 File file = new File(SOURCE_FILE_DIR + File.separator + id + ".xml");
387 if (!file.exists()) {
388 // Skip if the file doesn't exist.
389 return Collections.emptyMap();
390 }
391
392 info("..... main directory .....");
393 LDMLParseHandler handler = new LDMLParseHandler(id);
394 parseLDMLFile(file, handler);
395
396 bundle = handler.getData();
397 cldrBundles.put(id, bundle);
398
399 if (id.equals("root")) {
400 // Calendar data (firstDayOfWeek & minDaysInFirstWeek)
401 bundle = handlerSuppl.getData("root");
402 if (bundle != null) {
403 //merge two maps into one map
404 Map<String, Object> temp = cldrBundles.remove(id);
405 bundle.putAll(temp);
406 cldrBundles.put(id, bundle);
407 }
408 }
409 return bundle;
410 }
411
412 // Parsers for data in "supplemental" directory
413 //
414 private static void parseSupplemental() throws Exception {
415 // Parse SupplementalData file and store the information in the HashMap
416 // Calendar information such as firstDay and minDay are stored in
417 // supplementalData.xml as of CLDR1.4. Individual territory is listed
418 // with its ISO 3166 country code while default is listed using UNM49
419 // region and composition numerical code (001 for World.)
420 //
421 // SupplementalData file also provides the "parent" locales which
422 // are othrwise not to be fallen back. Process them here as well.
423 //
424 handlerSuppl = new SupplementDataParseHandler();
425 parseLDMLFile(new File(SPPL_SOURCE_FILE), handlerSuppl);
426 Map<String, Object> parentData = handlerSuppl.getData("root");
427 parentData.keySet().stream()
428 .filter(key -> key.startsWith(PARENT_LOCALE_PREFIX))
429 .forEach(key -> {
430 parentLocalesMap.put(key, new TreeSet(
431 Arrays.asList(((String)parentData.get(key)).split(" "))));
432 });
433
434 // Parse numberingSystems to get digit zero character information.
435 handlerNumbering = new NumberingSystemsParseHandler();
436 parseLDMLFile(new File(NUMBERING_SOURCE_FILE), handlerNumbering);
437
438 // Parse metaZones to create mappings between Olson tzids and CLDR meta zone names
439 handlerMetaZones = new MetaZonesParseHandler();
440 parseLDMLFile(new File(METAZONES_SOURCE_FILE), handlerMetaZones);
441
442 // Parse likelySubtags
443 handlerLikelySubtags = new LikelySubtagsParseHandler();
444 parseLDMLFile(new File(LIKELYSUBTAGS_SOURCE_FILE), handlerLikelySubtags);
445
446 // Parse supplementalMetadata
447 // Currently interested in deprecated time zone ids and language aliases.
448 handlerSupplMeta = new SupplementalMetadataParseHandler();
449 parseLDMLFile(new File(SPPL_META_SOURCE_FILE), handlerSupplMeta);
450
451 // Parse windowsZones
452 handlerWinZones = new WinZonesParseHandler();
453 parseLDMLFile(new File(WINZONES_SOURCE_FILE), handlerWinZones);
454 }
455
456 // Parsers for data in "bcp47" directory
457 //
458 private static void parseBCP47() throws Exception {
459 // Parse timezone
460 handlerTimeZone = new TimeZoneParseHandler();
461 parseLDMLFile(new File(TIMEZONE_SOURCE_FILE), handlerTimeZone);
462
463 // canonical tz name map
464 // alias -> primary
465 handlerTimeZone.getData().forEach((k, v) -> {
466 String[] ids = ((String)v).split("\\s");
467 for (int i = 1; i < ids.length; i++) {
468 canonicalTZMap.put(ids[i], ids[0]);
469 }
470 });
471 }
472
473 private static void parseLDMLFile(File srcfile, AbstractLDMLHandler handler) throws Exception {
474 info("..... Parsing " + srcfile.getName() + " .....");
475 SAXParserFactory pf = SAXParserFactory.newInstance();
476 pf.setValidating(true);
477 SAXParser parser = pf.newSAXParser();
478 enableFileAccess(parser);
479 parser.parse(srcfile, handler);
480 }
481
482 private static StringBuilder getCandLocales(Locale cldrLoc) {
483 List<Locale> candList = getCandidateLocales(cldrLoc);
484 StringBuilder sb = new StringBuilder();
485 for (Locale loc : candList) {
486 if (!loc.equals(Locale.ROOT)) {
487 sb.append(toLocaleName(loc.toLanguageTag()));
488 sb.append(",");
489 }
490 }
491 return sb;
492 }
493
494 private static List<Locale> getCandidateLocales(Locale cldrLoc) {
495 List<Locale> candList = new ArrayList<>();
496 candList = applyParentLocales("", defCon.getCandidateLocales("", cldrLoc));
497 return candList;
498 }
499
500 private static void convertBundles(List<Bundle> bundles) throws Exception {
501 // parent locales map. The mappings are put in base metaInfo file
502 // for now.
503 if (isBaseModule) {
504 metaInfo.putAll(parentLocalesMap);
505 }
506
507 for (Bundle bundle : bundles) {
508 // Get the target map, which contains all the data that should be
509 // visible for the bundle's locale
510
511 Map<String, Object> targetMap = bundle.getTargetMap();
512
513 EnumSet<Bundle.Type> bundleTypes = bundle.getBundleTypes();
514
515 if (bundle.isRoot()) {
516 // Add DateTimePatternChars because CLDR no longer supports localized patterns.
517 targetMap.put("DateTimePatternChars", "GyMdkHmsSEDFwWahKzZ");
518 }
519
520 // Now the map contains just the entries that need to be in the resources bundles.
521 // Go ahead and generate them.
522 if (bundleTypes.contains(Bundle.Type.LOCALENAMES)) {
523 Map<String, Object> localeNamesMap = extractLocaleNames(targetMap, bundle.getID());
524 if (!localeNamesMap.isEmpty() || bundle.isRoot()) {
525 bundleGenerator.generateBundle("util", "LocaleNames", bundle.getJavaID(), true, localeNamesMap, BundleType.OPEN);
526 }
527 }
528 if (bundleTypes.contains(Bundle.Type.CURRENCYNAMES)) {
529 Map<String, Object> currencyNamesMap = extractCurrencyNames(targetMap, bundle.getID(), bundle.getCurrencies());
530 if (!currencyNamesMap.isEmpty() || bundle.isRoot()) {
531 bundleGenerator.generateBundle("util", "CurrencyNames", bundle.getJavaID(), true, currencyNamesMap, BundleType.OPEN);
532 }
533 }
534 if (bundleTypes.contains(Bundle.Type.TIMEZONENAMES)) {
535 Map<String, Object> zoneNamesMap = extractZoneNames(targetMap, bundle.getID());
536 if (!zoneNamesMap.isEmpty() || bundle.isRoot()) {
537 bundleGenerator.generateBundle("util", "TimeZoneNames", bundle.getJavaID(), true, zoneNamesMap, BundleType.TIMEZONE);
538 }
539 }
540 if (bundleTypes.contains(Bundle.Type.CALENDARDATA)) {
541 Map<String, Object> calendarDataMap = extractCalendarData(targetMap, bundle.getID());
542 if (!calendarDataMap.isEmpty() || bundle.isRoot()) {
543 bundleGenerator.generateBundle("util", "CalendarData", bundle.getJavaID(), true, calendarDataMap, BundleType.PLAIN);
544 }
545 }
546 if (bundleTypes.contains(Bundle.Type.FORMATDATA)) {
547 Map<String, Object> formatDataMap = extractFormatData(targetMap, bundle.getID());
548 if (!formatDataMap.isEmpty() || bundle.isRoot()) {
549 bundleGenerator.generateBundle("text", "FormatData", bundle.getJavaID(), true, formatDataMap, BundleType.PLAIN);
550 }
551 }
552
553 // For AvailableLocales
554 metaInfo.get("AvailableLocales").add(toLanguageTag(bundle.getID()));
555 addLikelySubtags(metaInfo, "AvailableLocales", bundle.getID());
556 }
557 bundleGenerator.generateMetaInfo(metaInfo);
558 }
559
560 static final Map<String, String> aliases = new HashMap<>();
561
562 /**
563 * Translate the aliases into the real entries in the bundle map.
564 */
565 static void handleAliases(Map<String, Object> bundleMap) {
566 Set bundleKeys = bundleMap.keySet();
567 try {
568 for (String key : aliases.keySet()) {
569 String targetKey = aliases.get(key);
570 if (bundleKeys.contains(targetKey)) {
571 bundleMap.putIfAbsent(key, bundleMap.get(targetKey));
572 }
573 }
574 } catch (Exception ex) {
575 Logger.getLogger(CLDRConverter.class.getName()).log(Level.SEVERE, null, ex);
576 }
577 }
578
579 /*
580 * Returns the language portion of the given id.
581 * If id is "root", "" is returned.
582 */
583 static String getLanguageCode(String id) {
584 return "root".equals(id) ? "" : Locale.forLanguageTag(id.replaceAll("_", "-")).getLanguage();
585 }
586
587 /**
588 * Examine if the id includes the country (territory) code. If it does, it returns
589 * the country code.
590 * Otherwise, it returns null. eg. when the id is "zh_Hans_SG", it return "SG".
591 * It does NOT return UN M.49 code, e.g., '001', as those three digit numbers cannot
592 * be translated into package names.
593 */
594 static String getCountryCode(String id) {
595 String rgn = getRegionCode(id);
596 return rgn.length() == 2 ? rgn: null;
597 }
598
599 /**
600 * Examine if the id includes the region code. If it does, it returns
601 * the region code.
602 * Otherwise, it returns null. eg. when the id is "zh_Hans_SG", it return "SG".
603 * It DOES return UN M.49 code, e.g., '001', as well as ISO 3166 two letter country codes.
604 */
605 static String getRegionCode(String id) {
606 return Locale.forLanguageTag(id.replaceAll("_", "-")).getCountry();
607 }
608
609 private static class KeyComparator implements Comparator<String> {
610 static KeyComparator INSTANCE = new KeyComparator();
611
612 private KeyComparator() {
613 }
614
615 @Override
616 public int compare(String o1, String o2) {
617 int len1 = o1.length();
618 int len2 = o2.length();
619 if (!isDigit(o1.charAt(0)) && !isDigit(o2.charAt(0))) {
620 // Shorter string comes first unless either starts with a digit.
621 if (len1 < len2) {
622 return -1;
623 }
624 if (len1 > len2) {
625 return 1;
626 }
627 }
628 return o1.compareTo(o2);
629 }
630
631 private boolean isDigit(char c) {
632 return c >= '0' && c <= '9';
633 }
634 }
635
636 private static Map<String, Object> extractLocaleNames(Map<String, Object> map, String id) {
637 Map<String, Object> localeNames = new TreeMap<>(KeyComparator.INSTANCE);
638 for (String key : map.keySet()) {
639 if (key.startsWith(LOCALE_NAME_PREFIX)) {
640 switch (key) {
641 case LOCALE_SEPARATOR:
642 localeNames.put("ListCompositionPattern", map.get(key));
643 break;
644 case LOCALE_KEYTYPE:
645 localeNames.put("ListKeyTypePattern", map.get(key));
646 break;
647 default:
648 localeNames.put(key.substring(LOCALE_NAME_PREFIX.length()), map.get(key));
649 break;
650 }
651 }
652 }
653
654 if (id.equals("root")) {
655 // Add display name pattern, which is not in CLDR
656 localeNames.put("DisplayNamePattern", "{0,choice,0#|1#{1}|2#{1} ({2})}");
657 }
658
659 return localeNames;
660 }
661
662 @SuppressWarnings("AssignmentToForLoopParameter")
663 private static Map<String, Object> extractCurrencyNames(Map<String, Object> map, String id, String names)
664 throws Exception {
665 Map<String, Object> currencyNames = new TreeMap<>(KeyComparator.INSTANCE);
666 for (String key : map.keySet()) {
667 if (key.startsWith(CURRENCY_NAME_PREFIX)) {
668 currencyNames.put(key.substring(CURRENCY_NAME_PREFIX.length()), map.get(key));
669 } else if (key.startsWith(CURRENCY_SYMBOL_PREFIX)) {
670 currencyNames.put(key.substring(CURRENCY_SYMBOL_PREFIX.length()), map.get(key));
671 }
672 }
673 return currencyNames;
674 }
675
676 private static Map<String, Object> extractZoneNames(Map<String, Object> map, String id) {
677 Map<String, Object> names = new HashMap<>();
678
679 // Copy over missing time zone ids from JRE for English locale
680 if (id.equals("en")) {
681 Map<String[], String> jreMetaMap = new HashMap<>();
682 jreTimeZoneNames.stream().forEach(e -> {
683 String tzid = (String)e[0];
684 String[] data = (String[])e[1];
685
686 if (map.get(TIMEZONE_ID_PREFIX + tzid) == null &&
687 handlerMetaZones.get(tzid) == null ||
688 handlerMetaZones.get(tzid) != null &&
689 map.get(METAZONE_ID_PREFIX + handlerMetaZones.get(tzid)) == null) {
690
691 // First, check the alias
692 String canonID = canonicalTZMap.get(tzid);
693 if (canonID != null && !tzid.equals(canonID)) {
694 Object value = map.get(TIMEZONE_ID_PREFIX + canonID);
695 if (value != null) {
696 names.put(tzid, value);
697 return;
698 } else {
699 String meta = handlerMetaZones.get(canonID);
700 if (meta != null) {
701 value = map.get(METAZONE_ID_PREFIX + meta);
702 if (value != null) {
703 names.put(tzid, meta);
704 return;
705 }
706 }
707 }
708 }
709
710 // Check the CLDR meta key
711 Optional<Map.Entry<String, String>> cldrMeta =
712 handlerMetaZones.getData().entrySet().stream()
713 .filter(me ->
714 Arrays.deepEquals(data,
715 (String[])map.get(METAZONE_ID_PREFIX + me.getValue())))
716 .findAny();
717 cldrMeta.ifPresentOrElse(meta -> names.put(tzid, meta.getValue()), () -> {
718 // Check the JRE meta key, add if there is not.
719 Optional<Map.Entry<String[], String>> jreMeta =
720 jreMetaMap.entrySet().stream()
721 .filter(jm -> Arrays.deepEquals(data, jm.getKey()))
722 .findAny();
723 jreMeta.ifPresentOrElse(meta -> names.put(tzid, meta.getValue()), () -> {
724 String metaName = "JRE_" + tzid.replaceAll("[/-]", "_");
725 names.put(METAZONE_ID_PREFIX + metaName, data);
726 names.put(tzid, metaName);
727 });
728 });
729 }
730 });
731 }
732
733 Arrays.stream(AVAILABLE_TZIDS).forEach(tzid -> {
734 // If the tzid is deprecated, get the data for the replacement id
735 String tzKey = Optional.ofNullable((String)handlerSupplMeta.get(tzid))
736 .orElse(tzid);
737 Object data = map.get(TIMEZONE_ID_PREFIX + tzKey);
738
739 if (data instanceof String[]) {
740 names.put(tzid, data);
741 } else {
742 String meta = handlerMetaZones.get(tzKey);
743 if (meta != null) {
744 String metaKey = METAZONE_ID_PREFIX + meta;
745 data = map.get(metaKey);
746 if (data instanceof String[]) {
747 // Keep the metazone prefix here.
748 names.put(metaKey, data);
749 names.put(tzid, meta);
750 }
751 }
752 }
753 });
754
755 // exemplar cities.
756 Map<String, Object> exCities = map.entrySet().stream()
757 .filter(e -> e.getKey().startsWith(CLDRConverter.EXEMPLAR_CITY_PREFIX))
758 .collect(Collectors
759 .toMap(Map.Entry::getKey, Map.Entry::getValue));
760 names.putAll(exCities);
761
762 if (!id.equals("en") &&
763 !names.isEmpty()) {
764 // CLDR does not have UTC entry, so add it here.
765 names.put("UTC", EMPTY_ZONE);
766
767 // no metazone zones
768 Arrays.asList(handlerMetaZones.get(MetaZonesParseHandler.NO_METAZONE_KEY)
769 .split("\\s")).stream()
770 .forEach(tz -> {
771 names.put(tz, EMPTY_ZONE);
772 });
773 }
774
775 return names;
776 }
777
778 /**
779 * Extracts the language independent calendar data. Each of the two keys,
780 * "firstDayOfWeek" and "minimalDaysInFirstWeek" has a string value consists of
781 * one or multiple occurrences of:
782 * i: rg1 rg2 ... rgn;
783 * where "i" is the data for the following regions (delimited by a space) after
784 * ":", and ends with a ";".
785 */
786 private static Map<String, Object> extractCalendarData(Map<String, Object> map, String id) {
787 Map<String, Object> calendarData = new LinkedHashMap<>();
788 if (id.equals("root")) {
789 calendarData.put("firstDayOfWeek",
790 IntStream.range(1, 8)
791 .mapToObj(String::valueOf)
792 .filter(d -> map.keySet().contains(CALENDAR_FIRSTDAY_PREFIX + d))
793 .map(d -> d + ": " + map.get(CALENDAR_FIRSTDAY_PREFIX + d))
794 .collect(Collectors.joining(";")));
795 calendarData.put("minimalDaysInFirstWeek",
796 IntStream.range(0, 7)
797 .mapToObj(String::valueOf)
798 .filter(d -> map.keySet().contains(CALENDAR_MINDAYS_PREFIX + d))
799 .map(d -> d + ": " + map.get(CALENDAR_MINDAYS_PREFIX + d))
800 .collect(Collectors.joining(";")));
801 }
802 return calendarData;
803 }
804
805 static final String[] FORMAT_DATA_ELEMENTS = {
806 "MonthNames",
807 "standalone.MonthNames",
808 "MonthAbbreviations",
809 "standalone.MonthAbbreviations",
810 "MonthNarrows",
811 "standalone.MonthNarrows",
812 "DayNames",
813 "standalone.DayNames",
814 "DayAbbreviations",
815 "standalone.DayAbbreviations",
816 "DayNarrows",
817 "standalone.DayNarrows",
818 "QuarterNames",
819 "standalone.QuarterNames",
820 "QuarterAbbreviations",
821 "standalone.QuarterAbbreviations",
822 "QuarterNarrows",
823 "standalone.QuarterNarrows",
824 "AmPmMarkers",
825 "narrow.AmPmMarkers",
826 "abbreviated.AmPmMarkers",
827 "long.Eras",
828 "Eras",
829 "narrow.Eras",
830 "field.era",
831 "field.year",
832 "field.month",
833 "field.week",
834 "field.weekday",
835 "field.dayperiod",
836 "field.hour",
837 "timezone.hourFormat",
838 "timezone.gmtFormat",
839 "timezone.gmtZeroFormat",
840 "timezone.regionFormat",
841 "timezone.regionFormat.daylight",
842 "timezone.regionFormat.standard",
843 "field.minute",
844 "field.second",
845 "field.zone",
846 "TimePatterns",
847 "DatePatterns",
848 "DateTimePatterns",
849 "DateTimePatternChars"
850 };
851
852 private static Map<String, Object> extractFormatData(Map<String, Object> map, String id) {
853 Map<String, Object> formatData = new LinkedHashMap<>();
854 for (CalendarType calendarType : CalendarType.values()) {
855 if (calendarType == CalendarType.GENERIC) {
856 continue;
857 }
858 String prefix = calendarType.keyElementName();
859 for (String element : FORMAT_DATA_ELEMENTS) {
860 String key = prefix + element;
861 copyIfPresent(map, "java.time." + key, formatData);
862 copyIfPresent(map, key, formatData);
863 }
864 }
865
866 for (String key : map.keySet()) {
867 // Copy available calendar names
868 if (key.startsWith(CLDRConverter.LOCALE_TYPE_PREFIX_CA)) {
869 String type = key.substring(CLDRConverter.LOCALE_TYPE_PREFIX_CA.length());
870 for (CalendarType calendarType : CalendarType.values()) {
871 if (calendarType == CalendarType.GENERIC) {
872 continue;
873 }
874 if (type.equals(calendarType.lname())) {
875 Object value = map.get(key);
876 String dataKey = key.replace(LOCALE_TYPE_PREFIX_CA,
877 CALENDAR_NAME_PREFIX);
878 formatData.put(dataKey, value);
879 String ukey = CALENDAR_NAME_PREFIX + calendarType.uname();
880 if (!dataKey.equals(ukey)) {
881 formatData.put(ukey, value);
882 }
883 }
884 }
885 }
886 }
887
888 copyIfPresent(map, "DefaultNumberingSystem", formatData);
889
890 @SuppressWarnings("unchecked")
891 List<String> numberingScripts = (List<String>) map.remove("numberingScripts");
892 if (numberingScripts != null) {
893 for (String script : numberingScripts) {
894 copyIfPresent(map, script + "." + "NumberElements", formatData);
895 }
896 } else {
897 copyIfPresent(map, "NumberElements", formatData);
898 }
899 copyIfPresent(map, "NumberPatterns", formatData);
900 copyIfPresent(map, "short.CompactNumberPatterns", formatData);
901 copyIfPresent(map, "long.CompactNumberPatterns", formatData);
902
903 // put extra number elements for available scripts into formatData, if it is "root"
904 if (id.equals("root")) {
905 handlerNumbering.keySet().stream()
906 .filter(k -> !numberingScripts.contains(k))
907 .forEach(k -> {
908 String[] ne = (String[])map.get("latn.NumberElements");
909 String[] neNew = Arrays.copyOf(ne, ne.length);
910 neNew[4] = handlerNumbering.get(k).substring(0, 1);
911 formatData.put(k + ".NumberElements", neNew);
912 });
913 }
914 return formatData;
915 }
916
917 private static void copyIfPresent(Map<String, Object> src, String key, Map<String, Object> dest) {
918 Object value = src.get(key);
919 if (value != null) {
920 dest.put(key, value);
921 }
922 }
923
924 // --- code below here is adapted from java.util.Properties ---
925 private static final String specialSaveCharsJava = "\"";
926 private static final String specialSaveCharsProperties = "=: \t\r\n\f#!";
927
928 /*
929 * Converts unicodes to encoded \uxxxx
930 * and writes out any of the characters in specialSaveChars
931 * with a preceding slash
932 */
933 static String saveConvert(String theString, boolean useJava) {
934 if (theString == null) {
935 return "";
936 }
937
938 String specialSaveChars;
939 if (useJava) {
940 specialSaveChars = specialSaveCharsJava;
941 } else {
942 specialSaveChars = specialSaveCharsProperties;
943 }
944 boolean escapeSpace = false;
945
946 int len = theString.length();
947 StringBuilder outBuffer = new StringBuilder(len * 2);
948 Formatter formatter = new Formatter(outBuffer, Locale.ROOT);
949
950 for (int x = 0; x < len; x++) {
951 char aChar = theString.charAt(x);
952 switch (aChar) {
953 case ' ':
954 if (x == 0 || escapeSpace) {
955 outBuffer.append('\\');
956 }
957 outBuffer.append(' ');
958 break;
959 case '\\':
960 outBuffer.append('\\');
961 outBuffer.append('\\');
962 break;
963 case '\t':
964 outBuffer.append('\\');
965 outBuffer.append('t');
966 break;
967 case '\n':
968 outBuffer.append('\\');
969 outBuffer.append('n');
970 break;
971 case '\r':
972 outBuffer.append('\\');
973 outBuffer.append('r');
974 break;
975 case '\f':
976 outBuffer.append('\\');
977 outBuffer.append('f');
978 break;
979 default:
980 if (aChar < 0x0020 || (!USE_UTF8 && aChar > 0x007e)) {
981 formatter.format("\\u%04x", (int)aChar);
982 } else {
983 if (specialSaveChars.indexOf(aChar) != -1) {
984 outBuffer.append('\\');
985 }
986 outBuffer.append(aChar);
987 }
988 }
989 }
990 return outBuffer.toString();
991 }
992
993 private static String toLanguageTag(String locName) {
994 if (locName.indexOf('_') == -1) {
995 return locName;
996 }
997 String tag = locName.replaceAll("_", "-");
998 Locale loc = Locale.forLanguageTag(tag);
999 return loc.toLanguageTag();
1000 }
1001
1002 private static void addLikelySubtags(Map<String, SortedSet<String>> metaInfo, String category, String id) {
1003 String likelySubtag = handlerLikelySubtags.get(id);
1004 if (likelySubtag != null) {
1005 // Remove Script for now
1006 metaInfo.get(category).add(toLanguageTag(likelySubtag).replaceFirst("-[A-Z][a-z]{3}", ""));
1007 }
1008 }
1009
1010 private static String toLocaleName(String tag) {
1011 if (tag.indexOf('-') == -1) {
1012 return tag;
1013 }
1014 return tag.replaceAll("-", "_");
1015 }
1016
1017 private static void setupBaseLocales(String localeList) {
1018 Arrays.stream(localeList.split(","))
1019 .map(Locale::forLanguageTag)
1020 .map(l -> Control.getControl(Control.FORMAT_DEFAULT)
1021 .getCandidateLocales("", l))
1022 .forEach(BASE_LOCALES::addAll);
1023 }
1024
1025 // applying parent locale rules to the passed candidates list
1026 // This has to match with the one in sun.util.cldr.CLDRLocaleProviderAdapter
1027 private static Map<Locale, Locale> childToParentLocaleMap = null;
1028 private static List<Locale> applyParentLocales(String baseName, List<Locale> candidates) {
1029 if (Objects.isNull(childToParentLocaleMap)) {
1030 childToParentLocaleMap = new HashMap<>();
1031 parentLocalesMap.keySet().forEach(key -> {
1032 String parent = key.substring(PARENT_LOCALE_PREFIX.length()).replaceAll("_", "-");
1033 parentLocalesMap.get(key).stream().forEach(child -> {
1034 childToParentLocaleMap.put(Locale.forLanguageTag(child),
1035 "root".equals(parent) ? Locale.ROOT : Locale.forLanguageTag(parent));
1036 });
1037 });
1038 }
1039
1040 // check irregular parents
1041 for (int i = 0; i < candidates.size(); i++) {
1042 Locale l = candidates.get(i);
1043 Locale p = childToParentLocaleMap.get(l);
1044 if (!l.equals(Locale.ROOT) &&
1045 Objects.nonNull(p) &&
1046 !candidates.get(i+1).equals(p)) {
1047 List<Locale> applied = candidates.subList(0, i+1);
1048 applied.addAll(applyParentLocales(baseName, defCon.getCandidateLocales(baseName, p)));
1049 return applied;
1050 }
1051 }
1052
1053 return candidates;
1054 }
1055
1056 private static void generateZoneName() throws Exception {
1057 Files.createDirectories(Paths.get(DESTINATION_DIR, "java", "time", "format"));
1058 Files.write(Paths.get(DESTINATION_DIR, "java", "time", "format", "ZoneName.java"),
1059 Files.lines(Paths.get(zoneNameTempFile))
1060 .flatMap(l -> {
1061 if (l.equals("%%%%ZIDMAP%%%%")) {
1062 return zidMapEntry();
1063 } else if (l.equals("%%%%MZONEMAP%%%%")) {
1064 return handlerMetaZones.mzoneMapEntry();
1065 } else if (l.equals("%%%%DEPRECATED%%%%")) {
1066 return handlerSupplMeta.deprecatedMap();
1067 } else if (l.equals("%%%%TZDATALINK%%%%")) {
1068 return tzDataLinkEntry();
1069 } else {
1070 return Stream.of(l);
1071 }
1072 })
1073 .collect(Collectors.toList()),
1074 StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
1075 }
1076
1077 private static Stream<String> zidMapEntry() {
1078 return ZoneId.getAvailableZoneIds().stream()
1079 .map(id -> {
1080 String canonId = canonicalTZMap.getOrDefault(id, id);
1081 String meta = handlerMetaZones.get(canonId);
1082 String zone001 = handlerMetaZones.zidMap().get(meta);
1083 return zone001 == null ? "" :
1084 String.format(" \"%s\", \"%s\", \"%s\",",
1085 id, meta, zone001);
1086 })
1087 .filter(s -> !s.isEmpty())
1088 .sorted();
1089 }
1090
1091 private static Stream<String> tzDataLinkEntry() {
1092 try {
1093 return Files.walk(Paths.get(tzDataDir), 1)
1094 .filter(p -> !Files.isDirectory(p))
1095 .flatMap(CLDRConverter::extractLinks)
1096 .sorted();
1097 } catch (IOException e) {
1098 throw new UncheckedIOException(e);
1099 }
1100 }
1101
1102 private static Stream<String> extractLinks(Path tzFile) {
1103 try {
1104 return Files.lines(tzFile)
1105 .filter(l -> l.startsWith("Link"))
1106 .map(l -> l.replaceFirst("^Link[\\s]+(\\S+)\\s+(\\S+).*",
1107 " \"$2\", \"$1\","));
1108 } catch (IOException e) {
1109 throw new UncheckedIOException(e);
1110 }
1111 }
1112
1113 // Generate tzmappings for Windows. The format is:
1114 //
1115 // (Windows Zone Name):(REGION):(Java TZID)
1116 //
1117 // where:
1118 // Windows Zone Name: arbitrary time zone name string used in Windows
1119 // REGION: ISO3166 or UN M.49 code
1120 // Java TZID: Java's time zone ID
1121 //
1122 // Note: the entries are alphabetically sorted, *except* the "world" region
1123 // code, i.e., "001". It should be the last entry for the same windows time
1124 // zone name entries. (cf. TimeZone_md.c)
1125 private static void generateWindowsTZMappings() throws Exception {
1126 Files.createDirectories(Paths.get(DESTINATION_DIR, "windows", "conf"));
1127 Files.write(Paths.get(DESTINATION_DIR, "windows", "conf", "tzmappings"),
1128 handlerWinZones.keySet().stream()
1129 .map(k -> k + ":" + handlerWinZones.get(k) + ":")
1130 .sorted(new Comparator<String>() {
1131 public int compare(String t1, String t2) {
1132 String[] s1 = t1.split(":");
1133 String[] s2 = t2.split(":");
1134 if (s1[0].equals(s2[0])) {
1135 if (s1[1].equals("001")) {
1136 return 1;
1137 } else if (s2[1].equals("001")) {
1138 return -1;
1139 } else {
1140 return s1[1].compareTo(s2[1]);
1141 }
1142 } else {
1143 return s1[0].compareTo(s2[0]);
1144 }
1145 }
1146 })
1147 .collect(Collectors.toList()),
1148 StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
1149 }
1150 }