1 /*
2 * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25
26 package build.tools.cldrconverter;
27
28 import static build.tools.cldrconverter.Bundle.jreTimeZoneNames;
29 import build.tools.cldrconverter.BundleGenerator.BundleType;
30 import java.io.File;
31 import java.io.IOException;
32 import java.io.UncheckedIOException;
33 import java.nio.file.*;
34 import java.text.MessageFormat;
35 import java.time.*;
36 import java.util.*;
37 import java.util.ResourceBundle.Control;
38 import java.util.logging.Level;
39 import java.util.logging.Logger;
40 import java.util.stream.Collectors;
41 import java.util.stream.IntStream;
42 import java.util.stream.Stream;
43 import javax.xml.parsers.SAXParser;
44 import javax.xml.parsers.SAXParserFactory;
45 import org.xml.sax.SAXNotRecognizedException;
46 import org.xml.sax.SAXNotSupportedException;
47
48
49 /**
50 * Converts locale data from "Locale Data Markup Language" format to
51 * JRE resource bundle format. LDML is the format used by the Common
52 * Locale Data Repository maintained by the Unicode Consortium.
53 */
54 public class CLDRConverter {
55
56 static final String LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldml.dtd";
57 static final String SPPL_LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldmlSupplemental.dtd";
58 static final String BCP47_LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldmlBCP47.dtd";
59
60
61 private static String CLDR_BASE;
62 static String LOCAL_LDML_DTD;
63 static String LOCAL_SPPL_LDML_DTD;
64 static String LOCAL_BCP47_LDML_DTD;
65 private static String SOURCE_FILE_DIR;
66 private static String SPPL_SOURCE_FILE;
67 private static String SPPL_META_SOURCE_FILE;
68 private static String NUMBERING_SOURCE_FILE;
69 private static String METAZONES_SOURCE_FILE;
70 private static String LIKELYSUBTAGS_SOURCE_FILE;
71 private static String TIMEZONE_SOURCE_FILE;
72 static String DESTINATION_DIR = "build/gensrc";
73
74 static final String LOCALE_NAME_PREFIX = "locale.displayname.";
75 static final String LOCALE_SEPARATOR = LOCALE_NAME_PREFIX + "separator";
76 static final String LOCALE_KEYTYPE = LOCALE_NAME_PREFIX + "keytype";
77 static final String LOCALE_KEY_PREFIX = LOCALE_NAME_PREFIX + "key.";
78 static final String LOCALE_TYPE_PREFIX = LOCALE_NAME_PREFIX + "type.";
79 static final String LOCALE_TYPE_PREFIX_CA = LOCALE_TYPE_PREFIX + "ca.";
80 static final String CURRENCY_SYMBOL_PREFIX = "currency.symbol.";
81 static final String CURRENCY_NAME_PREFIX = "currency.displayname.";
82 static final String CALENDAR_NAME_PREFIX = "calendarname.";
83 static final String CALENDAR_FIRSTDAY_PREFIX = "firstDay.";
84 static final String CALENDAR_MINDAYS_PREFIX = "minDays.";
85 static final String TIMEZONE_ID_PREFIX = "timezone.id.";
86 static final String EXEMPLAR_CITY_PREFIX = "timezone.excity.";
87 static final String ZONE_NAME_PREFIX = "timezone.displayname.";
88 static final String METAZONE_ID_PREFIX = "metazone.id.";
89 static final String PARENT_LOCALE_PREFIX = "parentLocale.";
90 static final String[] EMPTY_ZONE = {"", "", "", "", "", ""};
91
92 private static SupplementDataParseHandler handlerSuppl;
93 private static LikelySubtagsParseHandler handlerLikelySubtags;
94 static SupplementalMetadataParseHandler handlerSupplMeta;
95 static NumberingSystemsParseHandler handlerNumbering;
96 static MetaZonesParseHandler handlerMetaZones;
97 static TimeZoneParseHandler handlerTimeZone;
98 private static BundleGenerator bundleGenerator;
99
100 // java.base module related
101 static boolean isBaseModule = false;
102 static final Set<Locale> BASE_LOCALES = new HashSet<>();
103
104 // "parentLocales" map
105 private static final Map<String, SortedSet<String>> parentLocalesMap = new HashMap<>();
106 private static final ResourceBundle.Control defCon =
107 ResourceBundle.Control.getControl(ResourceBundle.Control.FORMAT_DEFAULT);
108
109 private static final String[] AVAILABLE_TZIDS = TimeZone.getAvailableIDs();
110 private static String zoneNameTempFile;
111 private static String tzDataDir;
112 private static final Map<String, String> canonicalTZMap = new HashMap<>();
113
114 static enum DraftType {
115 UNCONFIRMED,
116 PROVISIONAL,
117 CONTRIBUTED,
118 APPROVED;
119
120 private static final Map<String, DraftType> map = new HashMap<>();
121 static {
122 for (DraftType dt : values()) {
123 map.put(dt.getKeyword(), dt);
124 }
125 }
126 static private DraftType defaultType = CONTRIBUTED;
127
128 private final String keyword;
129
130 private DraftType() {
131 keyword = this.name().toLowerCase(Locale.ROOT);
132
133 }
134
135 static DraftType forKeyword(String keyword) {
136 return map.get(keyword);
137 }
138
139 static DraftType getDefault() {
140 return defaultType;
141 }
142
143 static void setDefault(String keyword) {
144 defaultType = Objects.requireNonNull(forKeyword(keyword));
145 }
146
147 String getKeyword() {
148 return keyword;
149 }
150 }
151
152 static boolean USE_UTF8 = false;
153 private static boolean verbose;
154
155 private CLDRConverter() {
156 // no instantiation
157 }
158
159 @SuppressWarnings("AssignmentToForLoopParameter")
160 public static void main(String[] args) throws Exception {
161 if (args.length != 0) {
162 String currentArg = null;
163 try {
164 for (int i = 0; i < args.length; i++) {
165 currentArg = args[i];
166 switch (currentArg) {
167 case "-draft":
168 String draftDataType = args[++i];
169 try {
170 DraftType.setDefault(draftDataType);
171 } catch (NullPointerException e) {
172 severe("Error: incorrect draft value: %s%n", draftDataType);
173 System.exit(1);
174 }
175 info("Using the specified data type: %s%n", draftDataType);
176 break;
177
178 case "-base":
179 // base directory for input files
180 CLDR_BASE = args[++i];
181 if (!CLDR_BASE.endsWith("/")) {
182 CLDR_BASE += "/";
183 }
184 break;
185
186 case "-baselocales":
187 // base locales
188 setupBaseLocales(args[++i]);
189 break;
190
191 case "-basemodule":
192 // indicates java.base module resource generation
193 isBaseModule = true;
194 break;
195
196 case "-o":
197 // output directory
198 DESTINATION_DIR = args[++i];
199 break;
200
201 case "-utf8":
202 USE_UTF8 = true;
203 break;
204
205 case "-verbose":
206 verbose = true;
207 break;
208
209 case "-zntempfile":
210 zoneNameTempFile = args[++i];
211 break;
212
213 case "-tzdatadir":
214 tzDataDir = args[++i];
215 break;
216
217 case "-help":
218 usage();
219 System.exit(0);
220 break;
221
222 default:
223 throw new RuntimeException();
224 }
225 }
226 } catch (RuntimeException e) {
227 severe("unknown or imcomplete arg(s): " + currentArg);
228 usage();
229 System.exit(1);
230 }
231 }
232
233 // Set up path names
234 LOCAL_LDML_DTD = CLDR_BASE + "/dtd/ldml.dtd";
235 LOCAL_SPPL_LDML_DTD = CLDR_BASE + "/dtd/ldmlSupplemental.dtd";
236 LOCAL_BCP47_LDML_DTD = CLDR_BASE + "/dtd/ldmlBCP47.dtd";
237 SOURCE_FILE_DIR = CLDR_BASE + "/main";
238 SPPL_SOURCE_FILE = CLDR_BASE + "/supplemental/supplementalData.xml";
239 LIKELYSUBTAGS_SOURCE_FILE = CLDR_BASE + "/supplemental/likelySubtags.xml";
240 NUMBERING_SOURCE_FILE = CLDR_BASE + "/supplemental/numberingSystems.xml";
241 METAZONES_SOURCE_FILE = CLDR_BASE + "/supplemental/metaZones.xml";
242 TIMEZONE_SOURCE_FILE = CLDR_BASE + "/bcp47/timezone.xml";
243 SPPL_META_SOURCE_FILE = CLDR_BASE + "/supplemental/supplementalMetadata.xml";
244
245 if (BASE_LOCALES.isEmpty()) {
246 setupBaseLocales("en-US");
247 }
248
249 bundleGenerator = new ResourceBundleGenerator();
250
251 // Parse data independent of locales
252 parseSupplemental();
253 parseBCP47();
254
255 List<Bundle> bundles = readBundleList();
256 convertBundles(bundles);
257
258 // Generate java.time.format.ZoneName.java
259 if (isBaseModule) {
260 generateZoneName();
261 }
262 }
263
264 private static void usage() {
265 errout("Usage: java CLDRConverter [options]%n"
266 + "\t-help output this usage message and exit%n"
267 + "\t-verbose output information%n"
268 + "\t-draft [contributed | approved | provisional | unconfirmed]%n"
269 + "\t\t draft level for using data (default: contributed)%n"
270 + "\t-base dir base directory for CLDR input files%n"
271 + "\t-basemodule generates bundles that go into java.base module%n"
272 + "\t-baselocales loc(,loc)* locales that go into the base module%n"
273 + "\t-o dir output directory (default: ./build/gensrc)%n"
274 + "\t-zntempfile template file for java.time.format.ZoneName.java%n"
275 + "\t-tzdatadir tzdata directory for java.time.format.ZoneName.java%n"
276 + "\t-utf8 use UTF-8 rather than \\uxxxx (for debug)%n");
277 }
278
279 static void info(String fmt, Object... args) {
280 if (verbose) {
281 System.out.printf(fmt, args);
282 }
283 }
284
285 static void info(String msg) {
286 if (verbose) {
287 System.out.println(msg);
288 }
289 }
290
291 static void warning(String fmt, Object... args) {
292 System.err.print("Warning: ");
293 System.err.printf(fmt, args);
294 }
295
296 static void warning(String msg) {
297 System.err.print("Warning: ");
298 errout(msg);
299 }
300
301 static void severe(String fmt, Object... args) {
302 System.err.print("Error: ");
303 System.err.printf(fmt, args);
304 }
305
306 static void severe(String msg) {
307 System.err.print("Error: ");
308 errout(msg);
309 }
310
311 private static void errout(String msg) {
312 if (msg.contains("%n")) {
313 System.err.printf(msg);
314 } else {
315 System.err.println(msg);
316 }
317 }
318
319 /**
320 * Configure the parser to allow access to DTDs on the file system.
321 */
322 private static void enableFileAccess(SAXParser parser) throws SAXNotSupportedException {
323 try {
324 parser.setProperty("http://javax.xml.XMLConstants/property/accessExternalDTD", "file");
325 } catch (SAXNotRecognizedException ignore) {
326 // property requires >= JAXP 1.5
327 }
328 }
329
330 private static List<Bundle> readBundleList() throws Exception {
331 List<Bundle> retList = new ArrayList<>();
332 Path path = FileSystems.getDefault().getPath(SOURCE_FILE_DIR);
333 try (DirectoryStream<Path> dirStr = Files.newDirectoryStream(path)) {
334 for (Path entry : dirStr) {
335 String fileName = entry.getFileName().toString();
336 if (fileName.endsWith(".xml")) {
337 String id = fileName.substring(0, fileName.indexOf('.'));
338 Locale cldrLoc = Locale.forLanguageTag(toLanguageTag(id));
339 StringBuilder sb = getCandLocales(cldrLoc);
340 if (sb.indexOf("root") == -1) {
341 sb.append("root");
342 }
343 Bundle b = new Bundle(id, sb.toString(), null, null);
344 // Insert the bundle for root at the top so that it will get
345 // processed first.
346 if ("root".equals(id)) {
347 retList.add(0, b);
348 } else {
349 retList.add(b);
350 }
351 }
352 }
353 }
354 return retList;
355 }
356
357 private static final Map<String, Map<String, Object>> cldrBundles = new HashMap<>();
358
359 private static Map<String, SortedSet<String>> metaInfo = new HashMap<>();
360
361 static {
362 // For generating information on supported locales.
363 metaInfo.put("AvailableLocales", new TreeSet<>());
364 }
365
366 static Map<String, Object> getCLDRBundle(String id) throws Exception {
367 Map<String, Object> bundle = cldrBundles.get(id);
368 if (bundle != null) {
369 return bundle;
370 }
371 File file = new File(SOURCE_FILE_DIR + File.separator + id + ".xml");
372 if (!file.exists()) {
373 // Skip if the file doesn't exist.
374 return Collections.emptyMap();
375 }
376
377 info("..... main directory .....");
378 LDMLParseHandler handler = new LDMLParseHandler(id);
379 parseLDMLFile(file, handler);
380
381 bundle = handler.getData();
382 cldrBundles.put(id, bundle);
383
384 if (id.equals("root")) {
385 // Calendar data (firstDayOfWeek & minDaysInFirstWeek)
386 bundle = handlerSuppl.getData("root");
387 if (bundle != null) {
388 //merge two maps into one map
389 Map<String, Object> temp = cldrBundles.remove(id);
390 bundle.putAll(temp);
391 cldrBundles.put(id, bundle);
392 }
393 }
394 return bundle;
395 }
396
397 // Parsers for data in "supplemental" directory
398 //
399 private static void parseSupplemental() throws Exception {
400 // Parse SupplementalData file and store the information in the HashMap
401 // Calendar information such as firstDay and minDay are stored in
402 // supplementalData.xml as of CLDR1.4. Individual territory is listed
403 // with its ISO 3166 country code while default is listed using UNM49
404 // region and composition numerical code (001 for World.)
405 //
406 // SupplementalData file also provides the "parent" locales which
407 // are othrwise not to be fallen back. Process them here as well.
408 //
409 handlerSuppl = new SupplementDataParseHandler();
410 parseLDMLFile(new File(SPPL_SOURCE_FILE), handlerSuppl);
411 Map<String, Object> parentData = handlerSuppl.getData("root");
412 parentData.keySet().stream()
413 .filter(key -> key.startsWith(PARENT_LOCALE_PREFIX))
414 .forEach(key -> {
415 parentLocalesMap.put(key, new TreeSet(
416 Arrays.asList(((String)parentData.get(key)).split(" "))));
417 });
418
419 // Parse numberingSystems to get digit zero character information.
420 handlerNumbering = new NumberingSystemsParseHandler();
421 parseLDMLFile(new File(NUMBERING_SOURCE_FILE), handlerNumbering);
422
423 // Parse metaZones to create mappings between Olson tzids and CLDR meta zone names
424 handlerMetaZones = new MetaZonesParseHandler();
425 parseLDMLFile(new File(METAZONES_SOURCE_FILE), handlerMetaZones);
426
427 // Parse likelySubtags
428 handlerLikelySubtags = new LikelySubtagsParseHandler();
429 parseLDMLFile(new File(LIKELYSUBTAGS_SOURCE_FILE), handlerLikelySubtags);
430
431 // Parse supplementalMetadata
432 // Currently interested in deprecated time zone ids and language aliases.
433 handlerSupplMeta = new SupplementalMetadataParseHandler();
434 parseLDMLFile(new File(SPPL_META_SOURCE_FILE), handlerSupplMeta);
435 }
436
437 // Parsers for data in "bcp47" directory
438 //
439 private static void parseBCP47() throws Exception {
440 // Parse timezone
441 handlerTimeZone = new TimeZoneParseHandler();
442 parseLDMLFile(new File(TIMEZONE_SOURCE_FILE), handlerTimeZone);
443
444 // canonical tz name map
445 // alias -> primary
446 handlerTimeZone.getData().forEach((k, v) -> {
447 String[] ids = ((String)v).split("\\s");
448 for (int i = 1; i < ids.length; i++) {
449 canonicalTZMap.put(ids[i], ids[0]);
450 }
451 });
452 }
453
454 private static void parseLDMLFile(File srcfile, AbstractLDMLHandler handler) throws Exception {
455 info("..... Parsing " + srcfile.getName() + " .....");
456 SAXParserFactory pf = SAXParserFactory.newInstance();
457 pf.setValidating(true);
458 SAXParser parser = pf.newSAXParser();
459 enableFileAccess(parser);
460 parser.parse(srcfile, handler);
461 }
462
463 private static StringBuilder getCandLocales(Locale cldrLoc) {
464 List<Locale> candList = getCandidateLocales(cldrLoc);
465 StringBuilder sb = new StringBuilder();
466 for (Locale loc : candList) {
467 if (!loc.equals(Locale.ROOT)) {
468 sb.append(toLocaleName(loc.toLanguageTag()));
469 sb.append(",");
470 }
471 }
472 return sb;
473 }
474
475 private static List<Locale> getCandidateLocales(Locale cldrLoc) {
476 List<Locale> candList = new ArrayList<>();
477 candList = applyParentLocales("", defCon.getCandidateLocales("", cldrLoc));
478 return candList;
479 }
480
481 private static void convertBundles(List<Bundle> bundles) throws Exception {
482 // parent locales map. The mappings are put in base metaInfo file
483 // for now.
484 if (isBaseModule) {
485 metaInfo.putAll(parentLocalesMap);
486 }
487
488 for (Bundle bundle : bundles) {
489 // Get the target map, which contains all the data that should be
490 // visible for the bundle's locale
491
492 Map<String, Object> targetMap = bundle.getTargetMap();
493
494 EnumSet<Bundle.Type> bundleTypes = bundle.getBundleTypes();
495
496 if (bundle.isRoot()) {
497 // Add DateTimePatternChars because CLDR no longer supports localized patterns.
498 targetMap.put("DateTimePatternChars", "GyMdkHmsSEDFwWahKzZ");
499 }
500
501 // Now the map contains just the entries that need to be in the resources bundles.
502 // Go ahead and generate them.
503 if (bundleTypes.contains(Bundle.Type.LOCALENAMES)) {
504 Map<String, Object> localeNamesMap = extractLocaleNames(targetMap, bundle.getID());
505 if (!localeNamesMap.isEmpty() || bundle.isRoot()) {
506 bundleGenerator.generateBundle("util", "LocaleNames", bundle.getJavaID(), true, localeNamesMap, BundleType.OPEN);
507 }
508 }
509 if (bundleTypes.contains(Bundle.Type.CURRENCYNAMES)) {
510 Map<String, Object> currencyNamesMap = extractCurrencyNames(targetMap, bundle.getID(), bundle.getCurrencies());
511 if (!currencyNamesMap.isEmpty() || bundle.isRoot()) {
512 bundleGenerator.generateBundle("util", "CurrencyNames", bundle.getJavaID(), true, currencyNamesMap, BundleType.OPEN);
513 }
514 }
515 if (bundleTypes.contains(Bundle.Type.TIMEZONENAMES)) {
516 Map<String, Object> zoneNamesMap = extractZoneNames(targetMap, bundle.getID());
517 if (!zoneNamesMap.isEmpty() || bundle.isRoot()) {
518 bundleGenerator.generateBundle("util", "TimeZoneNames", bundle.getJavaID(), true, zoneNamesMap, BundleType.TIMEZONE);
519 }
520 }
521 if (bundleTypes.contains(Bundle.Type.CALENDARDATA)) {
522 Map<String, Object> calendarDataMap = extractCalendarData(targetMap, bundle.getID());
523 if (!calendarDataMap.isEmpty() || bundle.isRoot()) {
524 bundleGenerator.generateBundle("util", "CalendarData", bundle.getJavaID(), true, calendarDataMap, BundleType.PLAIN);
525 }
526 }
527 if (bundleTypes.contains(Bundle.Type.FORMATDATA)) {
528 Map<String, Object> formatDataMap = extractFormatData(targetMap, bundle.getID());
529 if (!formatDataMap.isEmpty() || bundle.isRoot()) {
530 bundleGenerator.generateBundle("text", "FormatData", bundle.getJavaID(), true, formatDataMap, BundleType.PLAIN);
531 }
532 }
533
534 // For AvailableLocales
535 metaInfo.get("AvailableLocales").add(toLanguageTag(bundle.getID()));
536 addLikelySubtags(metaInfo, "AvailableLocales", bundle.getID());
537 }
538 bundleGenerator.generateMetaInfo(metaInfo);
539 }
540
541 static final Map<String, String> aliases = new HashMap<>();
542
543 /**
544 * Translate the aliases into the real entries in the bundle map.
545 */
546 static void handleAliases(Map<String, Object> bundleMap) {
547 Set bundleKeys = bundleMap.keySet();
548 try {
549 for (String key : aliases.keySet()) {
550 String targetKey = aliases.get(key);
551 if (bundleKeys.contains(targetKey)) {
552 bundleMap.putIfAbsent(key, bundleMap.get(targetKey));
553 }
554 }
555 } catch (Exception ex) {
556 Logger.getLogger(CLDRConverter.class.getName()).log(Level.SEVERE, null, ex);
557 }
558 }
559
560 /*
561 * Returns the language portion of the given id.
562 * If id is "root", "" is returned.
563 */
564 static String getLanguageCode(String id) {
565 return "root".equals(id) ? "" : Locale.forLanguageTag(id.replaceAll("_", "-")).getLanguage();
566 }
567
568 /**
569 * Examine if the id includes the country (territory) code. If it does, it returns
570 * the country code.
571 * Otherwise, it returns null. eg. when the id is "zh_Hans_SG", it return "SG".
572 * It does NOT return UN M.49 code, e.g., '001', as those three digit numbers cannot
573 * be translated into package names.
574 */
575 static String getCountryCode(String id) {
576 String rgn = getRegionCode(id);
577 return rgn.length() == 2 ? rgn: null;
578 }
579
580 /**
581 * Examine if the id includes the region code. If it does, it returns
582 * the region code.
583 * Otherwise, it returns null. eg. when the id is "zh_Hans_SG", it return "SG".
584 * It DOES return UN M.49 code, e.g., '001', as well as ISO 3166 two letter country codes.
585 */
586 static String getRegionCode(String id) {
587 return Locale.forLanguageTag(id.replaceAll("_", "-")).getCountry();
588 }
589
590 private static class KeyComparator implements Comparator<String> {
591 static KeyComparator INSTANCE = new KeyComparator();
592
593 private KeyComparator() {
594 }
595
596 @Override
597 public int compare(String o1, String o2) {
598 int len1 = o1.length();
599 int len2 = o2.length();
600 if (!isDigit(o1.charAt(0)) && !isDigit(o2.charAt(0))) {
601 // Shorter string comes first unless either starts with a digit.
602 if (len1 < len2) {
603 return -1;
604 }
605 if (len1 > len2) {
606 return 1;
607 }
608 }
609 return o1.compareTo(o2);
610 }
611
612 private boolean isDigit(char c) {
613 return c >= '0' && c <= '9';
614 }
615 }
616
617 private static Map<String, Object> extractLocaleNames(Map<String, Object> map, String id) {
618 Map<String, Object> localeNames = new TreeMap<>(KeyComparator.INSTANCE);
619 for (String key : map.keySet()) {
620 if (key.startsWith(LOCALE_NAME_PREFIX)) {
621 switch (key) {
622 case LOCALE_SEPARATOR:
623 localeNames.put("ListCompositionPattern", map.get(key));
624 break;
625 case LOCALE_KEYTYPE:
626 localeNames.put("ListKeyTypePattern", map.get(key));
627 break;
628 default:
629 localeNames.put(key.substring(LOCALE_NAME_PREFIX.length()), map.get(key));
630 break;
631 }
632 }
633 }
634
635 if (id.equals("root")) {
636 // Add display name pattern, which is not in CLDR
637 localeNames.put("DisplayNamePattern", "{0,choice,0#|1#{1}|2#{1} ({2})}");
638 }
639
640 return localeNames;
641 }
642
643 @SuppressWarnings("AssignmentToForLoopParameter")
644 private static Map<String, Object> extractCurrencyNames(Map<String, Object> map, String id, String names)
645 throws Exception {
646 Map<String, Object> currencyNames = new TreeMap<>(KeyComparator.INSTANCE);
647 for (String key : map.keySet()) {
648 if (key.startsWith(CURRENCY_NAME_PREFIX)) {
649 currencyNames.put(key.substring(CURRENCY_NAME_PREFIX.length()), map.get(key));
650 } else if (key.startsWith(CURRENCY_SYMBOL_PREFIX)) {
651 currencyNames.put(key.substring(CURRENCY_SYMBOL_PREFIX.length()), map.get(key));
652 }
653 }
654 return currencyNames;
655 }
656
657 private static Map<String, Object> extractZoneNames(Map<String, Object> map, String id) {
658 Map<String, Object> names = new HashMap<>();
659
660 // Copy over missing time zone ids from JRE for English locale
661 if (id.equals("en")) {
662 Map<String[], String> jreMetaMap = new HashMap<>();
663 jreTimeZoneNames.stream().forEach(e -> {
664 String tzid = (String)e[0];
665 String[] data = (String[])e[1];
666
667 if (map.get(TIMEZONE_ID_PREFIX + tzid) == null &&
668 handlerMetaZones.get(tzid) == null ||
669 handlerMetaZones.get(tzid) != null &&
670 map.get(METAZONE_ID_PREFIX + handlerMetaZones.get(tzid)) == null) {
671
672 // First, check the alias
673 String canonID = canonicalTZMap.get(tzid);
674 if (canonID != null && !tzid.equals(canonID)) {
675 Object value = map.get(TIMEZONE_ID_PREFIX + canonID);
676 if (value != null) {
677 names.put(tzid, value);
678 return;
679 } else {
680 String meta = handlerMetaZones.get(canonID);
681 if (meta != null) {
682 value = map.get(METAZONE_ID_PREFIX + meta);
683 if (value != null) {
684 names.put(tzid, meta);
685 return;
686 }
687 }
688 }
689 }
690
691 // Check the CLDR meta key
692 Optional<Map.Entry<String, String>> cldrMeta =
693 handlerMetaZones.getData().entrySet().stream()
694 .filter(me ->
695 Arrays.deepEquals(data,
696 (String[])map.get(METAZONE_ID_PREFIX + me.getValue())))
697 .findAny();
698 cldrMeta.ifPresentOrElse(meta -> names.put(tzid, meta.getValue()), () -> {
699 // Check the JRE meta key, add if there is not.
700 Optional<Map.Entry<String[], String>> jreMeta =
701 jreMetaMap.entrySet().stream()
702 .filter(jm -> Arrays.deepEquals(data, jm.getKey()))
703 .findAny();
704 jreMeta.ifPresentOrElse(meta -> names.put(tzid, meta.getValue()), () -> {
705 String metaName = "JRE_" + tzid.replaceAll("[/-]", "_");
706 names.put(METAZONE_ID_PREFIX + metaName, data);
707 names.put(tzid, metaName);
708 });
709 });
710 }
711 });
712 }
713
714 Arrays.stream(AVAILABLE_TZIDS).forEach(tzid -> {
715 // If the tzid is deprecated, get the data for the replacement id
716 String tzKey = Optional.ofNullable((String)handlerSupplMeta.get(tzid))
717 .orElse(tzid);
718 Object data = map.get(TIMEZONE_ID_PREFIX + tzKey);
719
720 if (data instanceof String[]) {
721 names.put(tzid, data);
722 } else {
723 String meta = handlerMetaZones.get(tzKey);
724 if (meta != null) {
725 String metaKey = METAZONE_ID_PREFIX + meta;
726 data = map.get(metaKey);
727 if (data instanceof String[]) {
728 // Keep the metazone prefix here.
729 names.put(metaKey, data);
730 names.put(tzid, meta);
731 }
732 }
733 }
734 });
735
736 // exemplar cities.
737 Map<String, Object> exCities = map.entrySet().stream()
738 .filter(e -> e.getKey().startsWith(CLDRConverter.EXEMPLAR_CITY_PREFIX))
739 .collect(Collectors
740 .toMap(Map.Entry::getKey, Map.Entry::getValue));
741 names.putAll(exCities);
742
743 if (!id.equals("en") &&
744 !names.isEmpty()) {
745 // CLDR does not have UTC entry, so add it here.
746 names.put("UTC", EMPTY_ZONE);
747
748 // no metazone zones
749 Arrays.asList(handlerMetaZones.get(MetaZonesParseHandler.NO_METAZONE_KEY)
750 .split("\\s")).stream()
751 .forEach(tz -> {
752 names.put(tz, EMPTY_ZONE);
753 });
754 }
755
756 return names;
757 }
758
759 /**
760 * Extracts the language independent calendar data. Each of the two keys,
761 * "firstDayOfWeek" and "minimalDaysInFirstWeek" has a string value consists of
762 * one or multiple occurrences of:
763 * i: rg1 rg2 ... rgn;
764 * where "i" is the data for the following regions (delimited by a space) after
765 * ":", and ends with a ";".
766 */
767 private static Map<String, Object> extractCalendarData(Map<String, Object> map, String id) {
768 Map<String, Object> calendarData = new LinkedHashMap<>();
769 if (id.equals("root")) {
770 calendarData.put("firstDayOfWeek",
771 IntStream.range(1, 8)
772 .mapToObj(String::valueOf)
773 .filter(d -> map.keySet().contains(CALENDAR_FIRSTDAY_PREFIX + d))
774 .map(d -> d + ": " + map.get(CALENDAR_FIRSTDAY_PREFIX + d))
775 .collect(Collectors.joining(";")));
776 calendarData.put("minimalDaysInFirstWeek",
777 IntStream.range(0, 7)
778 .mapToObj(String::valueOf)
779 .filter(d -> map.keySet().contains(CALENDAR_MINDAYS_PREFIX + d))
780 .map(d -> d + ": " + map.get(CALENDAR_MINDAYS_PREFIX + d))
781 .collect(Collectors.joining(";")));
782 }
783 return calendarData;
784 }
785
786 static final String[] FORMAT_DATA_ELEMENTS = {
787 "MonthNames",
788 "standalone.MonthNames",
789 "MonthAbbreviations",
790 "standalone.MonthAbbreviations",
791 "MonthNarrows",
792 "standalone.MonthNarrows",
793 "DayNames",
794 "standalone.DayNames",
795 "DayAbbreviations",
796 "standalone.DayAbbreviations",
797 "DayNarrows",
798 "standalone.DayNarrows",
799 "QuarterNames",
800 "standalone.QuarterNames",
801 "QuarterAbbreviations",
802 "standalone.QuarterAbbreviations",
803 "QuarterNarrows",
804 "standalone.QuarterNarrows",
805 "AmPmMarkers",
806 "narrow.AmPmMarkers",
807 "abbreviated.AmPmMarkers",
808 "long.Eras",
809 "Eras",
810 "narrow.Eras",
811 "field.era",
812 "field.year",
813 "field.month",
814 "field.week",
815 "field.weekday",
816 "field.dayperiod",
817 "field.hour",
818 "timezone.hourFormat",
819 "timezone.gmtFormat",
820 "timezone.gmtZeroFormat",
821 "timezone.regionFormat",
822 "timezone.regionFormat.daylight",
823 "timezone.regionFormat.standard",
824 "field.minute",
825 "field.second",
826 "field.zone",
827 "TimePatterns",
828 "DatePatterns",
829 "DateTimePatterns",
830 "DateTimePatternChars"
831 };
832
833 private static Map<String, Object> extractFormatData(Map<String, Object> map, String id) {
834 Map<String, Object> formatData = new LinkedHashMap<>();
835 for (CalendarType calendarType : CalendarType.values()) {
836 if (calendarType == CalendarType.GENERIC) {
837 continue;
838 }
839 String prefix = calendarType.keyElementName();
840 for (String element : FORMAT_DATA_ELEMENTS) {
841 String key = prefix + element;
842 copyIfPresent(map, "java.time." + key, formatData);
843 copyIfPresent(map, key, formatData);
844 }
845 }
846
847 for (String key : map.keySet()) {
848 // Copy available calendar names
849 if (key.startsWith(CLDRConverter.LOCALE_TYPE_PREFIX_CA)) {
850 String type = key.substring(CLDRConverter.LOCALE_TYPE_PREFIX_CA.length());
851 for (CalendarType calendarType : CalendarType.values()) {
852 if (calendarType == CalendarType.GENERIC) {
853 continue;
854 }
855 if (type.equals(calendarType.lname())) {
856 Object value = map.get(key);
857 String dataKey = key.replace(LOCALE_TYPE_PREFIX_CA,
858 CALENDAR_NAME_PREFIX);
859 formatData.put(dataKey, value);
860 String ukey = CALENDAR_NAME_PREFIX + calendarType.uname();
861 if (!dataKey.equals(ukey)) {
862 formatData.put(ukey, value);
863 }
864 }
865 }
866 }
867 }
868
869 copyIfPresent(map, "DefaultNumberingSystem", formatData);
870
871 @SuppressWarnings("unchecked")
872 List<String> numberingScripts = (List<String>) map.remove("numberingScripts");
873 if (numberingScripts != null) {
874 for (String script : numberingScripts) {
875 copyIfPresent(map, script + "." + "NumberElements", formatData);
876 }
877 } else {
878 copyIfPresent(map, "NumberElements", formatData);
879 }
880 copyIfPresent(map, "NumberPatterns", formatData);
881
882 // put extra number elements for available scripts into formatData, if it is "root"
883 if (id.equals("root")) {
884 handlerNumbering.keySet().stream()
885 .filter(k -> !numberingScripts.contains(k))
886 .forEach(k -> {
887 String[] ne = (String[])map.get("latn.NumberElements");
888 String[] neNew = Arrays.copyOf(ne, ne.length);
889 neNew[4] = handlerNumbering.get(k).substring(0, 1);
890 formatData.put(k + ".NumberElements", neNew);
891 });
892 }
893 return formatData;
894 }
895
896 private static void copyIfPresent(Map<String, Object> src, String key, Map<String, Object> dest) {
897 Object value = src.get(key);
898 if (value != null) {
899 dest.put(key, value);
900 }
901 }
902
903 // --- code below here is adapted from java.util.Properties ---
904 private static final String specialSaveCharsJava = "\"";
905 private static final String specialSaveCharsProperties = "=: \t\r\n\f#!";
906
907 /*
908 * Converts unicodes to encoded \uxxxx
909 * and writes out any of the characters in specialSaveChars
910 * with a preceding slash
911 */
912 static String saveConvert(String theString, boolean useJava) {
913 if (theString == null) {
914 return "";
915 }
916
917 String specialSaveChars;
918 if (useJava) {
919 specialSaveChars = specialSaveCharsJava;
920 } else {
921 specialSaveChars = specialSaveCharsProperties;
922 }
923 boolean escapeSpace = false;
924
925 int len = theString.length();
926 StringBuilder outBuffer = new StringBuilder(len * 2);
927 Formatter formatter = new Formatter(outBuffer, Locale.ROOT);
928
929 for (int x = 0; x < len; x++) {
930 char aChar = theString.charAt(x);
931 switch (aChar) {
932 case ' ':
933 if (x == 0 || escapeSpace) {
934 outBuffer.append('\\');
935 }
936 outBuffer.append(' ');
937 break;
938 case '\\':
939 outBuffer.append('\\');
940 outBuffer.append('\\');
941 break;
942 case '\t':
943 outBuffer.append('\\');
944 outBuffer.append('t');
945 break;
946 case '\n':
947 outBuffer.append('\\');
948 outBuffer.append('n');
949 break;
950 case '\r':
951 outBuffer.append('\\');
952 outBuffer.append('r');
953 break;
954 case '\f':
955 outBuffer.append('\\');
956 outBuffer.append('f');
957 break;
958 default:
959 if (aChar < 0x0020 || (!USE_UTF8 && aChar > 0x007e)) {
960 formatter.format("\\u%04x", (int)aChar);
961 } else {
962 if (specialSaveChars.indexOf(aChar) != -1) {
963 outBuffer.append('\\');
964 }
965 outBuffer.append(aChar);
966 }
967 }
968 }
969 return outBuffer.toString();
970 }
971
972 private static String toLanguageTag(String locName) {
973 if (locName.indexOf('_') == -1) {
974 return locName;
975 }
976 String tag = locName.replaceAll("_", "-");
977 Locale loc = Locale.forLanguageTag(tag);
978 return loc.toLanguageTag();
979 }
980
981 private static void addLikelySubtags(Map<String, SortedSet<String>> metaInfo, String category, String id) {
982 String likelySubtag = handlerLikelySubtags.get(id);
983 if (likelySubtag != null) {
984 // Remove Script for now
985 metaInfo.get(category).add(toLanguageTag(likelySubtag).replaceFirst("-[A-Z][a-z]{3}", ""));
986 }
987 }
988
989 private static String toLocaleName(String tag) {
990 if (tag.indexOf('-') == -1) {
991 return tag;
992 }
993 return tag.replaceAll("-", "_");
994 }
995
996 private static void setupBaseLocales(String localeList) {
997 Arrays.stream(localeList.split(","))
998 .map(Locale::forLanguageTag)
999 .map(l -> Control.getControl(Control.FORMAT_DEFAULT)
1000 .getCandidateLocales("", l))
1001 .forEach(BASE_LOCALES::addAll);
1002 }
1003
1004 // applying parent locale rules to the passed candidates list
1005 // This has to match with the one in sun.util.cldr.CLDRLocaleProviderAdapter
1006 private static Map<Locale, Locale> childToParentLocaleMap = null;
1007 private static List<Locale> applyParentLocales(String baseName, List<Locale> candidates) {
1008 if (Objects.isNull(childToParentLocaleMap)) {
1009 childToParentLocaleMap = new HashMap<>();
1010 parentLocalesMap.keySet().forEach(key -> {
1011 String parent = key.substring(PARENT_LOCALE_PREFIX.length()).replaceAll("_", "-");
1012 parentLocalesMap.get(key).stream().forEach(child -> {
1013 childToParentLocaleMap.put(Locale.forLanguageTag(child),
1014 "root".equals(parent) ? Locale.ROOT : Locale.forLanguageTag(parent));
1015 });
1016 });
1017 }
1018
1019 // check irregular parents
1020 for (int i = 0; i < candidates.size(); i++) {
1021 Locale l = candidates.get(i);
1022 Locale p = childToParentLocaleMap.get(l);
1023 if (!l.equals(Locale.ROOT) &&
1024 Objects.nonNull(p) &&
1025 !candidates.get(i+1).equals(p)) {
1026 List<Locale> applied = candidates.subList(0, i+1);
1027 applied.addAll(applyParentLocales(baseName, defCon.getCandidateLocales(baseName, p)));
1028 return applied;
1029 }
1030 }
1031
1032 return candidates;
1033 }
1034
1035 private static void generateZoneName() throws Exception {
1036 Files.createDirectories(Paths.get(DESTINATION_DIR, "java", "time", "format"));
1037 Files.write(Paths.get(DESTINATION_DIR, "java", "time", "format", "ZoneName.java"),
1038 Files.lines(Paths.get(zoneNameTempFile))
1039 .flatMap(l -> {
1040 if (l.equals("%%%%ZIDMAP%%%%")) {
1041 return zidMapEntry();
1042 } else if (l.equals("%%%%MZONEMAP%%%%")) {
1043 return handlerMetaZones.mzoneMapEntry();
1044 } else if (l.equals("%%%%DEPRECATED%%%%")) {
1045 return handlerSupplMeta.deprecatedMap();
1046 } else if (l.equals("%%%%TZDATALINK%%%%")) {
1047 return tzDataLinkEntry();
1048 } else {
1049 return Stream.of(l);
1050 }
1051 })
1052 .collect(Collectors.toList()),
1053 StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
1054 }
1055
1056 private static Stream<String> zidMapEntry() {
1057 return ZoneId.getAvailableZoneIds().stream()
1058 .map(id -> {
1059 String canonId = canonicalTZMap.getOrDefault(id, id);
1060 String meta = handlerMetaZones.get(canonId);
1061 String zone001 = handlerMetaZones.zidMap().get(meta);
1062 return zone001 == null ? "" :
1063 String.format(" \"%s\", \"%s\", \"%s\",",
1064 id, meta, zone001);
1065 })
1066 .filter(s -> !s.isEmpty())
1067 .sorted();
1068 }
1069
1070 private static Stream<String> tzDataLinkEntry() {
1071 try {
1072 return Files.walk(Paths.get(tzDataDir), 1)
1073 .filter(p -> !Files.isDirectory(p))
1074 .flatMap(CLDRConverter::extractLinks)
1075 .sorted();
1076 } catch (IOException e) {
1077 throw new UncheckedIOException(e);
1078 }
1079 }
1080
1081 private static Stream<String> extractLinks(Path tzFile) {
1082 try {
1083 return Files.lines(tzFile)
1084 .filter(l -> l.startsWith("Link"))
1085 .map(l -> l.replaceFirst("^Link[\\s]+(\\S+)\\s+(\\S+).*",
1086 " \"$2\", \"$1\","));
1087 } catch (IOException e) {
1088 throw new UncheckedIOException(e);
1089 }
1090 }
1091 }