1 /*
2 * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25
26 package build.tools.cldrconverter;
27
28 import static build.tools.cldrconverter.Bundle.jreTimeZoneNames;
29 import build.tools.cldrconverter.BundleGenerator.BundleType;
30 import java.io.File;
31 import java.nio.file.*;
32 import java.time.*;
33 import java.util.*;
34 import java.util.ResourceBundle.Control;
35 import java.util.logging.Level;
36 import java.util.logging.Logger;
37 import java.util.stream.Collectors;
38 import java.util.stream.IntStream;
39 import java.util.stream.Stream;
40 import javax.xml.parsers.SAXParser;
41 import javax.xml.parsers.SAXParserFactory;
42 import org.xml.sax.SAXNotRecognizedException;
43 import org.xml.sax.SAXNotSupportedException;
44
45
46 /**
47 * Converts locale data from "Locale Data Markup Language" format to
48 * JRE resource bundle format. LDML is the format used by the Common
49 * Locale Data Repository maintained by the Unicode Consortium.
50 */
51 public class CLDRConverter {
52
53 static final String LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldml.dtd";
54 static final String SPPL_LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldmlSupplemental.dtd";
55 static final String BCP47_LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldmlBCP47.dtd";
56
57
58 private static String CLDR_BASE;
59 static String LOCAL_LDML_DTD;
60 static String LOCAL_SPPL_LDML_DTD;
61 static String LOCAL_BCP47_LDML_DTD;
62 private static String SOURCE_FILE_DIR;
63 private static String SPPL_SOURCE_FILE;
64 private static String SPPL_META_SOURCE_FILE;
65 private static String NUMBERING_SOURCE_FILE;
66 private static String METAZONES_SOURCE_FILE;
67 private static String LIKELYSUBTAGS_SOURCE_FILE;
68 private static String TIMEZONE_SOURCE_FILE;
69 static String DESTINATION_DIR = "build/gensrc";
70
71 static final String LOCALE_NAME_PREFIX = "locale.displayname.";
72 static final String LOCALE_SEPARATOR = LOCALE_NAME_PREFIX + "separator";
73 static final String LOCALE_KEYTYPE = LOCALE_NAME_PREFIX + "keytype";
74 static final String LOCALE_KEY_PREFIX = LOCALE_NAME_PREFIX + "key.";
75 static final String LOCALE_TYPE_PREFIX = LOCALE_NAME_PREFIX + "type.";
76 static final String LOCALE_TYPE_PREFIX_CA = LOCALE_TYPE_PREFIX + "ca.";
77 static final String CURRENCY_SYMBOL_PREFIX = "currency.symbol.";
78 static final String CURRENCY_NAME_PREFIX = "currency.displayname.";
79 static final String CALENDAR_NAME_PREFIX = "calendarname.";
80 static final String CALENDAR_FIRSTDAY_PREFIX = "firstDay.";
81 static final String CALENDAR_MINDAYS_PREFIX = "minDays.";
82 static final String TIMEZONE_ID_PREFIX = "timezone.id.";
83 static final String ZONE_NAME_PREFIX = "timezone.displayname.";
84 static final String METAZONE_ID_PREFIX = "metazone.id.";
85 static final String PARENT_LOCALE_PREFIX = "parentLocale.";
86
87 private static SupplementDataParseHandler handlerSuppl;
88 private static SupplementalMetadataParseHandler handlerSupplMeta;
89 private static LikelySubtagsParseHandler handlerLikelySubtags;
90 static NumberingSystemsParseHandler handlerNumbering;
91 static MetaZonesParseHandler handlerMetaZones;
92 static TimeZoneParseHandler handlerTimeZone;
93 private static BundleGenerator bundleGenerator;
94
95 // java.base module related
96 static boolean isBaseModule = false;
97 static final Set<Locale> BASE_LOCALES = new HashSet<>();
98
99 // "parentLocales" map
100 private static final Map<String, SortedSet<String>> parentLocalesMap = new HashMap<>();
101 private static final ResourceBundle.Control defCon =
102 ResourceBundle.Control.getControl(ResourceBundle.Control.FORMAT_DEFAULT);
103
104 private static final String[] AVAILABLE_TZIDS = TimeZone.getAvailableIDs();
105 private static String zoneNameTempFile;
106
107 static enum DraftType {
108 UNCONFIRMED,
109 PROVISIONAL,
110 CONTRIBUTED,
111 APPROVED;
112
113 private static final Map<String, DraftType> map = new HashMap<>();
114 static {
115 for (DraftType dt : values()) {
116 map.put(dt.getKeyword(), dt);
117 }
118 }
119 static private DraftType defaultType = CONTRIBUTED;
120
121 private final String keyword;
122
123 private DraftType() {
124 keyword = this.name().toLowerCase(Locale.ROOT);
125
126 }
127
128 static DraftType forKeyword(String keyword) {
129 return map.get(keyword);
130 }
131
132 static DraftType getDefault() {
133 return defaultType;
134 }
135
136 static void setDefault(String keyword) {
137 defaultType = Objects.requireNonNull(forKeyword(keyword));
138 }
139
140 String getKeyword() {
141 return keyword;
142 }
143 }
144
145 static boolean USE_UTF8 = false;
146 private static boolean verbose;
147
148 private CLDRConverter() {
149 // no instantiation
150 }
151
152 @SuppressWarnings("AssignmentToForLoopParameter")
153 public static void main(String[] args) throws Exception {
154 if (args.length != 0) {
155 String currentArg = null;
156 try {
157 for (int i = 0; i < args.length; i++) {
158 currentArg = args[i];
159 switch (currentArg) {
160 case "-draft":
161 String draftDataType = args[++i];
162 try {
163 DraftType.setDefault(draftDataType);
164 } catch (NullPointerException e) {
165 severe("Error: incorrect draft value: %s%n", draftDataType);
166 System.exit(1);
167 }
168 info("Using the specified data type: %s%n", draftDataType);
169 break;
170
171 case "-base":
172 // base directory for input files
173 CLDR_BASE = args[++i];
174 if (!CLDR_BASE.endsWith("/")) {
175 CLDR_BASE += "/";
176 }
177 break;
178
179 case "-baselocales":
180 // base locales
181 setupBaseLocales(args[++i]);
182 break;
183
184 case "-basemodule":
185 // indicates java.base module resource generation
186 isBaseModule = true;
187 break;
188
189 case "-o":
190 // output directory
191 DESTINATION_DIR = args[++i];
192 break;
193
194 case "-utf8":
195 USE_UTF8 = true;
196 break;
197
198 case "-verbose":
199 verbose = true;
200 break;
201
202 case "-zntempfile":
203 zoneNameTempFile = args[++i];
204 break;
205
206 case "-help":
207 usage();
208 System.exit(0);
209 break;
210
211 default:
212 throw new RuntimeException();
213 }
214 }
215 } catch (RuntimeException e) {
216 severe("unknown or imcomplete arg(s): " + currentArg);
217 usage();
218 System.exit(1);
219 }
220 }
221
222 // Set up path names
223 LOCAL_LDML_DTD = CLDR_BASE + "/dtd/ldml.dtd";
224 LOCAL_SPPL_LDML_DTD = CLDR_BASE + "/dtd/ldmlSupplemental.dtd";
225 LOCAL_BCP47_LDML_DTD = CLDR_BASE + "/dtd/ldmlBCP47.dtd";
226 SOURCE_FILE_DIR = CLDR_BASE + "/main";
227 SPPL_SOURCE_FILE = CLDR_BASE + "/supplemental/supplementalData.xml";
228 LIKELYSUBTAGS_SOURCE_FILE = CLDR_BASE + "/supplemental/likelySubtags.xml";
229 NUMBERING_SOURCE_FILE = CLDR_BASE + "/supplemental/numberingSystems.xml";
230 METAZONES_SOURCE_FILE = CLDR_BASE + "/supplemental/metaZones.xml";
231 TIMEZONE_SOURCE_FILE = CLDR_BASE + "/bcp47/timezone.xml";
232 SPPL_META_SOURCE_FILE = CLDR_BASE + "/supplemental/supplementalMetadata.xml";
233
234 if (BASE_LOCALES.isEmpty()) {
235 setupBaseLocales("en-US");
236 }
237
238 bundleGenerator = new ResourceBundleGenerator();
239
240 // Parse data independent of locales
241 parseSupplemental();
242 parseBCP47();
243
244 List<Bundle> bundles = readBundleList();
245 convertBundles(bundles);
246
247 // Generate java.time.format.ZoneName.java
248 if (isBaseModule) {
249 generateZoneName();
250 }
251 }
252
253 private static void usage() {
254 errout("Usage: java CLDRConverter [options]%n"
255 + "\t-help output this usage message and exit%n"
256 + "\t-verbose output information%n"
257 + "\t-draft [contributed | approved | provisional | unconfirmed]%n"
258 + "\t\t draft level for using data (default: contributed)%n"
259 + "\t-base dir base directory for CLDR input files%n"
260 + "\t-basemodule generates bundles that go into java.base module%n"
261 + "\t-baselocales loc(,loc)* locales that go into the base module%n"
262 + "\t-o dir output directory (default: ./build/gensrc)%n"
263 + "\t-zntempfile template file for java.time.format.ZoneName.java%n"
264 + "\t-utf8 use UTF-8 rather than \\uxxxx (for debug)%n");
265 }
266
267 static void info(String fmt, Object... args) {
268 if (verbose) {
269 System.out.printf(fmt, args);
270 }
271 }
272
273 static void info(String msg) {
274 if (verbose) {
275 System.out.println(msg);
276 }
277 }
278
279 static void warning(String fmt, Object... args) {
280 System.err.print("Warning: ");
281 System.err.printf(fmt, args);
282 }
283
284 static void warning(String msg) {
285 System.err.print("Warning: ");
286 errout(msg);
287 }
288
289 static void severe(String fmt, Object... args) {
290 System.err.print("Error: ");
291 System.err.printf(fmt, args);
292 }
293
294 static void severe(String msg) {
295 System.err.print("Error: ");
296 errout(msg);
297 }
298
299 private static void errout(String msg) {
300 if (msg.contains("%n")) {
301 System.err.printf(msg);
302 } else {
303 System.err.println(msg);
304 }
305 }
306
307 /**
308 * Configure the parser to allow access to DTDs on the file system.
309 */
310 private static void enableFileAccess(SAXParser parser) throws SAXNotSupportedException {
311 try {
312 parser.setProperty("http://javax.xml.XMLConstants/property/accessExternalDTD", "file");
313 } catch (SAXNotRecognizedException ignore) {
314 // property requires >= JAXP 1.5
315 }
316 }
317
318 private static List<Bundle> readBundleList() throws Exception {
319 List<Bundle> retList = new ArrayList<>();
320 Path path = FileSystems.getDefault().getPath(SOURCE_FILE_DIR);
321 try (DirectoryStream<Path> dirStr = Files.newDirectoryStream(path)) {
322 for (Path entry : dirStr) {
323 String fileName = entry.getFileName().toString();
324 if (fileName.endsWith(".xml")) {
325 String id = fileName.substring(0, fileName.indexOf('.'));
326 Locale cldrLoc = Locale.forLanguageTag(toLanguageTag(id));
327 StringBuilder sb = getCandLocales(cldrLoc);
328 if (sb.indexOf("root") == -1) {
329 sb.append("root");
330 }
331 Bundle b = new Bundle(id, sb.toString(), null, null);
332 // Insert the bundle for root at the top so that it will get
333 // processed first.
334 if ("root".equals(id)) {
335 retList.add(0, b);
336 } else {
337 retList.add(b);
338 }
339 }
340 }
341 }
342 return retList;
343 }
344
345 private static final Map<String, Map<String, Object>> cldrBundles = new HashMap<>();
346
347 private static Map<String, SortedSet<String>> metaInfo = new HashMap<>();
348
349 static {
350 // For generating information on supported locales.
351 metaInfo.put("AvailableLocales", new TreeSet<>());
352 }
353
354 static Map<String, Object> getCLDRBundle(String id) throws Exception {
355 Map<String, Object> bundle = cldrBundles.get(id);
356 if (bundle != null) {
357 return bundle;
358 }
359 File file = new File(SOURCE_FILE_DIR + File.separator + id + ".xml");
360 if (!file.exists()) {
361 // Skip if the file doesn't exist.
362 return Collections.emptyMap();
363 }
364
365 info("..... main directory .....");
366 LDMLParseHandler handler = new LDMLParseHandler(id);
367 parseLDMLFile(file, handler);
368
369 bundle = handler.getData();
370 cldrBundles.put(id, bundle);
371
372 if (id.equals("root")) {
373 // Calendar data (firstDayOfWeek & minDaysInFirstWeek)
374 bundle = handlerSuppl.getData("root");
375 if (bundle != null) {
376 //merge two maps into one map
377 Map<String, Object> temp = cldrBundles.remove(id);
378 bundle.putAll(temp);
379 cldrBundles.put(id, bundle);
380 }
381 }
382 return bundle;
383 }
384
385 // Parsers for data in "supplemental" directory
386 //
387 private static void parseSupplemental() throws Exception {
388 // Parse SupplementalData file and store the information in the HashMap
389 // Calendar information such as firstDay and minDay are stored in
390 // supplementalData.xml as of CLDR1.4. Individual territory is listed
391 // with its ISO 3166 country code while default is listed using UNM49
392 // region and composition numerical code (001 for World.)
393 //
394 // SupplementalData file also provides the "parent" locales which
395 // are othrwise not to be fallen back. Process them here as well.
396 //
397 handlerSuppl = new SupplementDataParseHandler();
398 parseLDMLFile(new File(SPPL_SOURCE_FILE), handlerSuppl);
399 Map<String, Object> parentData = handlerSuppl.getData("root");
400 parentData.keySet().stream()
401 .filter(key -> key.startsWith(PARENT_LOCALE_PREFIX))
402 .forEach(key -> {
403 parentLocalesMap.put(key, new TreeSet(
404 Arrays.asList(((String)parentData.get(key)).split(" "))));
405 });
406
407 // Parse numberingSystems to get digit zero character information.
408 handlerNumbering = new NumberingSystemsParseHandler();
409 parseLDMLFile(new File(NUMBERING_SOURCE_FILE), handlerNumbering);
410
411 // Parse metaZones to create mappings between Olson tzids and CLDR meta zone names
412 handlerMetaZones = new MetaZonesParseHandler();
413 parseLDMLFile(new File(METAZONES_SOURCE_FILE), handlerMetaZones);
414
415 // Parse likelySubtags
416 handlerLikelySubtags = new LikelySubtagsParseHandler();
417 parseLDMLFile(new File(LIKELYSUBTAGS_SOURCE_FILE), handlerLikelySubtags);
418
419 // Parse supplementalMetadata
420 // Currently only interested in deprecated time zone ids.
421 handlerSupplMeta = new SupplementalMetadataParseHandler();
422 parseLDMLFile(new File(SPPL_META_SOURCE_FILE), handlerSupplMeta);
423 }
424
425 // Parsers for data in "bcp47" directory
426 //
427 private static void parseBCP47() throws Exception {
428 // Parse timezone
429 handlerTimeZone = new TimeZoneParseHandler();
430 parseLDMLFile(new File(TIMEZONE_SOURCE_FILE), handlerTimeZone);
431 }
432
433 private static void parseLDMLFile(File srcfile, AbstractLDMLHandler handler) throws Exception {
434 info("..... Parsing " + srcfile.getName() + " .....");
435 SAXParserFactory pf = SAXParserFactory.newInstance();
436 pf.setValidating(true);
437 SAXParser parser = pf.newSAXParser();
438 enableFileAccess(parser);
439 parser.parse(srcfile, handler);
440 }
441
442 private static StringBuilder getCandLocales(Locale cldrLoc) {
443 List<Locale> candList = getCandidateLocales(cldrLoc);
444 StringBuilder sb = new StringBuilder();
445 for (Locale loc : candList) {
446 if (!loc.equals(Locale.ROOT)) {
447 sb.append(toLocaleName(loc.toLanguageTag()));
448 sb.append(",");
449 }
450 }
451 return sb;
452 }
453
454 private static List<Locale> getCandidateLocales(Locale cldrLoc) {
455 List<Locale> candList = new ArrayList<>();
456 candList = applyParentLocales("", defCon.getCandidateLocales("", cldrLoc));
457 return candList;
458 }
459
460 private static void convertBundles(List<Bundle> bundles) throws Exception {
461 // parent locales map. The mappings are put in base metaInfo file
462 // for now.
463 if (isBaseModule) {
464 metaInfo.putAll(parentLocalesMap);
465 }
466
467 for (Bundle bundle : bundles) {
468 // Get the target map, which contains all the data that should be
469 // visible for the bundle's locale
470
471 Map<String, Object> targetMap = bundle.getTargetMap();
472
473 EnumSet<Bundle.Type> bundleTypes = bundle.getBundleTypes();
474
475 if (bundle.isRoot()) {
476 // Add DateTimePatternChars because CLDR no longer supports localized patterns.
477 targetMap.put("DateTimePatternChars", "GyMdkHmsSEDFwWahKzZ");
478 }
479
480 // Now the map contains just the entries that need to be in the resources bundles.
481 // Go ahead and generate them.
482 if (bundleTypes.contains(Bundle.Type.LOCALENAMES)) {
483 Map<String, Object> localeNamesMap = extractLocaleNames(targetMap, bundle.getID());
484 if (!localeNamesMap.isEmpty() || bundle.isRoot()) {
485 bundleGenerator.generateBundle("util", "LocaleNames", bundle.getJavaID(), true, localeNamesMap, BundleType.OPEN);
486 }
487 }
488 if (bundleTypes.contains(Bundle.Type.CURRENCYNAMES)) {
489 Map<String, Object> currencyNamesMap = extractCurrencyNames(targetMap, bundle.getID(), bundle.getCurrencies());
490 if (!currencyNamesMap.isEmpty() || bundle.isRoot()) {
491 bundleGenerator.generateBundle("util", "CurrencyNames", bundle.getJavaID(), true, currencyNamesMap, BundleType.OPEN);
492 }
493 }
494 if (bundleTypes.contains(Bundle.Type.TIMEZONENAMES)) {
495 Map<String, Object> zoneNamesMap = extractZoneNames(targetMap, bundle.getID());
496 if (!zoneNamesMap.isEmpty() || bundle.isRoot()) {
497 bundleGenerator.generateBundle("util", "TimeZoneNames", bundle.getJavaID(), true, zoneNamesMap, BundleType.TIMEZONE);
498 }
499 }
500 if (bundleTypes.contains(Bundle.Type.CALENDARDATA)) {
501 Map<String, Object> calendarDataMap = extractCalendarData(targetMap, bundle.getID());
502 if (!calendarDataMap.isEmpty() || bundle.isRoot()) {
503 bundleGenerator.generateBundle("util", "CalendarData", bundle.getJavaID(), true, calendarDataMap, BundleType.PLAIN);
504 }
505 }
506 if (bundleTypes.contains(Bundle.Type.FORMATDATA)) {
507 Map<String, Object> formatDataMap = extractFormatData(targetMap, bundle.getID());
508 if (!formatDataMap.isEmpty() || bundle.isRoot()) {
509 bundleGenerator.generateBundle("text", "FormatData", bundle.getJavaID(), true, formatDataMap, BundleType.PLAIN);
510 }
511 }
512
513 // For AvailableLocales
514 metaInfo.get("AvailableLocales").add(toLanguageTag(bundle.getID()));
515 addLikelySubtags(metaInfo, "AvailableLocales", bundle.getID());
516 }
517 bundleGenerator.generateMetaInfo(metaInfo);
518 }
519
520 static final Map<String, String> aliases = new HashMap<>();
521
522 /**
523 * Translate the aliases into the real entries in the bundle map.
524 */
525 static void handleAliases(Map<String, Object> bundleMap) {
526 Set bundleKeys = bundleMap.keySet();
527 try {
528 for (String key : aliases.keySet()) {
529 String targetKey = aliases.get(key);
530 if (bundleKeys.contains(targetKey)) {
531 bundleMap.putIfAbsent(key, bundleMap.get(targetKey));
532 }
533 }
534 } catch (Exception ex) {
535 Logger.getLogger(CLDRConverter.class.getName()).log(Level.SEVERE, null, ex);
536 }
537 }
538
539 /*
540 * Returns the language portion of the given id.
541 * If id is "root", "" is returned.
542 */
543 static String getLanguageCode(String id) {
544 return "root".equals(id) ? "" : Locale.forLanguageTag(id.replaceAll("_", "-")).getLanguage();
545 }
546
547 /**
548 * Examine if the id includes the country (territory) code. If it does, it returns
549 * the country code.
550 * Otherwise, it returns null. eg. when the id is "zh_Hans_SG", it return "SG".
551 * It does NOT return UN M.49 code, e.g., '001', as those three digit numbers cannot
552 * be translated into package names.
553 */
554 static String getCountryCode(String id) {
555 String rgn = getRegionCode(id);
556 return rgn.length() == 2 ? rgn: null;
557 }
558
559 /**
560 * Examine if the id includes the region code. If it does, it returns
561 * the region code.
562 * Otherwise, it returns null. eg. when the id is "zh_Hans_SG", it return "SG".
563 * It DOES return UN M.49 code, e.g., '001', as well as ISO 3166 two letter country codes.
564 */
565 static String getRegionCode(String id) {
566 return Locale.forLanguageTag(id.replaceAll("_", "-")).getCountry();
567 }
568
569 private static class KeyComparator implements Comparator<String> {
570 static KeyComparator INSTANCE = new KeyComparator();
571
572 private KeyComparator() {
573 }
574
575 @Override
576 public int compare(String o1, String o2) {
577 int len1 = o1.length();
578 int len2 = o2.length();
579 if (!isDigit(o1.charAt(0)) && !isDigit(o2.charAt(0))) {
580 // Shorter string comes first unless either starts with a digit.
581 if (len1 < len2) {
582 return -1;
583 }
584 if (len1 > len2) {
585 return 1;
586 }
587 }
588 return o1.compareTo(o2);
589 }
590
591 private boolean isDigit(char c) {
592 return c >= '0' && c <= '9';
593 }
594 }
595
596 private static Map<String, Object> extractLocaleNames(Map<String, Object> map, String id) {
597 Map<String, Object> localeNames = new TreeMap<>(KeyComparator.INSTANCE);
598 for (String key : map.keySet()) {
599 if (key.startsWith(LOCALE_NAME_PREFIX)) {
600 switch (key) {
601 case LOCALE_SEPARATOR:
602 localeNames.put("ListCompositionPattern", map.get(key));
603 break;
604 case LOCALE_KEYTYPE:
605 localeNames.put("ListKeyTypePattern", map.get(key));
606 break;
607 default:
608 localeNames.put(key.substring(LOCALE_NAME_PREFIX.length()), map.get(key));
609 break;
610 }
611 }
612 }
613
614 if (id.equals("root")) {
615 // Add display name pattern, which is not in CLDR
616 localeNames.put("DisplayNamePattern", "{0,choice,0#|1#{1}|2#{1} ({2})}");
617 }
618
619 return localeNames;
620 }
621
622 @SuppressWarnings("AssignmentToForLoopParameter")
623 private static Map<String, Object> extractCurrencyNames(Map<String, Object> map, String id, String names)
624 throws Exception {
625 Map<String, Object> currencyNames = new TreeMap<>(KeyComparator.INSTANCE);
626 for (String key : map.keySet()) {
627 if (key.startsWith(CURRENCY_NAME_PREFIX)) {
628 currencyNames.put(key.substring(CURRENCY_NAME_PREFIX.length()), map.get(key));
629 } else if (key.startsWith(CURRENCY_SYMBOL_PREFIX)) {
630 currencyNames.put(key.substring(CURRENCY_SYMBOL_PREFIX.length()), map.get(key));
631 }
632 }
633 return currencyNames;
634 }
635
636 private static Map<String, Object> extractZoneNames(Map<String, Object> map, String id) {
637 Map<String, Object> names = new HashMap<>();
638
639 // Copy over missing time zone ids from JRE for English locale
640 if (id.equals("en")) {
641 Map<String[], String> jreMetaMap = new HashMap<>();
642 jreTimeZoneNames.stream().forEach(e -> {
643 String tzid = (String)e[0];
644 String[] data = (String[])e[1];
645
646 if (map.get(TIMEZONE_ID_PREFIX + tzid) == null &&
647 handlerMetaZones.get(tzid) == null ||
648 handlerMetaZones.get(tzid) != null &&
649 map.get(METAZONE_ID_PREFIX + handlerMetaZones.get(tzid)) == null) {
650 // First, check the CLDR meta key
651 Optional<Map.Entry<String, String>> cldrMeta =
652 handlerMetaZones.getData().entrySet().stream()
653 .filter(me ->
654 Arrays.deepEquals(data,
655 (String[])map.get(METAZONE_ID_PREFIX + me.getValue())))
656 .findAny();
657 if (cldrMeta.isPresent()) {
658 names.put(tzid, cldrMeta.get().getValue());
659 } else {
660 // check the JRE meta key, add if there is not.
661 Optional<Map.Entry<String[], String>> jreMeta =
662 jreMetaMap.entrySet().stream()
663 .filter(jm -> Arrays.deepEquals(data, jm.getKey()))
664 .findAny();
665 if (jreMeta.isPresent()) {
666 names.put(tzid, jreMeta.get().getValue());
667 } else {
668 String metaName = "JRE_" + tzid.replaceAll("[/-]", "_");
669 names.put(METAZONE_ID_PREFIX + metaName, data);
670 names.put(tzid, metaName);
671 jreMetaMap.put(data, metaName);
672 }
673 }
674 }
675 });
676 }
677
678 Arrays.stream(AVAILABLE_TZIDS).forEach(tzid -> {
679 // If the tzid is deprecated, get the data for the replacement id
680 String tzKey = Optional.ofNullable((String)handlerSupplMeta.get(tzid))
681 .orElse(tzid);
682 Object data = map.get(TIMEZONE_ID_PREFIX + tzKey);
683
684 if (data instanceof String[]) {
685 names.put(tzid, data);
686 } else {
687 String meta = handlerMetaZones.get(tzKey);
688 if (meta != null) {
689 String metaKey = METAZONE_ID_PREFIX + meta;
690 data = map.get(metaKey);
691 if (data instanceof String[]) {
692 // Keep the metazone prefix here.
693 names.put(metaKey, data);
694 names.put(tzid, meta);
695 }
696 }
697 }
698 });
699
700 return names;
701 }
702
703 /**
704 * Extracts the language independent calendar data. Each of the two keys,
705 * "firstDayOfWeek" and "minimalDaysInFirstWeek" has a string value consists of
706 * one or multiple occurrences of:
707 * i: rg1 rg2 ... rgn;
708 * where "i" is the data for the following regions (delimited by a space) after
709 * ":", and ends with a ";".
710 */
711 private static Map<String, Object> extractCalendarData(Map<String, Object> map, String id) {
712 Map<String, Object> calendarData = new LinkedHashMap<>();
713 if (id.equals("root")) {
714 calendarData.put("firstDayOfWeek",
715 IntStream.range(1, 8)
716 .mapToObj(String::valueOf)
717 .filter(d -> map.keySet().contains(CALENDAR_FIRSTDAY_PREFIX + d))
718 .map(d -> d + ": " + map.get(CALENDAR_FIRSTDAY_PREFIX + d))
719 .collect(Collectors.joining(";")));
720 calendarData.put("minimalDaysInFirstWeek",
721 IntStream.range(0, 7)
722 .mapToObj(String::valueOf)
723 .filter(d -> map.keySet().contains(CALENDAR_MINDAYS_PREFIX + d))
724 .map(d -> d + ": " + map.get(CALENDAR_MINDAYS_PREFIX + d))
725 .collect(Collectors.joining(";")));
726 }
727 return calendarData;
728 }
729
730 static final String[] FORMAT_DATA_ELEMENTS = {
731 "MonthNames",
732 "standalone.MonthNames",
733 "MonthAbbreviations",
734 "standalone.MonthAbbreviations",
735 "MonthNarrows",
736 "standalone.MonthNarrows",
737 "DayNames",
738 "standalone.DayNames",
739 "DayAbbreviations",
740 "standalone.DayAbbreviations",
741 "DayNarrows",
742 "standalone.DayNarrows",
743 "QuarterNames",
744 "standalone.QuarterNames",
745 "QuarterAbbreviations",
746 "standalone.QuarterAbbreviations",
747 "QuarterNarrows",
748 "standalone.QuarterNarrows",
749 "AmPmMarkers",
750 "narrow.AmPmMarkers",
751 "abbreviated.AmPmMarkers",
752 "long.Eras",
753 "Eras",
754 "narrow.Eras",
755 "field.era",
756 "field.year",
757 "field.month",
758 "field.week",
759 "field.weekday",
760 "field.dayperiod",
761 "field.hour",
762 "timezone.hourFormat",
763 "timezone.gmtFormat",
764 "field.minute",
765 "field.second",
766 "field.zone",
767 "TimePatterns",
768 "DatePatterns",
769 "DateTimePatterns",
770 "DateTimePatternChars"
771 };
772
773 private static Map<String, Object> extractFormatData(Map<String, Object> map, String id) {
774 Map<String, Object> formatData = new LinkedHashMap<>();
775 for (CalendarType calendarType : CalendarType.values()) {
776 if (calendarType == CalendarType.GENERIC) {
777 continue;
778 }
779 String prefix = calendarType.keyElementName();
780 for (String element : FORMAT_DATA_ELEMENTS) {
781 String key = prefix + element;
782 copyIfPresent(map, "java.time." + key, formatData);
783 copyIfPresent(map, key, formatData);
784 }
785 }
786
787 for (String key : map.keySet()) {
788 // Copy available calendar names
789 if (key.startsWith(CLDRConverter.LOCALE_TYPE_PREFIX_CA)) {
790 String type = key.substring(CLDRConverter.LOCALE_TYPE_PREFIX_CA.length());
791 for (CalendarType calendarType : CalendarType.values()) {
792 if (calendarType == CalendarType.GENERIC) {
793 continue;
794 }
795 if (type.equals(calendarType.lname())) {
796 Object value = map.get(key);
797 String dataKey = key.replace(LOCALE_TYPE_PREFIX_CA,
798 CALENDAR_NAME_PREFIX);
799 formatData.put(dataKey, value);
800 String ukey = CALENDAR_NAME_PREFIX + calendarType.uname();
801 if (!dataKey.equals(ukey)) {
802 formatData.put(ukey, value);
803 }
804 }
805 }
806 }
807 }
808
809 copyIfPresent(map, "DefaultNumberingSystem", formatData);
810
811 @SuppressWarnings("unchecked")
812 List<String> numberingScripts = (List<String>) map.remove("numberingScripts");
813 if (numberingScripts != null) {
814 for (String script : numberingScripts) {
815 copyIfPresent(map, script + "." + "NumberElements", formatData);
816 }
817 } else {
818 copyIfPresent(map, "NumberElements", formatData);
819 }
820 copyIfPresent(map, "NumberPatterns", formatData);
821
822 // put extra number elements for available scripts into formatData, if it is "root"
823 if (id.equals("root")) {
824 handlerNumbering.keySet().stream()
825 .filter(k -> !numberingScripts.contains(k))
826 .forEach(k -> {
827 String[] ne = (String[])map.get("latn.NumberElements");
828 String[] neNew = Arrays.copyOf(ne, ne.length);
829 neNew[4] = handlerNumbering.get(k).substring(0, 1);
830 formatData.put(k + ".NumberElements", neNew);
831 });
832 }
833 return formatData;
834 }
835
836 private static void copyIfPresent(Map<String, Object> src, String key, Map<String, Object> dest) {
837 Object value = src.get(key);
838 if (value != null) {
839 dest.put(key, value);
840 }
841 }
842
843 // --- code below here is adapted from java.util.Properties ---
844 private static final String specialSaveCharsJava = "\"";
845 private static final String specialSaveCharsProperties = "=: \t\r\n\f#!";
846
847 /*
848 * Converts unicodes to encoded \uxxxx
849 * and writes out any of the characters in specialSaveChars
850 * with a preceding slash
851 */
852 static String saveConvert(String theString, boolean useJava) {
853 if (theString == null) {
854 return "";
855 }
856
857 String specialSaveChars;
858 if (useJava) {
859 specialSaveChars = specialSaveCharsJava;
860 } else {
861 specialSaveChars = specialSaveCharsProperties;
862 }
863 boolean escapeSpace = false;
864
865 int len = theString.length();
866 StringBuilder outBuffer = new StringBuilder(len * 2);
867 Formatter formatter = new Formatter(outBuffer, Locale.ROOT);
868
869 for (int x = 0; x < len; x++) {
870 char aChar = theString.charAt(x);
871 switch (aChar) {
872 case ' ':
873 if (x == 0 || escapeSpace) {
874 outBuffer.append('\\');
875 }
876 outBuffer.append(' ');
877 break;
878 case '\\':
879 outBuffer.append('\\');
880 outBuffer.append('\\');
881 break;
882 case '\t':
883 outBuffer.append('\\');
884 outBuffer.append('t');
885 break;
886 case '\n':
887 outBuffer.append('\\');
888 outBuffer.append('n');
889 break;
890 case '\r':
891 outBuffer.append('\\');
892 outBuffer.append('r');
893 break;
894 case '\f':
895 outBuffer.append('\\');
896 outBuffer.append('f');
897 break;
898 default:
899 if (aChar < 0x0020 || (!USE_UTF8 && aChar > 0x007e)) {
900 formatter.format("\\u%04x", (int)aChar);
901 } else {
902 if (specialSaveChars.indexOf(aChar) != -1) {
903 outBuffer.append('\\');
904 }
905 outBuffer.append(aChar);
906 }
907 }
908 }
909 return outBuffer.toString();
910 }
911
912 private static String toLanguageTag(String locName) {
913 if (locName.indexOf('_') == -1) {
914 return locName;
915 }
916 String tag = locName.replaceAll("_", "-");
917 Locale loc = Locale.forLanguageTag(tag);
918 return loc.toLanguageTag();
919 }
920
921 private static void addLikelySubtags(Map<String, SortedSet<String>> metaInfo, String category, String id) {
922 String likelySubtag = handlerLikelySubtags.get(id);
923 if (likelySubtag != null) {
924 // Remove Script for now
925 metaInfo.get(category).add(toLanguageTag(likelySubtag).replaceFirst("-[A-Z][a-z]{3}", ""));
926 }
927 }
928
929 private static String toLocaleName(String tag) {
930 if (tag.indexOf('-') == -1) {
931 return tag;
932 }
933 return tag.replaceAll("-", "_");
934 }
935
936 private static void setupBaseLocales(String localeList) {
937 Arrays.stream(localeList.split(","))
938 .map(Locale::forLanguageTag)
939 .map(l -> Control.getControl(Control.FORMAT_DEFAULT)
940 .getCandidateLocales("", l))
941 .forEach(BASE_LOCALES::addAll);
942 }
943
944 // applying parent locale rules to the passed candidates list
945 // This has to match with the one in sun.util.cldr.CLDRLocaleProviderAdapter
946 private static Map<Locale, Locale> childToParentLocaleMap = null;
947 private static List<Locale> applyParentLocales(String baseName, List<Locale> candidates) {
948 if (Objects.isNull(childToParentLocaleMap)) {
949 childToParentLocaleMap = new HashMap<>();
950 parentLocalesMap.keySet().forEach(key -> {
951 String parent = key.substring(PARENT_LOCALE_PREFIX.length()).replaceAll("_", "-");
952 parentLocalesMap.get(key).stream().forEach(child -> {
953 childToParentLocaleMap.put(Locale.forLanguageTag(child),
954 "root".equals(parent) ? Locale.ROOT : Locale.forLanguageTag(parent));
955 });
956 });
957 }
958
959 // check irregular parents
960 for (int i = 0; i < candidates.size(); i++) {
961 Locale l = candidates.get(i);
962 Locale p = childToParentLocaleMap.get(l);
963 if (!l.equals(Locale.ROOT) &&
964 Objects.nonNull(p) &&
965 !candidates.get(i+1).equals(p)) {
966 List<Locale> applied = candidates.subList(0, i+1);
967 applied.addAll(applyParentLocales(baseName, defCon.getCandidateLocales(baseName, p)));
968 return applied;
969 }
970 }
971
972 return candidates;
973 }
974
975 private static void generateZoneName() throws Exception {
976 Files.createDirectories(Paths.get(DESTINATION_DIR, "java", "time", "format"));
977 Files.write(Paths.get(DESTINATION_DIR, "java", "time", "format", "ZoneName.java"),
978 Files.lines(Paths.get(zoneNameTempFile))
979 .flatMap(l -> {
980 if (l.equals("%%%%ZIDMAP%%%%")) {
981 return zidMapEntry();
982 } else if (l.equals("%%%%MZONEMAP%%%%")) {
983 return handlerMetaZones.mzoneMapEntry();
984 } else if (l.equals("%%%%DEPRECATED%%%%")) {
985 return handlerSupplMeta.deprecatedMap();
986 } else {
987 return Stream.of(l);
988 }
989 })
990 .collect(Collectors.toList()),
991 StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING);
992 }
993
994 private static Stream<String> zidMapEntry() {
995 Map<String, String> canonMap = new HashMap<>();
996 handlerTimeZone.getData().entrySet().stream()
997 .forEach(e -> {
998 String[] ids = ((String)e.getValue()).split("\\s");
999 for (int i = 1; i < ids.length; i++) {
1000 canonMap.put(ids[i], ids[0]);
1001 }});
1002 return ZoneId.getAvailableZoneIds().stream()
1003 .map(id -> {
1004 String canonId = canonMap.getOrDefault(id, id);
1005 String meta = handlerMetaZones.get(canonId);
1006 String zone001 = handlerMetaZones.zidMap().get(meta);
1007 return zone001 == null ? "" :
1008 String.format(" \"%s\", \"%s\", \"%s\",",
1009 id, meta, zone001);
1010 })
1011 .filter(s -> !s.isEmpty())
1012 .sorted();
1013 }
1014 }