1 /*
2 * Copyright (c) 2012, 2017, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25
26 package build.tools.cldrconverter;
27
28 import static build.tools.cldrconverter.Bundle.jreTimeZoneNames;
29 import build.tools.cldrconverter.BundleGenerator.BundleType;
30 import java.io.File;
31 import java.nio.file.DirectoryStream;
32 import java.nio.file.FileSystems;
33 import java.nio.file.Files;
34 import java.nio.file.Path;
35 import java.util.*;
36 import java.util.ResourceBundle.Control;
37 import java.util.logging.Level;
38 import java.util.logging.Logger;
39 import java.util.stream.Collectors;
40 import javax.xml.parsers.SAXParser;
41 import javax.xml.parsers.SAXParserFactory;
42 import org.xml.sax.SAXNotRecognizedException;
43 import org.xml.sax.SAXNotSupportedException;
44
45
46 /**
47 * Converts locale data from "Locale Data Markup Language" format to
48 * JRE resource bundle format. LDML is the format used by the Common
49 * Locale Data Repository maintained by the Unicode Consortium.
50 */
51 public class CLDRConverter {
52
53 static final String LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldml.dtd";
54 static final String SPPL_LDML_DTD_SYSTEM_ID = "http://www.unicode.org/cldr/dtd/2.0/ldmlSupplemental.dtd";
55
56 private static String CLDR_BASE = "../CLDR/21.0.1/";
57 static String LOCAL_LDML_DTD;
58 static String LOCAL_SPPL_LDML_DTD;
59 private static String SOURCE_FILE_DIR;
60 private static String SPPL_SOURCE_FILE;
61 private static String NUMBERING_SOURCE_FILE;
62 private static String METAZONES_SOURCE_FILE;
63 private static String LIKELYSUBTAGS_SOURCE_FILE;
64 static String DESTINATION_DIR = "build/gensrc";
65
66 static final String LOCALE_NAME_PREFIX = "locale.displayname.";
67 static final String CURRENCY_SYMBOL_PREFIX = "currency.symbol.";
68 static final String CURRENCY_NAME_PREFIX = "currency.displayname.";
69 static final String CALENDAR_NAME_PREFIX = "calendarname.";
70 static final String TIMEZONE_ID_PREFIX = "timezone.id.";
71 static final String ZONE_NAME_PREFIX = "timezone.displayname.";
72 static final String METAZONE_ID_PREFIX = "metazone.id.";
73 static final String PARENT_LOCALE_PREFIX = "parentLocale.";
74
75 private static SupplementDataParseHandler handlerSuppl;
76 private static LikelySubtagsParseHandler handlerLikelySubtags;
77 static NumberingSystemsParseHandler handlerNumbering;
78 static MetaZonesParseHandler handlerMetaZones;
79 private static BundleGenerator bundleGenerator;
80
81 // java.base module related
82 static boolean isBaseModule = false;
83 static final Set<Locale> BASE_LOCALES = new HashSet<>();
84
85 // "parentLocales" map
86 private static final Map<String, SortedSet<String>> parentLocalesMap = new HashMap<>();
87 private static final ResourceBundle.Control defCon =
88 ResourceBundle.Control.getControl(ResourceBundle.Control.FORMAT_DEFAULT);
89
90 static enum DraftType {
91 UNCONFIRMED,
92 PROVISIONAL,
93 CONTRIBUTED,
94 APPROVED;
95
96 private static final Map<String, DraftType> map = new HashMap<>();
97 static {
98 for (DraftType dt : values()) {
99 map.put(dt.getKeyword(), dt);
100 }
101 }
102 static private DraftType defaultType = CONTRIBUTED;
103
104 private final String keyword;
105
106 private DraftType() {
107 keyword = this.name().toLowerCase(Locale.ROOT);
108
109 }
110
111 static DraftType forKeyword(String keyword) {
112 return map.get(keyword);
113 }
114
115 static DraftType getDefault() {
116 return defaultType;
117 }
118
119 static void setDefault(String keyword) {
120 defaultType = Objects.requireNonNull(forKeyword(keyword));
121 }
122
123 String getKeyword() {
124 return keyword;
125 }
126 }
127
128 static boolean USE_UTF8 = false;
129 private static boolean verbose;
130
131 private CLDRConverter() {
132 // no instantiation
133 }
134
135 @SuppressWarnings("AssignmentToForLoopParameter")
136 public static void main(String[] args) throws Exception {
137 if (args.length != 0) {
138 String currentArg = null;
139 try {
140 for (int i = 0; i < args.length; i++) {
141 currentArg = args[i];
142 switch (currentArg) {
143 case "-draft":
144 String draftDataType = args[++i];
145 try {
146 DraftType.setDefault(draftDataType);
147 } catch (NullPointerException e) {
148 severe("Error: incorrect draft value: %s%n", draftDataType);
149 System.exit(1);
150 }
151 info("Using the specified data type: %s%n", draftDataType);
152 break;
153
154 case "-base":
155 // base directory for input files
156 CLDR_BASE = args[++i];
157 if (!CLDR_BASE.endsWith("/")) {
158 CLDR_BASE += "/";
159 }
160 break;
161
162 case "-baselocales":
163 // base locales
164 setupBaseLocales(args[++i]);
165 break;
166
167 case "-basemodule":
168 // indicates java.base module resource generation
169 isBaseModule = true;
170 break;
171
172 case "-o":
173 // output directory
174 DESTINATION_DIR = args[++i];
175 break;
176
177 case "-utf8":
178 USE_UTF8 = true;
179 break;
180
181 case "-verbose":
182 verbose = true;
183 break;
184
185 case "-help":
186 usage();
187 System.exit(0);
188 break;
189
190 default:
191 throw new RuntimeException();
192 }
193 }
194 } catch (RuntimeException e) {
195 severe("unknown or imcomplete arg(s): " + currentArg);
196 usage();
197 System.exit(1);
198 }
199 }
200
201 // Set up path names
202 LOCAL_LDML_DTD = CLDR_BASE + "/dtd/ldml.dtd";
203 LOCAL_SPPL_LDML_DTD = CLDR_BASE + "/dtd/ldmlSupplemental.dtd";
204 SOURCE_FILE_DIR = CLDR_BASE + "/main";
205 SPPL_SOURCE_FILE = CLDR_BASE + "/supplemental/supplementalData.xml";
206 LIKELYSUBTAGS_SOURCE_FILE = CLDR_BASE + "/supplemental/likelySubtags.xml";
207 NUMBERING_SOURCE_FILE = CLDR_BASE + "/supplemental/numberingSystems.xml";
208 METAZONES_SOURCE_FILE = CLDR_BASE + "/supplemental/metaZones.xml";
209
210 if (BASE_LOCALES.isEmpty()) {
211 setupBaseLocales("en-US");
212 }
213
214 bundleGenerator = new ResourceBundleGenerator();
215
216 // Parse data independent of locales
217 parseSupplemental();
218
219 List<Bundle> bundles = readBundleList();
220 convertBundles(bundles);
221 convertBundles(addedBundles);
222 }
223
224 private static void usage() {
225 errout("Usage: java CLDRConverter [options]%n"
226 + "\t-help output this usage message and exit%n"
227 + "\t-verbose output information%n"
228 + "\t-draft [contributed | approved | provisional | unconfirmed]%n"
229 + "\t\t draft level for using data (default: contributed)%n"
230 + "\t-base dir base directory for CLDR input files%n"
231 + "\t-basemodule generates bundles that go into java.base module%n"
232 + "\t-baselocales loc(,loc)* locales that go into the base module%n"
233 + "\t-o dir output directory (default: ./build/gensrc)%n"
234 + "\t-o dir output directory (defaut: ./build/gensrc)%n"
235 + "\t-utf8 use UTF-8 rather than \\uxxxx (for debug)%n");
236 }
237
238 static void info(String fmt, Object... args) {
239 if (verbose) {
240 System.out.printf(fmt, args);
241 }
242 }
243
244 static void info(String msg) {
245 if (verbose) {
246 System.out.println(msg);
247 }
248 }
249
250 static void warning(String fmt, Object... args) {
251 System.err.print("Warning: ");
252 System.err.printf(fmt, args);
253 }
254
255 static void warning(String msg) {
256 System.err.print("Warning: ");
257 errout(msg);
258 }
259
260 static void severe(String fmt, Object... args) {
261 System.err.print("Error: ");
262 System.err.printf(fmt, args);
263 }
264
265 static void severe(String msg) {
266 System.err.print("Error: ");
267 errout(msg);
268 }
269
270 private static void errout(String msg) {
271 if (msg.contains("%n")) {
272 System.err.printf(msg);
273 } else {
274 System.err.println(msg);
275 }
276 }
277
278 /**
279 * Configure the parser to allow access to DTDs on the file system.
280 */
281 private static void enableFileAccess(SAXParser parser) throws SAXNotSupportedException {
282 try {
283 parser.setProperty("http://javax.xml.XMLConstants/property/accessExternalDTD", "file");
284 } catch (SAXNotRecognizedException ignore) {
285 // property requires >= JAXP 1.5
286 }
287 }
288
289 private static List<Bundle> readBundleList() throws Exception {
290 List<Bundle> retList = new ArrayList<>();
291 Path path = FileSystems.getDefault().getPath(SOURCE_FILE_DIR);
292 try (DirectoryStream<Path> dirStr = Files.newDirectoryStream(path)) {
293 for (Path entry : dirStr) {
294 String fileName = entry.getFileName().toString();
295 if (fileName.endsWith(".xml")) {
296 String id = fileName.substring(0, fileName.indexOf('.'));
297 Locale cldrLoc = Locale.forLanguageTag(toLanguageTag(id));
298 StringBuilder sb = getCandLocales(cldrLoc);
299 if (sb.indexOf("root") == -1) {
300 sb.append("root");
301 }
302 Bundle b = new Bundle(id, sb.toString(), null, null);
303 // Insert the bundle for root at the top so that it will get
304 // processed first.
305 if ("root".equals(id)) {
306 retList.add(0, b);
307 } else {
308 retList.add(b);
309 }
310 }
311 }
312 }
313 return retList;
314 }
315
316 private static final Map<String, Map<String, Object>> cldrBundles = new HashMap<>();
317 // this list will contain additional bundles to be generated for Region dependent Data.
318 private static List<Bundle> addedBundles = new ArrayList<>();
319
320 private static Map<String, SortedSet<String>> metaInfo = new HashMap<>();
321
322 static {
323 // For generating information on supported locales.
324 metaInfo.put("LocaleNames", new TreeSet<>());
325 metaInfo.put("CurrencyNames", new TreeSet<>());
326 metaInfo.put("TimeZoneNames", new TreeSet<>());
327 metaInfo.put("CalendarData", new TreeSet<>());
328 metaInfo.put("FormatData", new TreeSet<>());
329 metaInfo.put("AvailableLocales", new TreeSet<>());
330 }
331
332
333 private static Set<String> calendarDataFields = Set.of("firstDayOfWeek", "minimalDaysInFirstWeek");
334
335 static Map<String, Object> getCLDRBundle(String id) throws Exception {
336 Map<String, Object> bundle = cldrBundles.get(id);
337 if (bundle != null) {
338 return bundle;
339 }
340 SAXParserFactory factory = SAXParserFactory.newInstance();
341 factory.setValidating(true);
342 SAXParser parser = factory.newSAXParser();
343 enableFileAccess(parser);
344 LDMLParseHandler handler = new LDMLParseHandler(id);
345 File file = new File(SOURCE_FILE_DIR + File.separator + id + ".xml");
346 if (!file.exists()) {
347 // Skip if the file doesn't exist.
348 return Collections.emptyMap();
349 }
350
351 info("..... main directory .....");
352 info("Reading file " + file);
353 parser.parse(file, handler);
354
355 bundle = handler.getData();
356 cldrBundles.put(id, bundle);
357 String country = getCountryCode(id);
358 if (country != null) {
359 bundle = handlerSuppl.getData(country);
360 if (bundle != null) {
361 //merge two maps into one map
362 Map<String, Object> temp = cldrBundles.remove(id);
363 bundle.putAll(temp);
364 cldrBundles.put(id, bundle);
365 }
366 }
367 return bundle;
368 }
369
370 // Parsers for data in "supplemental" directory
371 //
372 private static void parseSupplemental() throws Exception {
373 // Parse SupplementalData file and store the information in the HashMap
374 // Calendar information such as firstDay and minDay are stored in
375 // supplementalData.xml as of CLDR1.4. Individual territory is listed
376 // with its ISO 3166 country code while default is listed using UNM49
377 // region and composition numerical code (001 for World.)
378 //
379 // SupplementalData file also provides the "parent" locales which
380 // are othrwise not to be fallen back. Process them here as well.
381 //
382 info("..... Parsing supplementalData.xml .....");
383 SAXParserFactory factorySuppl = SAXParserFactory.newInstance();
384 factorySuppl.setValidating(true);
385 SAXParser parserSuppl = factorySuppl.newSAXParser();
386 enableFileAccess(parserSuppl);
387 handlerSuppl = new SupplementDataParseHandler();
388 File fileSupply = new File(SPPL_SOURCE_FILE);
389 parserSuppl.parse(fileSupply, handlerSuppl);
390 Map<String, Object> parentData = handlerSuppl.getData("root");
391 parentData.keySet().forEach(key -> {
392 parentLocalesMap.put(key, new TreeSet(
393 Arrays.asList(((String)parentData.get(key)).split(" "))));
394 });
395
396 // Parse numberingSystems to get digit zero character information.
397 SAXParserFactory numberingParser = SAXParserFactory.newInstance();
398 numberingParser.setValidating(true);
399 SAXParser parserNumbering = numberingParser.newSAXParser();
400 enableFileAccess(parserNumbering);
401 handlerNumbering = new NumberingSystemsParseHandler();
402 File fileNumbering = new File(NUMBERING_SOURCE_FILE);
403 parserNumbering.parse(fileNumbering, handlerNumbering);
404
405 // Parse metaZones to create mappings between Olson tzids and CLDR meta zone names
406 info("..... Parsing metaZones.xml .....");
407 SAXParserFactory metazonesParser = SAXParserFactory.newInstance();
408 metazonesParser.setValidating(true);
409 SAXParser parserMetaZones = metazonesParser.newSAXParser();
410 enableFileAccess(parserMetaZones);
411 handlerMetaZones = new MetaZonesParseHandler();
412 File fileMetaZones = new File(METAZONES_SOURCE_FILE);
413 parserMetaZones.parse(fileMetaZones, handlerMetaZones);
414
415 // Parse likelySubtags
416 info("..... Parsing likelySubtags.xml .....");
417 SAXParserFactory likelySubtagsParser = SAXParserFactory.newInstance();
418 likelySubtagsParser.setValidating(true);
419 SAXParser parserLikelySubtags = likelySubtagsParser.newSAXParser();
420 enableFileAccess(parserLikelySubtags);
421 handlerLikelySubtags = new LikelySubtagsParseHandler();
422 File fileLikelySubtags = new File(LIKELYSUBTAGS_SOURCE_FILE);
423 parserLikelySubtags.parse(fileLikelySubtags, handlerLikelySubtags);
424 }
425
426 /**
427 * This method will check if a new region dependent Bundle needs to be
428 * generated for this Locale id and targetMap. New Bundle will be generated
429 * when Locale id has non empty script and country code and targetMap
430 * contains region dependent data. This method will also remove region
431 * dependent data from this targetMap after candidate locales check. E.g. It
432 * will call genRegionDependentBundle() in case of az_Latn_AZ locale and
433 * remove region dependent data from this targetMap so that az_Latn_AZ
434 * bundle will not be created. For az_Cyrl_AZ, new Bundle will be generated
435 * but region dependent data will not be removed from targetMap as its candidate
436 * locales are [az_Cyrl_AZ, az_Cyrl, root], which does not include az_AZ for
437 * fallback.
438 *
439 */
440
441 private static void checkRegionDependentBundle(Map<String, Object> targetMap, String id) {
442 if ((CLDRConverter.getScript(id) != "")
443 && (CLDRConverter.getCountryCode(id) != "")) {
444 Map<String, Object> regionDepDataMap = targetMap
445 .keySet()
446 .stream()
447 .filter(calendarDataFields::contains)
448 .collect(Collectors.toMap(k -> k, targetMap::get));
449 if (!regionDepDataMap.isEmpty()) {
450 Locale cldrLoc = new Locale(CLDRConverter.getLanguageCode(id),
451 CLDRConverter.getCountryCode(id));
452 genRegionDependentBundle(regionDepDataMap, cldrLoc);
453 if (checkCandidateLocales(id, cldrLoc)) {
454 // Remove matchedKeys from this targetMap only if checkCandidateLocales() returns true.
455 regionDepDataMap.keySet().forEach(targetMap::remove);
456 }
457 }
458 }
459 }
460 /**
461 * This method will generate a new Bundle for region dependent data,
462 * minimalDaysInFirstWeek and firstDayOfWeek. Newly generated Bundle will be added
463 * to addedBundles list.
464 */
465 private static void genRegionDependentBundle(Map<String, Object> targetMap, Locale cldrLoc) {
466 String localeId = cldrLoc.toString();
467 StringBuilder sb = getCandLocales(cldrLoc);
468 if (sb.indexOf(localeId) == -1) {
469 sb.append(localeId);
470 }
471 Bundle bundle = new Bundle(localeId, sb.toString(), null, null);
472 cldrBundles.put(localeId, targetMap);
473 addedBundles.add(bundle);
474 }
475
476 private static StringBuilder getCandLocales(Locale cldrLoc) {
477 List<Locale> candList = getCandidateLocales(cldrLoc);
478 StringBuilder sb = new StringBuilder();
479 for (Locale loc : candList) {
480 if (!loc.equals(Locale.ROOT)) {
481 sb.append(toLocaleName(loc.toLanguageTag()));
482 sb.append(",");
483 }
484 }
485 return sb;
486 }
487
488 private static List<Locale> getCandidateLocales(Locale cldrLoc) {
489 List<Locale> candList = new ArrayList<>();
490 candList = applyParentLocales("", defCon.getCandidateLocales("", cldrLoc));
491 return candList;
492 }
493
494 /**
495 * This method will return true, if for a given locale, its language and
496 * country specific locale will exist in runtime lookup path. E.g. it will
497 * return true for bs_Latn_BA.
498 */
499 private static boolean checkCandidateLocales(String id, Locale cldrLoc) {
500 return(getCandidateLocales(Locale.forLanguageTag(id.replaceAll("_", "-")))
501 .contains(cldrLoc));
502 }
503
504 private static void convertBundles(List<Bundle> bundles) throws Exception {
505 // parent locales map. The mappings are put in base metaInfo file
506 // for now.
507 if (isBaseModule) {
508 metaInfo.putAll(parentLocalesMap);
509 }
510
511 for (Bundle bundle : bundles) {
512 // Get the target map, which contains all the data that should be
513 // visible for the bundle's locale
514
515 Map<String, Object> targetMap = bundle.getTargetMap();
516
517 // check if new region DependentBundle needs to be generated for this Locale.
518 checkRegionDependentBundle(targetMap, bundle.getID());
519 EnumSet<Bundle.Type> bundleTypes = bundle.getBundleTypes();
520
521 if (bundle.isRoot()) {
522 // Add DateTimePatternChars because CLDR no longer supports localized patterns.
523 targetMap.put("DateTimePatternChars", "GyMdkHmsSEDFwWahKzZ");
524 }
525
526 // Now the map contains just the entries that need to be in the resources bundles.
527 // Go ahead and generate them.
528 if (bundleTypes.contains(Bundle.Type.LOCALENAMES)) {
529 Map<String, Object> localeNamesMap = extractLocaleNames(targetMap, bundle.getID());
530 if (!localeNamesMap.isEmpty() || bundle.isRoot()) {
531 metaInfo.get("LocaleNames").add(toLanguageTag(bundle.getID()));
532 addLikelySubtags(metaInfo, "LocaleNames", bundle.getID());
533 bundleGenerator.generateBundle("util", "LocaleNames", bundle.getJavaID(), true, localeNamesMap, BundleType.OPEN);
534 }
535 }
536 if (bundleTypes.contains(Bundle.Type.CURRENCYNAMES)) {
537 Map<String, Object> currencyNamesMap = extractCurrencyNames(targetMap, bundle.getID(), bundle.getCurrencies());
538 if (!currencyNamesMap.isEmpty() || bundle.isRoot()) {
539 metaInfo.get("CurrencyNames").add(toLanguageTag(bundle.getID()));
540 addLikelySubtags(metaInfo, "CurrencyNames", bundle.getID());
541 bundleGenerator.generateBundle("util", "CurrencyNames", bundle.getJavaID(), true, currencyNamesMap, BundleType.OPEN);
542 }
543 }
544 if (bundleTypes.contains(Bundle.Type.TIMEZONENAMES)) {
545 Map<String, Object> zoneNamesMap = extractZoneNames(targetMap, bundle.getID());
546 if (!zoneNamesMap.isEmpty() || bundle.isRoot()) {
547 metaInfo.get("TimeZoneNames").add(toLanguageTag(bundle.getID()));
548 addLikelySubtags(metaInfo, "TimeZoneNames", bundle.getID());
549 bundleGenerator.generateBundle("util", "TimeZoneNames", bundle.getJavaID(), true, zoneNamesMap, BundleType.TIMEZONE);
550 }
551 }
552 if (bundleTypes.contains(Bundle.Type.CALENDARDATA)) {
553 Map<String, Object> calendarDataMap = extractCalendarData(targetMap, bundle.getID());
554 if (!calendarDataMap.isEmpty() || bundle.isRoot()) {
555 metaInfo.get("CalendarData").add(toLanguageTag(bundle.getID()));
556 addLikelySubtags(metaInfo, "CalendarData", bundle.getID());
557 bundleGenerator.generateBundle("util", "CalendarData", bundle.getJavaID(), true, calendarDataMap, BundleType.PLAIN);
558 }
559 }
560 if (bundleTypes.contains(Bundle.Type.FORMATDATA)) {
561 Map<String, Object> formatDataMap = extractFormatData(targetMap, bundle.getID());
562 if (!formatDataMap.isEmpty() || bundle.isRoot()) {
563 metaInfo.get("FormatData").add(toLanguageTag(bundle.getID()));
564 addLikelySubtags(metaInfo, "FormatData", bundle.getID());
565 bundleGenerator.generateBundle("text", "FormatData", bundle.getJavaID(), true, formatDataMap, BundleType.PLAIN);
566 }
567 }
568
569 // For AvailableLocales
570 metaInfo.get("AvailableLocales").add(toLanguageTag(bundle.getID()));
571 addLikelySubtags(metaInfo, "AvailableLocales", bundle.getID());
572 }
573 addCldrImplicitLocales(metaInfo);
574 bundleGenerator.generateMetaInfo(metaInfo);
575 }
576
577 /**
578 * These are the Locales that are implicitly supported by CLDR.
579 * Adding them explicitly as likelySubtags here, will ensure that
580 * COMPAT locales do not precede them during ResourceBundle search path.
581 */
582 private static void addCldrImplicitLocales(Map<String, SortedSet<String>> metaInfo) {
583 metaInfo.get("LocaleNames").add("zh-Hans-CN");
584 metaInfo.get("LocaleNames").add("zh-Hans-SG");
585 metaInfo.get("LocaleNames").add("zh-Hant-HK");
586 metaInfo.get("LocaleNames").add("zh-Hant-MO");
587 metaInfo.get("LocaleNames").add("zh-Hant-TW");
588 metaInfo.get("CurrencyNames").add("zh-Hans-CN");
589 metaInfo.get("CurrencyNames").add("zh-Hans-SG");
590 metaInfo.get("CurrencyNames").add("zh-Hant-HK");
591 metaInfo.get("CurrencyNames").add("zh-Hant-MO");
592 metaInfo.get("CurrencyNames").add("zh-Hant-TW");
593 metaInfo.get("TimeZoneNames").add("zh-Hans-CN");
594 metaInfo.get("TimeZoneNames").add("zh-Hans-SG");
595 metaInfo.get("TimeZoneNames").add("zh-Hant-HK");
596 metaInfo.get("TimeZoneNames").add("zh-Hant-MO");
597 metaInfo.get("TimeZoneNames").add("zh-Hant-TW");
598 metaInfo.get("TimeZoneNames").add("zh-HK");
599 metaInfo.get("CalendarData").add("zh-Hans-CN");
600 metaInfo.get("CalendarData").add("zh-Hans-SG");
601 metaInfo.get("CalendarData").add("zh-Hant-HK");
602 metaInfo.get("CalendarData").add("zh-Hant-MO");
603 metaInfo.get("CalendarData").add("zh-Hant-TW");
604 metaInfo.get("FormatData").add("zh-Hans-CN");
605 metaInfo.get("FormatData").add("zh-Hans-SG");
606 metaInfo.get("FormatData").add("zh-Hant-HK");
607 metaInfo.get("FormatData").add("zh-Hant-MO");
608 metaInfo.get("FormatData").add("zh-Hant-TW");
609 }
610 static final Map<String, String> aliases = new HashMap<>();
611
612 /**
613 * Translate the aliases into the real entries in the bundle map.
614 */
615 static void handleAliases(Map<String, Object> bundleMap) {
616 Set bundleKeys = bundleMap.keySet();
617 try {
618 for (String key : aliases.keySet()) {
619 String targetKey = aliases.get(key);
620 if (bundleKeys.contains(targetKey)) {
621 bundleMap.putIfAbsent(key, bundleMap.get(targetKey));
622 }
623 }
624 } catch (Exception ex) {
625 Logger.getLogger(CLDRConverter.class.getName()).log(Level.SEVERE, null, ex);
626 }
627 }
628
629 /*
630 * Returns the language portion of the given id.
631 * If id is "root", "" is returned.
632 */
633 static String getLanguageCode(String id) {
634 return "root".equals(id) ? "" : Locale.forLanguageTag(id.replaceAll("_", "-")).getLanguage();
635 }
636
637 /**
638 * Examine if the id includes the country (territory) code. If it does, it returns
639 * the country code.
640 * Otherwise, it returns null. eg. when the id is "zh_Hans_SG", it return "SG".
641 * It does NOT return UN M.49 code, e.g., '001', as those three digit numbers cannot
642 * be translated into package names.
643 */
644 static String getCountryCode(String id) {
645 String rgn = getRegionCode(id);
646 return rgn.length() == 2 ? rgn: null;
647 }
648
649 /**
650 * Examine if the id includes the region code. If it does, it returns
651 * the region code.
652 * Otherwise, it returns null. eg. when the id is "zh_Hans_SG", it return "SG".
653 * It DOES return UN M.49 code, e.g., '001', as well as ISO 3166 two letter country codes.
654 */
655 static String getRegionCode(String id) {
656 return Locale.forLanguageTag(id.replaceAll("_", "-")).getCountry();
657 }
658
659 /*
660 * Returns the script portion of the given id.
661 * If id is "root", "" is returned.
662 */
663 static String getScript(String id) {
664 return "root".equals(id) ? "" : Locale.forLanguageTag(id.replaceAll("_", "-")).getScript();
665 }
666
667 private static class KeyComparator implements Comparator<String> {
668 static KeyComparator INSTANCE = new KeyComparator();
669
670 private KeyComparator() {
671 }
672
673 @Override
674 public int compare(String o1, String o2) {
675 int len1 = o1.length();
676 int len2 = o2.length();
677 if (!isDigit(o1.charAt(0)) && !isDigit(o2.charAt(0))) {
678 // Shorter string comes first unless either starts with a digit.
679 if (len1 < len2) {
680 return -1;
681 }
682 if (len1 > len2) {
683 return 1;
684 }
685 }
686 return o1.compareTo(o2);
687 }
688
689 private boolean isDigit(char c) {
690 return c >= '0' && c <= '9';
691 }
692 }
693
694 private static Map<String, Object> extractLocaleNames(Map<String, Object> map, String id) {
695 Map<String, Object> localeNames = new TreeMap<>(KeyComparator.INSTANCE);
696 for (String key : map.keySet()) {
697 if (key.startsWith(LOCALE_NAME_PREFIX)) {
698 localeNames.put(key.substring(LOCALE_NAME_PREFIX.length()), map.get(key));
699 }
700 }
701 return localeNames;
702 }
703
704 @SuppressWarnings("AssignmentToForLoopParameter")
705 private static Map<String, Object> extractCurrencyNames(Map<String, Object> map, String id, String names)
706 throws Exception {
707 Map<String, Object> currencyNames = new TreeMap<>(KeyComparator.INSTANCE);
708 for (String key : map.keySet()) {
709 if (key.startsWith(CURRENCY_NAME_PREFIX)) {
710 currencyNames.put(key.substring(CURRENCY_NAME_PREFIX.length()), map.get(key));
711 } else if (key.startsWith(CURRENCY_SYMBOL_PREFIX)) {
712 currencyNames.put(key.substring(CURRENCY_SYMBOL_PREFIX.length()), map.get(key));
713 }
714 }
715 return currencyNames;
716 }
717
718 private static Map<String, Object> extractZoneNames(Map<String, Object> map, String id) {
719 Map<String, Object> names = new HashMap<>();
720
721 // Copy over missing time zone ids from JRE for English locale
722 if (id.equals("en")) {
723 Map<String[], String> jreMetaMap = new HashMap<>();
724 jreTimeZoneNames.stream().forEach(e -> {
725 String tzid = (String)e[0];
726 String[] data = (String[])e[1];
727
728 if (map.get(TIMEZONE_ID_PREFIX + tzid) == null &&
729 handlerMetaZones.get(tzid) == null ||
730 handlerMetaZones.get(tzid) != null &&
731 map.get(METAZONE_ID_PREFIX + handlerMetaZones.get(tzid)) == null) {
732 // First, check the CLDR meta key
733 Optional<Map.Entry<String, String>> cldrMeta =
734 handlerMetaZones.getData().entrySet().stream()
735 .filter(me ->
736 Arrays.deepEquals(data,
737 (String[])map.get(METAZONE_ID_PREFIX + me.getValue())))
738 .findAny();
739 if (cldrMeta.isPresent()) {
740 names.put(tzid, cldrMeta.get().getValue());
741 } else {
742 // check the JRE meta key, add if there is not.
743 Optional<Map.Entry<String[], String>> jreMeta =
744 jreMetaMap.entrySet().stream()
745 .filter(jm -> Arrays.deepEquals(data, jm.getKey()))
746 .findAny();
747 if (jreMeta.isPresent()) {
748 names.put(tzid, jreMeta.get().getValue());
749 } else {
750 String metaName = "JRE_" + tzid.replaceAll("[/-]", "_");
751 names.put(METAZONE_ID_PREFIX + metaName, data);
752 names.put(tzid, metaName);
753 jreMetaMap.put(data, metaName);
754 }
755 }
756 }
757 });
758 }
759
760 for (String tzid : handlerMetaZones.keySet()) {
761 String tzKey = TIMEZONE_ID_PREFIX + tzid;
762 Object data = map.get(tzKey);
763 if (data instanceof String[]) {
764 names.put(tzid, data);
765 } else {
766 String meta = handlerMetaZones.get(tzid);
767 if (meta != null) {
768 String metaKey = METAZONE_ID_PREFIX + meta;
769 data = map.get(metaKey);
770 if (data instanceof String[]) {
771 // Keep the metazone prefix here.
772 names.put(metaKey, data);
773 names.put(tzid, meta);
774 }
775 }
776 }
777 }
778 return names;
779 }
780
781 private static Map<String, Object> extractCalendarData(Map<String, Object> map, String id) {
782 Map<String, Object> calendarData = new LinkedHashMap<>();
783 copyIfPresent(map, "firstDayOfWeek", calendarData);
784 copyIfPresent(map, "minimalDaysInFirstWeek", calendarData);
785 return calendarData;
786 }
787
788 static final String[] FORMAT_DATA_ELEMENTS = {
789 "MonthNames",
790 "standalone.MonthNames",
791 "MonthAbbreviations",
792 "standalone.MonthAbbreviations",
793 "MonthNarrows",
794 "standalone.MonthNarrows",
795 "DayNames",
796 "standalone.DayNames",
797 "DayAbbreviations",
798 "standalone.DayAbbreviations",
799 "DayNarrows",
800 "standalone.DayNarrows",
801 "QuarterNames",
802 "standalone.QuarterNames",
803 "QuarterAbbreviations",
804 "standalone.QuarterAbbreviations",
805 "QuarterNarrows",
806 "standalone.QuarterNarrows",
807 "AmPmMarkers",
808 "narrow.AmPmMarkers",
809 "abbreviated.AmPmMarkers",
810 "long.Eras",
811 "Eras",
812 "narrow.Eras",
813 "field.era",
814 "field.year",
815 "field.month",
816 "field.week",
817 "field.weekday",
818 "field.dayperiod",
819 "field.hour",
820 "timezone.hourFormat",
821 "timezone.gmtFormat",
822 "field.minute",
823 "field.second",
824 "field.zone",
825 "TimePatterns",
826 "DatePatterns",
827 "DateTimePatterns",
828 "DateTimePatternChars"
829 };
830
831 private static Map<String, Object> extractFormatData(Map<String, Object> map, String id) {
832 Map<String, Object> formatData = new LinkedHashMap<>();
833 for (CalendarType calendarType : CalendarType.values()) {
834 if (calendarType == CalendarType.GENERIC) {
835 continue;
836 }
837 String prefix = calendarType.keyElementName();
838 for (String element : FORMAT_DATA_ELEMENTS) {
839 String key = prefix + element;
840 copyIfPresent(map, "java.time." + key, formatData);
841 copyIfPresent(map, key, formatData);
842 }
843 }
844
845 for (String key : map.keySet()) {
846 // Copy available calendar names
847 if (key.startsWith(CLDRConverter.CALENDAR_NAME_PREFIX)) {
848 String type = key.substring(CLDRConverter.CALENDAR_NAME_PREFIX.length());
849 for (CalendarType calendarType : CalendarType.values()) {
850 if (calendarType == CalendarType.GENERIC) {
851 continue;
852 }
853 if (type.equals(calendarType.lname())) {
854 Object value = map.get(key);
855 formatData.put(key, value);
856 String ukey = CLDRConverter.CALENDAR_NAME_PREFIX + calendarType.uname();
857 if (!key.equals(ukey)) {
858 formatData.put(ukey, value);
859 }
860 }
861 }
862 }
863 }
864
865 copyIfPresent(map, "DefaultNumberingSystem", formatData);
866
867 @SuppressWarnings("unchecked")
868 List<String> numberingScripts = (List<String>) map.remove("numberingScripts");
869 if (numberingScripts != null) {
870 for (String script : numberingScripts) {
871 copyIfPresent(map, script + "." + "NumberElements", formatData);
872 }
873 } else {
874 copyIfPresent(map, "NumberElements", formatData);
875 }
876 copyIfPresent(map, "NumberPatterns", formatData);
877 return formatData;
878 }
879
880 private static void copyIfPresent(Map<String, Object> src, String key, Map<String, Object> dest) {
881 Object value = src.get(key);
882 if (value != null) {
883 dest.put(key, value);
884 }
885 }
886
887 // --- code below here is adapted from java.util.Properties ---
888 private static final String specialSaveCharsJava = "\"";
889 private static final String specialSaveCharsProperties = "=: \t\r\n\f#!";
890
891 /*
892 * Converts unicodes to encoded \uxxxx
893 * and writes out any of the characters in specialSaveChars
894 * with a preceding slash
895 */
896 static String saveConvert(String theString, boolean useJava) {
897 if (theString == null) {
898 return "";
899 }
900
901 String specialSaveChars;
902 if (useJava) {
903 specialSaveChars = specialSaveCharsJava;
904 } else {
905 specialSaveChars = specialSaveCharsProperties;
906 }
907 boolean escapeSpace = false;
908
909 int len = theString.length();
910 StringBuilder outBuffer = new StringBuilder(len * 2);
911 Formatter formatter = new Formatter(outBuffer, Locale.ROOT);
912
913 for (int x = 0; x < len; x++) {
914 char aChar = theString.charAt(x);
915 switch (aChar) {
916 case ' ':
917 if (x == 0 || escapeSpace) {
918 outBuffer.append('\\');
919 }
920 outBuffer.append(' ');
921 break;
922 case '\\':
923 outBuffer.append('\\');
924 outBuffer.append('\\');
925 break;
926 case '\t':
927 outBuffer.append('\\');
928 outBuffer.append('t');
929 break;
930 case '\n':
931 outBuffer.append('\\');
932 outBuffer.append('n');
933 break;
934 case '\r':
935 outBuffer.append('\\');
936 outBuffer.append('r');
937 break;
938 case '\f':
939 outBuffer.append('\\');
940 outBuffer.append('f');
941 break;
942 default:
943 if (aChar < 0x0020 || (!USE_UTF8 && aChar > 0x007e)) {
944 formatter.format("\\u%04x", (int)aChar);
945 } else {
946 if (specialSaveChars.indexOf(aChar) != -1) {
947 outBuffer.append('\\');
948 }
949 outBuffer.append(aChar);
950 }
951 }
952 }
953 return outBuffer.toString();
954 }
955
956 private static String toLanguageTag(String locName) {
957 if (locName.indexOf('_') == -1) {
958 return locName;
959 }
960 String tag = locName.replaceAll("_", "-");
961 Locale loc = Locale.forLanguageTag(tag);
962 return loc.toLanguageTag();
963 }
964
965 private static void addLikelySubtags(Map<String, SortedSet<String>> metaInfo, String category, String id) {
966 String likelySubtag = handlerLikelySubtags.get(id);
967 if (likelySubtag != null) {
968 // Remove Script for now
969 metaInfo.get(category).add(toLanguageTag(likelySubtag).replaceFirst("-[A-Z][a-z]{3}", ""));
970 }
971 }
972
973 private static String toLocaleName(String tag) {
974 if (tag.indexOf('-') == -1) {
975 return tag;
976 }
977 return tag.replaceAll("-", "_");
978 }
979
980 private static void setupBaseLocales(String localeList) {
981 Arrays.stream(localeList.split(","))
982 .map(Locale::forLanguageTag)
983 .map(l -> Control.getControl(Control.FORMAT_DEFAULT)
984 .getCandidateLocales("", l))
985 .forEach(BASE_LOCALES::addAll);
986 }
987
988 // applying parent locale rules to the passed candidates list
989 // This has to match with the one in sun.util.cldr.CLDRLocaleProviderAdapter
990 private static Map<Locale, Locale> childToParentLocaleMap = null;
991 private static List<Locale> applyParentLocales(String baseName, List<Locale> candidates) {
992 if (Objects.isNull(childToParentLocaleMap)) {
993 childToParentLocaleMap = new HashMap<>();
994 parentLocalesMap.keySet().forEach(key -> {
995 String parent = key.substring(PARENT_LOCALE_PREFIX.length()).replaceAll("_", "-");
996 parentLocalesMap.get(key).stream().forEach(child -> {
997 childToParentLocaleMap.put(Locale.forLanguageTag(child),
998 "root".equals(parent) ? Locale.ROOT : Locale.forLanguageTag(parent));
999 });
1000 });
1001 }
1002
1003 // check irregular parents
1004 for (int i = 0; i < candidates.size(); i++) {
1005 Locale l = candidates.get(i);
1006 Locale p = childToParentLocaleMap.get(l);
1007 if (!l.equals(Locale.ROOT) &&
1008 Objects.nonNull(p) &&
1009 !candidates.get(i+1).equals(p)) {
1010 List<Locale> applied = candidates.subList(0, i+1);
1011 applied.addAll(applyParentLocales(baseName, defCon.getCandidateLocales(baseName, p)));
1012 return applied;
1013 }
1014 }
1015
1016 return candidates;
1017 }
1018 }