--- old/make/jdk/src/classes/build/tools/cldrconverter/CLDRConverter.java 2018-04-26 14:03:57.000000000 +0530 +++ new/make/jdk/src/classes/build/tools/cldrconverter/CLDRConverter.java 2018-04-26 14:03:56.000000000 +0530 @@ -87,8 +87,8 @@ static final String PARENT_LOCALE_PREFIX = "parentLocale."; private static SupplementDataParseHandler handlerSuppl; - private static SupplementalMetadataParseHandler handlerSupplMeta; private static LikelySubtagsParseHandler handlerLikelySubtags; + static SupplementalMetadataParseHandler handlerSupplMeta; static NumberingSystemsParseHandler handlerNumbering; static MetaZonesParseHandler handlerMetaZones; static TimeZoneParseHandler handlerTimeZone; @@ -425,7 +425,7 @@ parseLDMLFile(new File(LIKELYSUBTAGS_SOURCE_FILE), handlerLikelySubtags); // Parse supplementalMetadata - // Currently only interested in deprecated time zone ids. + // Currently interested in deprecated time zone ids and language aliases. handlerSupplMeta = new SupplementalMetadataParseHandler(); parseLDMLFile(new File(SPPL_META_SOURCE_FILE), handlerSupplMeta); } --- old/make/jdk/src/classes/build/tools/cldrconverter/ResourceBundleGenerator.java 2018-04-26 14:04:00.000000000 +0530 +++ new/make/jdk/src/classes/build/tools/cldrconverter/ResourceBundleGenerator.java 2018-04-26 14:03:59.000000000 +0530 @@ -268,7 +268,8 @@ out.printf("public class %s implements LocaleDataMetaInfo {\n", className); out.printf(" private static final Map resourceNameToLocales = new HashMap<>();\n" + (CLDRConverter.isBaseModule ? - " private static final Map parentLocalesMap = new HashMap<>();\n\n" : + " private static final Map parentLocalesMap = new HashMap<>();\n" + + " private static final Map languageAliasMap = new HashMap<>();\n\n" : "\n") + " static {\n"); @@ -299,10 +300,16 @@ } else { if ("AvailableLocales".equals(key)) { out.printf(" resourceNameToLocales.put(\"%s\",\n", key); - out.printf(" \"%s\");\n", toLocaleList(metaInfo.get(key), false)); + out.printf(" \"%s\");\n", toLocaleList(applyLanguageAliases(metaInfo.get(key)), false)); } } } + // for languageAliasMap + if (CLDRConverter.isBaseModule) { + CLDRConverter.handlerSupplMeta.getLanguageAliasData().forEach((key, value) -> { + out.printf(" languageAliasMap.put(\"%s\", \"%s\");\n", key, value); + }); + } out.printf(" }\n\n"); @@ -333,6 +340,10 @@ if (CLDRConverter.isBaseModule) { out.printf(" @Override\n" + + " public Map getLanguageAliasMap() {\n" + + " return languageAliasMap;\n" + + " }\n\n"); + out.printf(" @Override\n" + " public Map tzShortIDs() {\n" + " return TZShortIDMapHolder.tzShortIDMap;\n" + " }\n\n"); @@ -370,4 +381,13 @@ } return sb.toString(); } + + private static SortedSet applyLanguageAliases(SortedSet tags) { + CLDRConverter.handlerSupplMeta.getLanguageAliasData().forEach((key, value) -> { + if (tags.remove(key)) { + tags.add(value); + } + }); + return tags; + } } --- old/make/jdk/src/classes/build/tools/cldrconverter/SupplementalMetadataParseHandler.java 2018-04-26 14:04:03.000000000 +0530 +++ new/make/jdk/src/classes/build/tools/cldrconverter/SupplementalMetadataParseHandler.java 2018-04-26 14:04:02.000000000 +0530 @@ -27,6 +27,8 @@ import java.io.File; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import java.util.stream.Stream; import org.xml.sax.Attributes; import org.xml.sax.InputSource; @@ -38,6 +40,12 @@ */ class SupplementalMetadataParseHandler extends AbstractLDMLHandler { + private final Map languageAliasMap; + + SupplementalMetadataParseHandler() { + languageAliasMap = new HashMap<>(); + } + @Override public InputSource resolveEntity(String publicID, String systemID) throws IOException, SAXException { // avoid HTTP traffic to unicode.org @@ -57,6 +65,17 @@ } pushIgnoredContainer(qName); break; + case "languageAlias": + String aliasReason = attributes.getValue("reason"); + if ("deprecated".equals(aliasReason) || "legacy".equals(aliasReason)) { + String tag = attributes.getValue("type"); + if (!checkLegacyLocales(tag)) { + languageAliasMap.put(tag.replaceAll("_", "-"), + attributes.getValue("replacement").replaceAll("_", "-")); + } + } + pushIgnoredContainer(qName); + break; default: // treat anything else as a container pushContainer(qName, attributes); @@ -69,4 +88,13 @@ .map(k -> String.format(" \"%s\", \"%s\",", k, get(k))) .sorted(); } + Map getLanguageAliasData() { + return languageAliasMap; + } + + // skip language aliases for JDK legacy locales for ISO compatibility + private boolean checkLegacyLocales(String tag) { + return (tag.startsWith("no") || tag.startsWith("in") + || tag.startsWith("iw") || tag.startsWith("ji")); + } } --- old/src/java.base/share/classes/sun/util/cldr/CLDRLocaleProviderAdapter.java 2018-04-26 14:04:06.000000000 +0530 +++ new/src/java.base/share/classes/sun/util/cldr/CLDRLocaleProviderAdapter.java 2018-04-26 14:04:05.000000000 +0530 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -63,8 +63,14 @@ // parent locales map private static volatile Map parentLocalesMap; + // language aliases map + private static volatile Map langAliasesMap; + // cache to hold locale to locale mapping for language aliases. + private static final Map langAliasesCache; static { parentLocalesMap = new ConcurrentHashMap<>(); + langAliasesMap = new ConcurrentHashMap<>(); + langAliasesCache = new ConcurrentHashMap<>(); // Assuming these locales do NOT have irregular parent locales. parentLocalesMap.put(Locale.ROOT, Locale.ROOT); parentLocalesMap.put(Locale.ENGLISH, Locale.ENGLISH); @@ -140,6 +146,22 @@ } return locs; } + + private Locale applyAliases(Locale loc) { + if (langAliasesMap.isEmpty()) { + langAliasesMap = baseMetaInfo.getLanguageAliasMap(); + } + Locale locale = langAliasesCache.get(loc); + if (locale == null) { + String locTag = loc.toLanguageTag(); + Locale aliasLocale = langAliasesMap.containsKey(locTag) + ? Locale.forLanguageTag(langAliasesMap.get(locTag)) : loc; + langAliasesCache.putIfAbsent(loc, aliasLocale); + return aliasLocale; + } else { + return locale; + } + } @Override protected Set createLanguageTagSet(String category) { @@ -175,7 +197,7 @@ // Implementation of ResourceBundleBasedAdapter @Override public List getCandidateLocales(String baseName, Locale locale) { - List candidates = super.getCandidateLocales(baseName, locale); + List candidates = super.getCandidateLocales(baseName, applyAliases(locale)); return applyParentLocales(baseName, candidates); } --- old/src/java.base/share/classes/sun/util/locale/provider/LocaleDataMetaInfo.java 2018-04-26 14:04:09.000000000 +0530 +++ new/src/java.base/share/classes/sun/util/locale/provider/LocaleDataMetaInfo.java 2018-04-26 14:04:08.000000000 +0530 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -57,4 +57,13 @@ default public Map tzShortIDs() { return null; } + + /** + * Returns a map for language aliases which specifies mapping from source language + * to from which it should be replaced. + * @return map of source language to replacement language, separated by a space. + */ + default public Map getLanguageAliasMap(){ + return null; + } } --- old/test/jdk/sun/text/resources/LocaleData.cldr 2018-04-26 14:04:12.000000000 +0530 +++ new/test/jdk/sun/text/resources/LocaleData.cldr 2018-04-26 14:04:11.000000000 +0530 @@ -5419,10 +5419,10 @@ FormatData/sr_BA/MonthNames/6=\u0458\u0443\u043b FormatData/sr_BA/DayNames/3=\u0441\u0440\u0435\u0434\u0430 FormatData/sr_BA/DayAbbreviations/3=\u0441\u0440\u0435 -FormatData/sr_BA/TimePatterns/0=HH.mm.ss zzzz -FormatData/sr_BA/TimePatterns/1=HH.mm.ss z -FormatData/sr_BA/TimePatterns/2=HH.mm.ss -FormatData/sr_BA/TimePatterns/3=HH.mm +FormatData/sr_BA/TimePatterns/0=HH:mm:ss zzzz +FormatData/sr_BA/TimePatterns/1=HH:mm:ss z +FormatData/sr_BA/TimePatterns/2=HH:mm:ss +FormatData/sr_BA/TimePatterns/3=HH:mm FormatData/sr_BA/DatePatterns/0=EEEE, dd. MMMM y. FormatData/sr_BA/DatePatterns/1=dd. MMMM y. FormatData/sr_BA/DatePatterns/2=dd.MM.y. --- old/test/jdk/sun/text/resources/LocaleDataTest.java 2018-04-26 14:04:16.000000000 +0530 +++ new/test/jdk/sun/text/resources/LocaleDataTest.java 2018-04-26 14:04:15.000000000 +0530 @@ -38,7 +38,7 @@ * 7114053 7074882 7040556 8008577 8013836 8021121 6192407 6931564 8027695 * 8017142 8037343 8055222 8042126 8074791 8075173 8080774 8129361 8134916 * 8145136 8145952 8164784 8037111 8081643 7037368 8178872 8185841 8190918 - * 8187946 8195478 + * 8187946 8195478 8179071 * @summary Verify locale data * @modules java.base/sun.util.resources * @modules jdk.localedata --- old/test/jdk/tools/jlink/plugins/IncludeLocalesPluginTest.java 2018-04-26 14:04:18.000000000 +0530 +++ new/test/jdk/tools/jlink/plugins/IncludeLocalesPluginTest.java 2018-04-26 14:04:18.000000000 +0530 @@ -40,7 +40,7 @@ /* * @test - * @bug 8152143 8152704 8155649 8165804 8185841 8176841 8190918 + * @bug 8152143 8152704 8155649 8165804 8185841 8176841 8190918 8179071 * @summary IncludeLocalesPlugin tests * @author Naoto Sato * @requires (vm.compMode != "Xcomp" & os.maxMemory >= 2g) @@ -256,7 +256,7 @@ "(root)", "as_IN", "as", "bn_IN", "bn", "bo_IN", "bo", "brx_IN", "brx", "en", "en_001", "en_IN", "en_US", "en_US_POSIX", "gu_IN", "gu", "hi_IN", "hi", "kn_IN", "kn", "kok_IN", "kok", "ks_IN", "ks", "ml_IN", "ml", - "mr_IN", "mr", "ne_IN", "ne", "or_IN", "or", "pa_IN", "pa", "pa_IN_#Guru", + "mr_IN", "mr", "ne_IN", "ne", "or_IN", "or", "pa", "pa_IN_#Guru", "pa__#Guru", "ta_IN", "ta", "te_IN", "te", "ur_IN", "ur"), "", }, --- /dev/null 2018-04-26 14:04:22.000000000 +0530 +++ new/test/jdk/java/util/Locale/Bug8179071.java 2018-04-26 14:04:20.000000000 +0530 @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + /* + * @test + * @bug 8179071 + * @summary Test that language aliases of CLDR supplemental metadata are handled correctly. + * @modules jdk.localedata + * @run main/othervm -Djava.locale.providers=CLDR Bug8179071 + */ + +/** + * This fix is dependent on a particular version of CLDR data. + */ + +import java.time.Month; +import java.time.format.TextStyle; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +public class Bug8179071 { + + // Deprecated and Legacy tags. + private static final Set LegacyAliases = Set.of("pa-PK", "ug-Arab-CN", "kk-Cyrl-KZ", + "bs-BA", "ks-Arab-IN", "mn-Cyrl-MN", "ha-Latn-NE", + "shi-MA", "ha-Latn-NG", "ms-Latn-BN","ms-Latn-SG", + "ky-Cyrl-KG", "az-AZ", "zh-guoyu", "zh-min-nan", "i-klingon", "i-tsu", + "sr-XK", "sgn-CH-DE", "mo", "i-tay", "scc", "uz-UZ", "uz-AF", "sr-RS", + "i-hak", "sgn-BE-FR", "i-lux", "vai-LR", "tl", "zh-hakka", "i-ami", "aa-SAAHO", "ha-Latn-GH", + "zh-xiang", "i-pwn", "sgn-BE-NL", "jw", "sh", "tzm-Latn-MA", "i-bnn"); + // expected month format data for locales after language aliases replacement. + private static Map shortJanuaryNames = Map.of( "pa-PK", "\u062c\u0646\u0648\u0631\u06cc", + "uz-AF" , "\u062c\u0646\u0648", + "sr-ME", "jan", + "scc", "\u0458\u0430\u043d", + "sh", "jan", + "ha-Latn-NE", "Jan", + "i-lux", "Jan."); + + + private static void test(String tag, String expected) { + Locale target = Locale.forLanguageTag(tag); + Month day = Month.JANUARY; + TextStyle style = TextStyle.SHORT; + String actual = day.getDisplayName(style, target); + if (!actual.equals(expected)) { + throw new RuntimeException("failed for locale " + tag + " actual output " + actual +" does not match with " + expected); + } + } + + /** + * getAvailableLocales() should not contain any deprecated or Legacy language tags + */ + private static void checkInvalidTags() { + Set invalidTags = new HashSet<>(); + Arrays.asList(Locale.getAvailableLocales()).stream() + .map(loc -> loc.toLanguageTag()) + .forEach( tag -> {if(LegacyAliases.contains(tag)) {invalidTags.add(tag);}}); + if (!invalidTags.isEmpty()) { + throw new RuntimeException("failed: Deprecated and Legacy tags found " + invalidTags + " in AvailableLocales "); + } + } + + public static void main(String[] args) { + shortJanuaryNames.forEach((key, value) -> test(key, value)); + checkInvalidTags(); + } +}