--- old/src/java.base/share/classes/module-info.java 2016-07-12 14:23:13.583743873 -0700 +++ new/src/java.base/share/classes/module-info.java 2016-07-12 14:23:13.353739838 -0700 @@ -282,15 +282,18 @@ jdk.security.auth; exports sun.text.resources to jdk.localedata; - exports sun.util.resources to - jdk.localedata; + exports sun.util.cldr to + jdk.jlink; exports sun.util.locale.provider to java.desktop, + jdk.jlink, jdk.localedata; exports sun.util.logging to java.desktop, java.logging, java.prefs; + exports sun.util.resources to + jdk.localedata; // JDK-internal service types uses jdk.internal.logger.DefaultLoggerFinder; --- old/src/jdk.jlink/share/classes/jdk/tools/jlink/internal/plugins/IncludeLocalesPlugin.java 2016-07-12 14:23:14.845766009 -0700 +++ new/src/jdk.jlink/share/classes/jdk/tools/jlink/internal/plugins/IncludeLocalesPlugin.java 2016-07-12 14:23:14.625762150 -0700 @@ -24,6 +24,7 @@ */ package jdk.tools.jlink.internal.plugins; +import java.util.AbstractMap; import java.util.ArrayList; import java.util.Arrays; import java.util.IllformedLocaleException; @@ -31,6 +32,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import static java.util.ResourceBundle.Control; import java.util.Set; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -45,6 +47,10 @@ import jdk.tools.jlink.plugin.PluginException; import jdk.tools.jlink.plugin.ModulePool; import jdk.tools.jlink.plugin.Plugin; +import sun.util.cldr.CLDRBaseLocaleDataMetaInfo; +import sun.util.locale.provider.LocaleProviderAdapter; +import sun.util.locale.provider.LocaleProviderAdapter.Type; +import sun.util.locale.provider.ResourceBundleBasedAdapter; /** * Plugin to explicitly specify the locale data included in jdk.localedata @@ -95,6 +101,42 @@ private List available; private List filtered; + private static final ResourceBundleBasedAdapter CLDR_ADAPTER = + (ResourceBundleBasedAdapter)LocaleProviderAdapter.forType(Type.CLDR); + private static final Map CLDR_PARENT_LOCALES = + new CLDRBaseLocaleDataMetaInfo().parentLocales(); + + // Equivalent map + private static final Map> EQUIV_MAP = + Stream.concat( + // COMPAT equivalence + Map.of( + "zh-Hans", List.of("zh-Hans", "zh-CN", "zh-SG"), + "zh-Hant", List.of("zh-Hant", "zh-HK", "zh-MO", "zh-TW")) + .entrySet() + .stream(), + + // CLDR parent locales + CLDR_PARENT_LOCALES.entrySet().stream() + .map(entry -> { + String parent = entry.getKey().toLanguageTag(); + List children = new ArrayList<>(); + children.add(parent); + + Arrays.stream(entry.getValue()) + .filter(child -> !child.isEmpty()) + .flatMap(child -> + Stream.concat( + Arrays.stream(CLDR_PARENT_LOCALES.getOrDefault( + Locale.forLanguageTag(child), new String[0])) + .filter(grandchild -> !grandchild.isEmpty()), + List.of(child).stream())) + .distinct() + .forEach(children::add); + return new AbstractMap.SimpleEntry>(parent, children); + }) + ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + // Special COMPAT provider locales private static final String jaJPJPTag = "ja-JP-JP"; private static final String noNONYTag = "no-NO-NY"; @@ -152,16 +194,14 @@ @Override public void configure(Map config) { userParam = config.get(NAME); - priorityList = Arrays.stream(userParam.split(",")) - .map(s -> { - try { - return new Locale.LanguageRange(s); - } catch (IllegalArgumentException iae) { - throw new IllegalArgumentException(String.format( - PluginsResourceBundle.getMessage(NAME + ".invalidtag"), s)); - } - }) - .collect(Collectors.toList()); + + try { + priorityList = Locale.LanguageRange.parse(userParam, EQUIV_MAP); + } catch (IllegalArgumentException iae) { + throw new IllegalArgumentException(String.format( + PluginsResourceBundle.getMessage(NAME + ".invalidtag"), + iae.getMessage().replaceFirst("^range=", ""))); + } } @Override @@ -193,6 +233,7 @@ // jdk.localedata is not added. throw new PluginException(PluginsResourceBundle.getMessage(NAME + ".localedatanotfound")); } + filtered = filterLocales(available); if (filtered.isEmpty()) { @@ -205,56 +246,26 @@ filtered.stream().flatMap(s -> includeLocaleFilePatterns(s).stream())) .map(s -> "regex:" + s) .collect(Collectors.toList()); + predicate = ResourceFilter.includeFilter(value); } private List includeLocaleFilePatterns(String tag) { - List files = new ArrayList<>(); - String pTag = tag.replaceAll("-", "_"); - int lastDelimiter = tag.length(); - String isoSpecial = pTag.matches("^(he|yi|id).*") ? - pTag.replaceFirst("he", "iw") - .replaceFirst("yi", "ji") - .replaceFirst("id", "in") : ""; - - // Add tag patterns including parents - while (true) { - pTag = pTag.substring(0, lastDelimiter); - files.addAll(includeLocaleFiles(pTag)); - - if (!isoSpecial.isEmpty()) { - isoSpecial = isoSpecial.substring(0, lastDelimiter); - files.addAll(includeLocaleFiles(isoSpecial)); - } - - lastDelimiter = pTag.lastIndexOf('_'); - if (lastDelimiter == -1) { - break; - } + // Ignore extension variations + if (tag.matches(".+-[a-z]-.+")) { + return List.of(); } - final String lang = pTag; - - // Add possible special locales of the COMPAT provider - Set.of(jaJPJPTag, noNONYTag, thTHTHTag).stream() - .filter(stag -> lang.equals(stag.substring(0,2))) - .map(t -> includeLocaleFiles(t.replaceAll("-", "_"))) - .forEach(files::addAll); - - // Add possible UN.M49 files (unconditional for now) for each language - files.addAll(includeLocaleFiles(lang + "_[0-9]{3}")); - if (!isoSpecial.isEmpty()) { - files.addAll(includeLocaleFiles(isoSpecial + "_[0-9]{3}")); - } + List files = new ArrayList<>(includeLocaleFiles(tag.replaceAll("-", "_"))); // Add Thai BreakIterator related data files - if (lang.equals("th")) { + if (tag.equals("th")) { files.add(".+sun/text/resources/thai_dict"); files.add(".+sun/text/resources/[^_]+BreakIteratorData_th"); } // Add Taiwan resource bundles for Hong Kong - if (tag.startsWith("zh-HK")) { + if (tag.equals("zh-HK")) { files.addAll(includeLocaleFiles("zh_TW")); } @@ -306,6 +317,11 @@ byte[] filteredBytes = filterLocales(locales).stream() .collect(Collectors.joining(" ")) .getBytes(); + + if (filteredBytes.length > b.length) { + throw new InternalError("Size of filtered locales is bigger than the original one"); + } + System.arraycopy(filteredBytes, 0, b, 0, filteredBytes.length); Arrays.fill(b, filteredBytes.length, b.length, (byte)' '); return true; @@ -314,6 +330,9 @@ private List filterLocales(List locales) { List ret = Locale.filter(priorityList, locales, Locale.FilteringMode.EXTENDED_FILTERING).stream() + .flatMap(loc -> Stream.concat(Control.getNoFallbackControl(Control.FORMAT_DEFAULT) + .getCandidateLocales("", loc).stream(), + CLDR_ADAPTER.getCandidateLocales("", loc).stream())) .map(loc -> // Locale.filter() does not preserve the case, which is // significant for "variant" equality. Retrieve the original @@ -321,15 +340,12 @@ locales.stream() .filter(l -> l.toString().equalsIgnoreCase(loc.toString())) .findAny() - .orElse(Locale.ROOT) - .toLanguageTag()) + .orElse(Locale.ROOT)) + .filter(loc -> !loc.equals(Locale.ROOT)) + .flatMap(IncludeLocalesPlugin::localeToTags) + .distinct() .collect(Collectors.toList()); - // no-NO-NY.toLanguageTag() returns "nn-NO", so specially handle it here - if (ret.contains("no-NO")) { - ret.add(noNONYTag); - } - return ret; } @@ -338,6 +354,7 @@ // ISO3166 compatibility tag = tag.replaceFirst("^iw", "he").replaceFirst("^ji", "yi").replaceFirst("^in", "id"); + // Special COMPAT provider locales switch (tag) { case jaJPJPTag: return jaJPJP; @@ -351,4 +368,42 @@ return LOCALE_BUILDER.build(); } } + + private static Stream localeToTags(Locale loc) { + String tag = loc.toLanguageTag(); + Stream ret = null; + + switch (loc.getLanguage()) { + // ISO3166 compatibility + case "iw": + ret = List.of(tag, tag.replaceFirst("^he", "iw")).stream(); + break; + case "in": + ret = List.of(tag, tag.replaceFirst("^id", "in")).stream(); + break; + case "ji": + ret = List.of(tag, tag.replaceFirst("^yi", "ji")).stream(); + break; + + // Special COMPAT provider locales + case "ja": + if (loc.getCountry() == "JP") { + ret = List.of(tag, jaJPJPTag).stream(); + } + break; + case "no": + case "nn": + if (loc.getCountry() == "NO") { + ret = List.of(tag, noNONYTag).stream(); + } + break; + case "th": + if (loc.getCountry() == "TH") { + ret = List.of(tag, thTHTHTag).stream(); + } + break; + } + + return ret == null ? List.of(tag).stream() : ret; + } } --- old/test/tools/jlink/plugins/IncludeLocalesPluginTest.java 2016-07-12 14:23:16.090787846 -0700 +++ new/test/tools/jlink/plugins/IncludeLocalesPluginTest.java 2016-07-12 14:23:15.870783987 -0700 @@ -90,6 +90,71 @@ "", }, + // Asterisk works exactly the same as above + { + "*", + "jdk.localedata", + List.of( + "/jdk.localedata/sun/text/resources/ext/FormatData_en_GB.class", + "/jdk.localedata/sun/text/resources/ext/FormatData_ja.class", + "/jdk.localedata/sun/text/resources/ext/FormatData_th.class", + "/jdk.localedata/sun/text/resources/ext/FormatData_zh.class", + "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_en_001.class", + "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_ja.class", + "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_th.class", + "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_zh.class"), + List.of(), + Arrays.stream(Locale.getAvailableLocales()) + // "(root)" for Locale.ROOT rather than "" + .map(loc -> loc.equals(Locale.ROOT) ? "(root)" : loc.toString()) + .collect(Collectors.toList()), + "", + }, + + // World English/Spanish in Latin America + { + "--include-locales=en-001,es-419", + "jdk.localedata", + List.of( + "/jdk.localedata/sun/text/resources/ext/FormatData_en_AU.class", + "/jdk.localedata/sun/text/resources/ext/FormatData_es.class", + "/jdk.localedata/sun/text/resources/ext/FormatData_es_AR.class", + "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_en_001.class", + "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_en_150.class", + "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_en_AT.class", + "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_es.class", + "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_es_419.class", + "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_es_AR.class"), + List.of( + "/jdk.localedata/sun/text/resources/LineBreakIteratorData_th", + "/jdk.localedata/sun/text/resources/thai_dict", + "/jdk.localedata/sun/text/resources/WordBreakIteratorData_th", + "/jdk.localedata/sun/text/resources/ext/BreakIteratorInfo_th.class", + "/jdk.localedata/sun/text/resources/ext/BreakIteratorRules_th.class", + "/jdk.localedata/sun/text/resources/ext/FormatData_ja.class", + "/jdk.localedata/sun/text/resources/ext/FormatData_th.class", + "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_ja.class", + "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_th.class"), + List.of( + "(root)", "en", "en_US", "en_US_POSIX", "en_001", "en_150", "en_AG", "en_AI", + "en_AT", "en_AU", "en_BB", "en_BE", "en_BM", "en_BS", "en_BW", "en_BZ", + "en_CA", "en_CC", "en_CH", "en_CK", "en_CM", "en_CX", "en_CY", "en_DE", + "en_DG", "en_DK", "en_DM", "en_ER", "en_FI", "en_FJ", "en_FK", "en_FM", + "en_GB", "en_GD", "en_GG", "en_GH", "en_GI", "en_GM", "en_GY", "en_HK", + "en_IE", "en_IL", "en_IM", "en_IN", "en_IO", "en_JE", "en_JM", "en_KE", + "en_KI", "en_KN", "en_KY", "en_LC", "en_LR", "en_LS", "en_MG", "en_MO", + "en_MS", "en_MT", "en_MU", "en_MW", "en_MY", "en_NA", "en_NF", "en_NG", + "en_NL", "en_NR", "en_NU", "en_NZ", "en_PG", "en_PH", "en_PK", "en_PN", + "en_PW", "en_RW", "en_SB", "en_SC", "en_SD", "en_SE", "en_SG", "en_SH", + "en_SI", "en_SL", "en_SS", "en_SX", "en_SZ", "en_TC", "en_TK", "en_TO", + "en_TT", "en_TV", "en_TZ", "en_UG", "en_VC", "en_VG", "en_VU", "en_WS", + "en_ZA", "en_ZM", "en_ZW", "es", "es_419", "es_AR", "es_BO", "es_BR", + "es_CL", "es_CO", "es_CR", "es_CU", "es_DO", "es_EC", "es_GT", "es_HN", + "es_MX", "es_NI", "es_PA", "es_PE", "es_PR", "es_PY", "es_SV", "es_US", + "es_UY", "es_VE"), + "", + }, + // All English and Japanese locales { "--include-locales=en,ja", @@ -128,6 +193,35 @@ "", }, + // All locales in Austria + { + "--include-locales=*-AT", + "jdk.localedata", + List.of( + "/jdk.localedata/sun/text/resources/ext/FormatData_de.class", + "/jdk.localedata/sun/text/resources/ext/FormatData_de_AT.class", + "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_de.class", + "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_de_AT.class", + "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_en_001.class", + "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_en_150.class", + "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_en_AT.class"), + List.of( + "/jdk.localedata/sun/text/resources/LineBreakIteratorData_th", + "/jdk.localedata/sun/text/resources/thai_dict", + "/jdk.localedata/sun/text/resources/WordBreakIteratorData_th", + "/jdk.localedata/sun/text/resources/ext/BreakIteratorInfo_th.class", + "/jdk.localedata/sun/text/resources/ext/BreakIteratorRules_th.class", + "/jdk.localedata/sun/text/resources/ext/FormatData_en_GB.class", + "/jdk.localedata/sun/text/resources/ext/FormatData_ja.class", + "/jdk.localedata/sun/text/resources/ext/FormatData_th.class", + "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_ja.class", + "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_th.class"), + List.of( + "(root)", "en", "en_US", "en_US_POSIX", "en_001", "en_150", "en_AT", + "de", "de_AT"), + "", + }, + // All locales in India { "--include-locales=*-IN", @@ -154,10 +248,11 @@ "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_th.class", "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_zh.class"), List.of( - "(root)", "as_IN", "bn_IN", "bo_IN", "brx_IN", "en", /* "en_001", */ - "en_IN", "en_US", "en_US_POSIX", "gu_IN", "hi_IN", "kn_IN", "kok_IN", - "ks_IN", "ml_IN", "mr_IN", "ne_IN", "or_IN", "pa_IN", "pa_IN_#Guru", - "ta_IN", "te_IN", "ur_IN"), + "(root)", "as_IN", "as", "bn_IN", "bn", "bo_IN", "bo", "brx_IN", "brx", + "en", "en_001", "en_IN", "en_US", "en_US_POSIX", "gu_IN", "gu", "hi_IN", + "hi", "kn_IN", "kn", "kok_IN", "kok", "ks_IN", "ks", "ml_IN", "ml", + "mr_IN", "mr", "ne_IN", "ne", "or_IN", "or", "pa_IN", "pa", "pa_IN_#Guru", + "pa__#Guru", "ta_IN", "ta", "te_IN", "te", "ur_IN", "ur"), "", }, @@ -203,12 +298,40 @@ "/jdk.localedata/sun/text/resources/ext/FormatData_en_GB.class", "/jdk.localedata/sun/text/resources/ext/FormatData_ja.class", "/jdk.localedata/sun/text/resources/ext/FormatData_th.class", + "/jdk.localedata/sun/text/resources/ext/FormatData_zh_CN.class", + "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_en_001.class", + "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_ja.class", + "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_th.class"), + List.of( + "(root)", "en", "en_US", "en_US_POSIX", "zh", "zh__#Hans", "zh__#Hant", + "zh_HK", "zh_HK_#Hans", "zh_HK_#Hant"), + "", + }, + + // Simplified Chinese + { + "--include-locales=zh-Hans", + "jdk.localedata", + List.of( + "/jdk.localedata/sun/text/resources/ext/FormatData_zh.class", + "/jdk.localedata/sun/text/resources/ext/FormatData_zh_CN.class", + "/jdk.localedata/sun/text/resources/ext/FormatData_zh_SG.class", + "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_zh.class"), + List.of( + "/jdk.localedata/sun/text/resources/LineBreakIteratorData_th", + "/jdk.localedata/sun/text/resources/thai_dict", + "/jdk.localedata/sun/text/resources/WordBreakIteratorData_th", + "/jdk.localedata/sun/text/resources/ext/BreakIteratorInfo_th.class", + "/jdk.localedata/sun/text/resources/ext/BreakIteratorRules_th.class", + "/jdk.localedata/sun/text/resources/ext/FormatData_en_GB.class", + "/jdk.localedata/sun/text/resources/ext/FormatData_ja.class", + "/jdk.localedata/sun/text/resources/ext/FormatData_th.class", "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_en_001.class", "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_ja.class", "/jdk.localedata/sun/text/resources/cldr/ext/FormatData_th.class"), List.of( - "(root)", "en", "en_US", "en_US_POSIX", "zh_HK", "zh_HK_#Hans", - "zh_HK_#Hant"), + "(root)", "en", "en_US", "en_US_POSIX", "zh", "zh__#Hans", "zh_CN", + "zh_CN_#Hans", "zh_HK_#Hans", "zh_MO_#Hans", "zh_SG", "zh_SG_#Hans"), "", }, @@ -290,7 +413,7 @@ null, null, new PluginException(String.format( - PluginsResourceBundle.getMessage("include-locales.invalidtag"), "zh_HK")) + PluginsResourceBundle.getMessage("include-locales.invalidtag"), "zh_hk")) .getMessage(), },