1 /*
   2  * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 package jdk.tools.jlink.internal.plugins;
  26 
  27 import java.io.ByteArrayInputStream;
  28 import java.util.Arrays;
  29 import java.util.Collections;
  30 import java.util.HashSet;
  31 import java.util.IllformedLocaleException;
  32 import java.util.Locale;
  33 import java.util.List;
  34 import java.util.Map;
  35 import java.util.Optional;
  36 import java.util.Set;
  37 import java.util.function.Predicate;
  38 import java.util.regex.Pattern;
  39 import java.util.stream.Collectors;
  40 import java.util.stream.IntStream;
  41 import java.util.stream.Stream;
  42 import jdk.internal.org.objectweb.asm.ClassReader;
  43 import jdk.tools.jlink.internal.ResourcePrevisitor;
  44 import jdk.tools.jlink.internal.StringTable;
  45 import jdk.tools.jlink.plugin.LinkModule;
  46 import jdk.tools.jlink.plugin.ModuleEntry;
  47 import jdk.tools.jlink.plugin.PluginException;
  48 import jdk.tools.jlink.plugin.ModulePool;
  49 import jdk.tools.jlink.plugin.TransformerPlugin;
  50 
  51 /**
  52  * Plugin to explicitly specify the locale data included in jdk.localedata
  53  * module. This plugin provides a jlink command line option "--include-locales"
  54  * with an argument. The argument is a list of BCP 47 language tags separated
  55  * by a comma. E.g.,
  56  *
  57  *  "jlink --include-locales en,ja,*-IN"
  58  *
  59  * This option will include locale data for all available English and Japanese
  60  * languages, and ones for the country of India. All other locale data are
  61  * filtered out on the image creation.
  62  *
  63  * Here are a few assumptions:
  64  *
  65  *  0. All locale data in java.base are unconditionally included.
  66  *  1. All the selective locale data are in jdk.localedata module
  67  *  2. Their package names are constructed by appending ".ext" to
  68  *     the corresponding ones in java.base module.
  69  *  3. Available locales string in LocaleDataMetaInfo class should
  70  *     start with at least one white space character, e.g., " ar ar-EG ..."
  71  *                                                           ^
  72  */
  73 public final class IncludeLocalesPlugin implements TransformerPlugin, ResourcePrevisitor {
  74 
  75     public static final String NAME = "include-locales";
  76     private static final String MODULENAME = "jdk.localedata";
  77     private static final Set<String> LOCALEDATA_PACKAGES = Set.of(
  78         "sun.text.resources.cldr.ext",
  79         "sun.text.resources.ext",
  80         "sun.util.resources.cldr.ext",
  81         "sun.util.resources.cldr.provider",
  82         "sun.util.resources.ext",
  83         "sun.util.resources.provider");
  84     private static final String METAINFONAME = "LocaleDataMetaInfo";
  85     private static final String META_FILES =
  86         "regex:.+module-info.class," +
  87         "regex:.+LocaleDataProvider.class," +
  88         "regex:.+" + METAINFONAME + ".class,";
  89     private static final String INCLUDE_LOCALE_FILES =
  90         "regex:.+sun/text/resources/ext/[^_]+_%%.class," +
  91         "regex:.+sun/util/resources/ext/[^_]+_%%.class," +
  92         "regex:.+sun/text/resources/cldr/ext/[^_]+_%%.class," +
  93         "regex:.+sun/util/resources/cldr/ext/[^_]+_%%.class,";
  94     private Predicate<String> predicate;
  95     private String userParam;
  96     private List<Locale.LanguageRange> priorityList;
  97     private List<Locale> available;
  98     private List<String> filtered;
  99 
 100     // Special COMPAT provider locales
 101     private static final String jaJPJPTag = "ja-JP-JP";
 102     private static final String noNONYTag = "no-NO-NY";
 103     private static final String thTHTHTag = "th-TH-TH";
 104     private static final Locale jaJPJP = new Locale("ja", "JP", "JP");
 105     private static final Locale noNONY = new Locale("no", "NO", "NY");
 106     private static final Locale thTHTH = new Locale("th", "TH", "TH");
 107 
 108     @Override
 109     public String getName() {
 110         return NAME;
 111     }
 112 
 113     @Override
 114     public void visit(ModulePool in, ModulePool out) {
 115         in.transformAndCopy((resource) -> {
 116             if (resource.getModule().equals(MODULENAME)) {
 117                 String path = resource.getPath();
 118                 resource = predicate.test(path) ? resource: null;
 119                 if (resource != null &&
 120                     resource.getType().equals(ModuleEntry.Type.CLASS_OR_RESOURCE)) {
 121                     byte[] bytes = resource.getBytes();
 122                     ClassReader cr = new ClassReader(bytes);
 123                     if (Arrays.stream(cr.getInterfaces())
 124                         .anyMatch(i -> i.contains(METAINFONAME)) &&
 125                         stripUnsupportedLocales(bytes, cr)) {
 126                         resource = ModuleEntry.create(MODULENAME, path,
 127                             resource.getType(),
 128                             new ByteArrayInputStream(bytes), bytes.length);
 129                     }
 130                 }
 131             }
 132             return resource;
 133         }, out);
 134     }
 135 
 136     @Override
 137     public Set<Category> getType() {
 138         Set<Category> set = new HashSet<>();
 139         set.add(Category.FILTER);
 140         return Collections.unmodifiableSet(set);
 141     }
 142 
 143     @Override
 144     public String getDescription() {
 145         return PluginsResourceBundle.getDescription(NAME);
 146     }
 147 
 148     @Override
 149     public boolean hasArguments() {
 150         return true;
 151     }
 152 
 153     @Override
 154     public String getArgumentsDescription() {
 155        return PluginsResourceBundle.getArgument(NAME);
 156     }
 157 
 158     @Override
 159     public void configure(Map<String, String> config) {
 160         userParam = config.get(NAME);
 161         priorityList = Arrays.stream(userParam.split(","))
 162             .map(s -> {
 163                 try {
 164                     return new Locale.LanguageRange(s);
 165                 } catch (IllegalArgumentException iae) {
 166                     throw new IllegalArgumentException(String.format(
 167                         PluginsResourceBundle.getMessage(NAME + ".invalidtag"), s));
 168                 }
 169             })
 170             .collect(Collectors.toList());
 171     }
 172 
 173     @Override
 174     public void previsit(ModulePool resources, StringTable strings) {
 175         final Pattern p = Pattern.compile(".*((Data_)|(Names_))(?<tag>.*)\\.class");
 176         Optional<LinkModule> optMod = resources.findModule(MODULENAME);
 177 
 178         // jdk.localedata module validation
 179         if (optMod.isPresent()) {
 180             LinkModule module = optMod.get();
 181             Set<String> packages = module.getAllPackages();
 182             if (!packages.containsAll(LOCALEDATA_PACKAGES)) {
 183                 throw new PluginException(PluginsResourceBundle.getMessage(NAME + ".missingpackages") +
 184                     LOCALEDATA_PACKAGES.stream()
 185                         .filter(pn -> !packages.contains(pn))
 186                         .collect(Collectors.joining(",\n\t")));
 187             }
 188 
 189             available = Stream.concat(module.entries()
 190                                         .map(md -> p.matcher(md.getPath()))
 191                                         .filter(m -> m.matches())
 192                                         .map(m -> m.group("tag").replaceAll("_", "-")),
 193                                     Stream.concat(Stream.of(jaJPJPTag), Stream.of(thTHTHTag)))
 194                 .distinct()
 195                 .sorted()
 196                 .map(IncludeLocalesPlugin::tagToLocale)
 197                 .collect(Collectors.toList());
 198         } else {
 199             // jdk.localedata is not added.
 200             throw new PluginException(PluginsResourceBundle.getMessage(NAME + ".localedatanotfound"));
 201         }
 202         filtered = filterLocales(available);
 203 
 204         if (filtered.isEmpty()) {
 205             throw new PluginException(
 206                 String.format(PluginsResourceBundle.getMessage(NAME + ".nomatchinglocales"), userParam));
 207         }
 208 
 209         String value = META_FILES + filtered.stream()
 210             .map(s -> includeLocaleFilePatterns(s))
 211             .collect(Collectors.joining(","));
 212         predicate = ResourceFilter.includeFilter(value);
 213     }
 214 
 215     private String includeLocaleFilePatterns(String tag) {
 216         String pTag = tag.replaceAll("-", "_");
 217         String files = "";
 218         int lastDelimiter = tag.length();
 219         String isoSpecial = pTag.matches("^(he|yi|id).*") ?
 220                             pTag.replaceFirst("he", "iw")
 221                                 .replaceFirst("yi", "ji")
 222                                 .replaceFirst("id", "in") : "";
 223 
 224         // Add tag patterns including parents
 225         while (true) {
 226             pTag = pTag.substring(0, lastDelimiter);
 227             files += INCLUDE_LOCALE_FILES.replaceAll("%%", pTag);
 228 
 229             if (!isoSpecial.isEmpty()) {
 230                 isoSpecial = isoSpecial.substring(0, lastDelimiter);
 231                 files += INCLUDE_LOCALE_FILES.replaceAll("%%", isoSpecial);
 232             }
 233 
 234             lastDelimiter = pTag.lastIndexOf('_');
 235             if (lastDelimiter == -1) {
 236                 break;
 237             }
 238         }
 239 
 240         final String lang = pTag;
 241 
 242         // Add possible special locales of the COMPAT provider
 243         files += Set.of(jaJPJPTag, noNONYTag, thTHTHTag).stream()
 244             .filter(stag -> lang.equals(stag.substring(0,2)))
 245             .map(t -> INCLUDE_LOCALE_FILES.replaceAll("%%", t.replaceAll("-", "_")))
 246             .collect(Collectors.joining(","));
 247 
 248         // Add possible UN.M49 files (unconditional for now) for each language
 249         files += INCLUDE_LOCALE_FILES.replaceAll("%%", lang + "_[0-9]{3}");
 250         if (!isoSpecial.isEmpty()) {
 251             files += INCLUDE_LOCALE_FILES.replaceAll("%%", isoSpecial + "_[0-9]{3}");
 252         }
 253 
 254         // Add Thai BreakIterator related data files
 255         if (lang.equals("th")) {
 256             files += "regex:.+sun/text/resources/thai_dict," +
 257                      "regex:.+sun/text/resources/[^_]+BreakIteratorData_th,";
 258         }
 259 
 260         // Add Taiwan resource bundles for Hong Kong
 261         if (tag.startsWith("zh-HK")) {
 262             files += INCLUDE_LOCALE_FILES.replaceAll("%%", "zh_TW");
 263         }
 264 
 265         return files;
 266     }
 267 
 268     private boolean stripUnsupportedLocales(byte[] bytes, ClassReader cr) {
 269         char[] buf = new char[cr.getMaxStringLength()];
 270         boolean[] modified = new boolean[1];
 271 
 272         IntStream.range(1, cr.getItemCount())
 273             .map(item -> cr.getItem(item))
 274             .forEach(itemIndex -> {
 275                 if (bytes[itemIndex - 1] == 1 &&         // UTF-8
 276                     bytes[itemIndex + 2] == (byte)' ') { // fast check for leading space
 277                     int length = cr.readUnsignedShort(itemIndex);
 278                     byte[] b = new byte[length];
 279                     System.arraycopy(bytes, itemIndex + 2, b, 0, length);
 280                     if (filterOutUnsupportedTags(b)) {
 281                         // copy back
 282                         System.arraycopy(b, 0, bytes, itemIndex + 2, length);
 283                         modified[0] = true;
 284                     }
 285                 }
 286             });
 287 
 288         return modified[0];
 289     }
 290 
 291     private boolean filterOutUnsupportedTags(byte[] b) {
 292         List<Locale> locales;
 293 
 294         try {
 295             locales = Arrays.asList(new String(b).split(" ")).stream()
 296                 .filter(tag -> !tag.isEmpty())
 297                 .map(IncludeLocalesPlugin::tagToLocale)
 298                 .collect(Collectors.toList());
 299         } catch (IllformedLocaleException ile) {
 300             // Seems not an available locales string literal.
 301             return false;
 302         }
 303 
 304         byte[] filteredBytes = filterLocales(locales).stream()
 305             .collect(Collectors.joining(" "))
 306             .getBytes();
 307         System.arraycopy(filteredBytes, 0, b, 0, filteredBytes.length);
 308         Arrays.fill(b, filteredBytes.length, b.length, (byte)' ');
 309         return true;
 310     }
 311 
 312     private List<String> filterLocales(List<Locale> locales) {
 313         List<String> ret =
 314             Locale.filter(priorityList, locales, Locale.FilteringMode.EXTENDED_FILTERING).stream()
 315                 .map(loc ->
 316                     // Locale.filter() does not preserve the case, which is
 317                     // significant for "variant" equality. Retrieve the original
 318                     // locales from the pre-filtered list.
 319                     locales.stream()
 320                         .filter(l -> l.toString().equalsIgnoreCase(loc.toString()))
 321                         .findAny()
 322                         .orElse(Locale.ROOT)
 323                         .toLanguageTag())
 324                 .collect(Collectors.toList());
 325 
 326         // no-NO-NY.toLanguageTag() returns "nn-NO", so specially handle it here
 327         if (ret.contains("no-NO")) {
 328             ret.add(noNONYTag);
 329         }
 330 
 331         return ret;
 332     }
 333 
 334     private static final Locale.Builder LOCALE_BUILDER = new Locale.Builder();
 335     private static Locale tagToLocale(String tag) {
 336         // ISO3166 compatibility
 337         tag = tag.replaceFirst("^iw", "he").replaceFirst("^ji", "yi").replaceFirst("^in", "id");
 338 
 339         switch (tag) {
 340             case jaJPJPTag:
 341                 return jaJPJP;
 342             case noNONYTag:
 343                 return noNONY;
 344             case thTHTHTag:
 345                 return thTHTH;
 346             default:
 347                 LOCALE_BUILDER.clear();
 348                 LOCALE_BUILDER.setLanguageTag(tag);
 349                 return LOCALE_BUILDER.build();
 350         }
 351     }
 352 }