1 /*
   2  * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 package com.sun.tools.jextract;
  24 
  25 import jdk.internal.clang.*;

  26 
  27 import java.io.ByteArrayOutputStream;
  28 import java.io.File;
  29 import java.io.IOException;
  30 import java.io.OutputStream;
  31 import java.io.PrintWriter;
  32 import java.io.UncheckedIOException;
  33 import java.lang.invoke.MethodHandles;
  34 import java.lang.invoke.MethodHandles.Lookup;
  35 import java.foreign.Library;
  36 import java.foreign.Libraries;
  37 import java.nio.file.Files;
  38 import java.nio.file.Path;
  39 import java.nio.file.Paths;
  40 import java.util.ArrayList;
  41 import java.util.Arrays;
  42 import java.util.Collections;
  43 import java.util.HashMap;
  44 import java.util.List;
  45 import java.util.Map;
  46 import java.util.Optional;
  47 import java.util.Properties;
  48 import java.util.Set;
  49 import java.util.TreeSet;
  50 import java.util.function.Function;
  51 import java.util.function.Predicate;
  52 import java.util.jar.JarOutputStream;
  53 import java.util.logging.Logger;
  54 import java.util.regex.Pattern;
  55 import java.util.stream.Collectors;
  56 import java.util.zip.ZipEntry;
  57 import com.sun.tools.jextract.parser.Parser;
  58 import com.sun.tools.jextract.tree.FunctionTree;
  59 import com.sun.tools.jextract.tree.HeaderTree;
  60 import com.sun.tools.jextract.tree.Tree;
  61 
  62 import static java.nio.file.StandardOpenOption.CREATE;
  63 import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
  64 import static java.nio.file.StandardOpenOption.WRITE;
  65 
  66 /**
  67  * The setup for the tool execution
  68  */
  69 public final class Context {
  70     // package name to TypeDictionary
  71     private final Map<String, TypeDictionary> tdMap;
  72     // The folder path mapping to package name
  73     private final Map<Path, String> pkgMap;
  74     // The header file parsed
  75     private final Map<Path, HeaderFile> headerMap;
  76     // The args for parsing C
  77     private final List<String> clangArgs;
  78     // The set of source header files
  79     private final Set<Path>  sources;
  80     // The list of library names
  81     private final List<String> libraryNames;
  82     // The list of library paths
  83     private final List<String> libraryPaths;
  84     // The list of library paths for link checks
  85     private final List<String> linkCheckPaths;
  86     // Symbol patterns to be excluded
  87     private final List<Pattern> excludeSymbols;
  88 
  89     final PrintWriter out;
  90     final PrintWriter err;
  91 
  92     private Predicate<String> symChecker;
  93     private Predicate<String> symFilter;
  94 
  95     private final Parser parser;
  96 
  97     private final static String defaultPkg = "jextract.dump";
  98     final Logger logger = Logger.getLogger(getClass().getPackage().getName());
  99 
 100     public Context(PrintWriter out, PrintWriter err) {
 101         this.tdMap = new HashMap<>();
 102         this.pkgMap = new HashMap<>();
 103         this.headerMap = new HashMap<>();
 104         this.clangArgs = new ArrayList<>();
 105         this.sources = new TreeSet<>();
 106         this.libraryNames = new ArrayList<>();
 107         this.libraryPaths = new ArrayList<>();
 108         this.linkCheckPaths = new ArrayList<>();
 109         this.excludeSymbols = new ArrayList<>();
 110         this.parser = new Parser(out, err, Main.INCLUDE_MACROS);
 111         this.out = out;
 112         this.err = err;
 113     }
 114 
 115     public Context() {
 116         this(new PrintWriter(System.out, true), new PrintWriter(System.err, true));
 117     }
 118 
 119     TypeDictionary typeDictionaryFor(String pkg) {
 120         return tdMap.computeIfAbsent(pkg, p->new TypeDictionary(this, p));
 121     }
 122 
 123     void addClangArg(String arg) {
 124         clangArgs.add(arg);
 125     }
 126 
 127     public void addSource(Path path) {
 128         sources.add(path);
 129     }
 130 
 131     void addLibraryName(String name) {
 132         libraryNames.add(name);
 133     }
 134 
 135     void addLibraryPath(String path) {
 136         libraryPaths.add(path);
 137     }
 138 
 139     void addLinkCheckPath(String path) {
 140         linkCheckPaths.add(path);
 141     }
 142 
 143     void addExcludeSymbols(String pattern) {
 144         excludeSymbols.add(Pattern.compile(pattern));
 145     }
 146 
 147     // return the absolute path of the library of given name by searching
 148     // in the given array of paths.
 149     private static Optional<Path> findLibraryPath(Path[] paths, String libName) {
 150          return Arrays.stream(paths).
 151               map(p -> p.resolve(System.mapLibraryName(libName))).
 152               filter(Files::isRegularFile).map(Path::toAbsolutePath).findFirst();
 153     }
 154 
 155     /*
 156      * Load the specified shared libraries from the specified paths.
 157      *
 158      * @param lookup Lookup object of the caller.
 159      * @param pathStrs array of paths to load the shared libraries from.
 160      * @param names array of shared library names.
 161      */
 162     // used by jextract tool to load libraries for symbol checks.
 163     public static Library[] loadLibraries(Lookup lookup, String[] pathStrs, String[] names) {
 164         if (pathStrs == null || pathStrs.length == 0) {
 165             return Arrays.stream(names).map(
 166                 name -> Libraries.loadLibrary(lookup, name)).toArray(Library[]::new);
 167         } else {
 168             Path[] paths = Arrays.stream(pathStrs).map(Paths::get).toArray(Path[]::new);
 169             return Arrays.stream(names).map(libName -> {
 170                 Optional<Path> absPath = findLibraryPath(paths, libName);
 171                 return absPath.isPresent() ?
 172                     Libraries.load(lookup, absPath.get().toString()) :
 173                     Libraries.loadLibrary(lookup, libName);
 174             }).toArray(Library[]::new);
 175         }
 176     }
 177 
 178     private void initSymChecker() {
 179         if (!libraryNames.isEmpty() && !linkCheckPaths.isEmpty()) {
 180             Library[] libs = loadLibraries(MethodHandles.lookup(),
 181                 linkCheckPaths.toArray(new String[0]),
 182                 libraryNames.toArray(new String[0]));
 183 
 184             // check if the given symbol is found in any of the libraries or not.
 185             // If not found, warn the user for the missing symbol.
 186             symChecker = name -> {
 187                 if (Main.DEBUG) {
 188                     err.println("Searching symbol: " + name);
 189                 }
 190                 return (Arrays.stream(libs).filter(lib -> {
 191                         try {
 192                             lib.lookup(name);
 193                             if (Main.DEBUG) {
 194                                 err.println("Found symbol: " + name);
 195                             }
 196                             return true;
 197                         } catch (NoSuchMethodException nsme) {
 198                             return false;
 199                         }
 200                     }).findFirst().isPresent());
 201             };
 202         } else {
 203             symChecker = null;
 204         }
 205     }
 206 
 207     private boolean isSymbolFound(String name) {
 208         return symChecker == null? true : symChecker.test(name);
 209     }
 210 
 211     private void initSymFilter() {
 212         if (!excludeSymbols.isEmpty()) {
 213             Pattern[] pats = excludeSymbols.toArray(new Pattern[0]);
 214             symFilter = name -> {
 215                 return Arrays.stream(pats).filter(pat -> pat.matcher(name).matches()).
 216                     findFirst().isPresent();
 217             };
 218         } else {
 219             symFilter = null;
 220         }
 221     }
 222 
 223     private boolean isSymbolExcluded(String name) {
 224         return symFilter == null? false : symFilter.test(name);
 225     }
 226 
 227     /**
 228      * Setup a package name for a given folder.
 229      *
 230      * @param folder The path to the folder, use null to set catch-all.
 231      * @param pkg    The package name
 232      * @return True if the folder is setup successfully. False is a package
 233      * has been assigned for the folder.
 234      */
 235     public boolean usePackageForFolder(Path folder, String pkg) {
 236         if (folder != null) {
 237             folder = folder.toAbsolutePath();
 238             if (!Files.isDirectory(folder)) {
 239                 folder = folder.getParent();
 240             }
 241         }
 242         String existing = pkgMap.putIfAbsent(folder, pkg);
 243         final String finalFolder = (null == folder) ? "all folders not configured" : folder.toString();
 244         if (null == existing) {
 245             logger.config(() -> "Package " + pkg + " is selected for " + finalFolder);
 246             return true;
 247         } else {
 248             logger.warning(() -> "Package " + existing + " had been selected for " + finalFolder + ", request to use " + pkg + " is ignored.");
 249             return false;
 250         }
 251     }
 252 
 253     static class Entity {
 254         final String pkg;
 255         final String entity;
 256 
 257         Entity(String pkg, String entity) {
 258             this.pkg = pkg;
 259             this.entity = entity;
 260         }
 261     }
 262 
 263     /**
 264      * Determine package and interface name given a path. If the path is
 265      * a folder, then only package name is determined. The package name is
 266      * determined with the longest path matching the setup. If the path is not
 267      * setup for any package, the default package name is returned.
 268      *
 269      * @param origin The source path
 270      * @return The Entity
 271      * @see Context::usePackageForFolder(Path, String)
 272      */
 273     Entity whatis(Path origin) {
 274         // normalize to absolute path
 275         origin = origin.toAbsolutePath();
 276         String filename = null;
 277         if (!Files.isDirectory(origin)) {
 278             // ensure it's a folder name
 279             filename = origin.getFileName().toString();
 280             origin = origin.getParent();
 281         }
 282         Path path = origin;
 283 
 284         // search the map for a hit with longest path
 285         while (path != null && !pkgMap.containsKey(path)) {
 286             path = path.getParent();
 287         }
 288 
 289         int start;
 290         String pkg;
 291         if (path != null) {
 292             start = path.getNameCount();
 293             pkg = pkgMap.get(path);
 294         } else {
 295             pkg = pkgMap.get(null);
 296             if (pkg == null) {
 297                 start = 0;
 298                 pkg = defaultPkg;
 299             } else {
 300                 start = origin.getNameCount();
 301             }
 302         }
 303 
 304         if (filename == null) {
 305             // a folder, only pkg name matters
 306             return new Entity(pkg, null);
 307         }
 308 
 309         StringBuilder sb = new StringBuilder();
 310         while (start < origin.getNameCount()) {
 311             sb.append(Utils.toJavaIdentifier(origin.getName(start++).toString()));
 312             sb.append("_");
 313         }
 314 
 315         int ext = filename.lastIndexOf('.');
 316         if (ext != -1) {
 317             sb.append(filename.substring(0, ext));
 318         } else {
 319             sb.append(filename);
 320         }
 321         return new Entity(pkg, Utils.toClassName(sb.toString()));
 322     }
 323 
 324     HeaderFile getHeaderFile(Path header, HeaderFile main) {
 325         if (!Files.isRegularFile(header)) {
 326             logger.warning(() -> "Not a regular file: " + header.toString());
 327             throw new IllegalArgumentException(header.toString());
 328         }
 329 
 330         final Context.Entity e = whatis(header);
 331         return new HeaderFile(this, header, e.pkg, e.entity, main);
 332     }
 333 
 334     void processTree(Tree tree, HeaderFile main, Function<HeaderFile, AsmCodeFactory> fn) {
 335         SourceLocation loc = tree.location();
 336 
 337         HeaderFile header;
 338         boolean isBuiltIn = false;
 339 
 340         if (tree.isFromMain()) {
 341             header = main;
 342         } else {
 343             SourceLocation.Location src = loc.getFileLocation();
 344             if (src == null) {
 345                 logger.info(() -> "Tree " + tree.name() + "@" + tree.USR() + " has no FileLocation");
 346                 return;
 347             }
 348 
 349             Path p = src.path();
 350             if (p == null) {
 351                 logger.fine(() -> "Found built-in type: " + tree.name());
 352                 header = main;
 353                 isBuiltIn = true;
 354             } else {
 355                 p = p.normalize().toAbsolutePath();
 356                 header = headerMap.get(p);
 357                 if (header == null) {
 358                     final HeaderFile hf = header = getHeaderFile(p, main);
 359                     logger.config(() -> "First encounter of header file " + hf.path + ", assigned to package " + hf.pkgName);
 360                     // Only generate code for header files sepcified or in the same package
 361                     // System headers are excluded, they need to be explicitly specified in jextract cmdline
 362                     if (sources.contains(p) ||
 363                         (!loc.isInSystemHeader()) && (header.pkgName.equals(main.pkgName))) {
 364                         logger.config("Code gen for header " + p + " enabled in package " + header.pkgName);
 365                         header.useCodeFactory(fn.apply(header));
 366                     }
 367                     headerMap.put(p, header);
 368                 }
 369             }
 370         }
 371 
 372         header.processTree(tree, main, isBuiltIn);
 373     }
 374 
 375     public void parse() {
 376         parse(header -> new AsmCodeFactory(this, header));
 377     }
 378 
 379     private boolean symbolFilter(Tree tree) {
 380          String name = tree.name();
 381          if (isSymbolExcluded(name)) {
 382              return false;
 383          }
 384 
 385          // check for function symbols in libraries & warn missing symbols
 386          if (tree instanceof FunctionTree && !isSymbolFound(name)) {
 387              err.println(Main.format("warn.symbol.not.found", name));
 388          }
 389 
 390          return true;
 391     }
 392 
 393     public void parse(Function<HeaderFile, AsmCodeFactory> fn) {
 394         initSymChecker();
 395         initSymFilter();
 396 
 397         List<HeaderTree> headers = parser.parse(sources, clangArgs);
 398         processHeaders(headers, fn);
 399     }
 400 
 401     private void processHeaders(List<HeaderTree> headers, Function<HeaderFile, AsmCodeFactory> fn) {
 402         headers.stream().
 403                 map(new TreeFilter(this::symbolFilter)).
 404                 map(new TypedefHandler()).
 405                 map(new EmptyNameHandler()).
 406                 forEach(header -> {
 407             HeaderFile hf = headerMap.computeIfAbsent(header.path(), p -> getHeaderFile(p, null));
 408             hf.useLibraries(libraryNames, libraryPaths);
 409             hf.useCodeFactory(fn.apply(hf));
 410             logger.info(() -> "Processing header file " + header.path());
 411 
 412             header.declarations().stream()
 413                     .peek(decl -> logger.finest(
 414                         () -> "Cursor: " + decl.name() + "@" + decl.USR() + "?" + decl.isDeclaration()))
 415                     .forEach(decl -> processTree(decl, hf, fn));
 416         });
 417     }
 418 
 419     private Map<String, List<AsmCodeFactory>> getPkgCfMap() {
 420         final Map<String, List<AsmCodeFactory>> mapPkgCf = new HashMap<>();
 421         // Build the pkg to CodeFactory map
 422         headerMap.values().forEach(header -> {
 423             AsmCodeFactory cf = header.getCodeFactory();
 424             String pkg = header.pkgName;
 425             logger.config(() -> "File " + header + " is in package: " + pkg);
 426             if (cf == null) {
 427                 logger.config(() -> "File " + header + " code generation is not activated!");
 428                 return;
 429             }
 430             List<AsmCodeFactory> l = mapPkgCf.computeIfAbsent(pkg, k -> new ArrayList<>());
 431             l.add(cf);
 432             logger.config(() -> "Add cf " + cf + " to pkg " + pkg + ", size is now " + l.size());
 433         });
 434         return Collections.unmodifiableMap(mapPkgCf);
 435     }
 436 
 437     public Map<String, byte[]> collectClasses(String... pkgs) {
 438         final Map<String, byte[]> rv = new HashMap<>();
 439         final Map<String, List<AsmCodeFactory>> mapPkgCf = getPkgCfMap();
 440         for (String pkg_name : pkgs) {
 441             mapPkgCf.getOrDefault(pkg_name, Collections.emptyList())
 442                     .forEach(cf -> rv.putAll(cf.collect()));
 443         }
 444         return Collections.unmodifiableMap(rv);
 445     }
 446 
 447     private static final String JEXTRACT_MANIFEST = "META-INFO" + File.separatorChar + "jextract.properties";
 448 
 449     @SuppressWarnings("deprecation")
 450     private byte[] getJextractProperties(String[] args) {
 451         Properties props = new Properties();
 452         props.setProperty("os.name", System.getProperty("os.name"));
 453         props.setProperty("os.version", System.getProperty("os.version"));
 454         props.setProperty("os.arch", System.getProperty("os.arch"));
 455         props.setProperty("jextract.args", Arrays.toString(args));
 456         ByteArrayOutputStream baos = new ByteArrayOutputStream();
 457         props.save(baos, "jextract meta data");
 458         return baos.toByteArray();
 459     }
 460 
 461     void collectClassFiles(Path destDir, String[] args, String... pkgs) throws IOException {
 462         try {
 463             collectClasses(pkgs).entrySet().stream().forEach(e -> {
 464                 try {
 465                     String path = e.getKey().replace('.', File.separatorChar) + ".class";
 466                     logger.fine(() -> "Writing " + path);
 467                     Path fullPath = destDir.resolve(path).normalize();
 468                     Files.createDirectories(fullPath.getParent());
 469                     try (OutputStream fos = Files.newOutputStream(fullPath)) {
 470                         fos.write(e.getValue());
 471                         fos.flush();
 472                     }
 473                 } catch (IOException ioe) {
 474                     throw new UncheckedIOException(ioe);
 475                 }
 476             });
 477 
 478             Path propsPath = destDir.resolve(JEXTRACT_MANIFEST).normalize();
 479             Files.createDirectories(propsPath.getParent());
 480             try (OutputStream fos = Files.newOutputStream(propsPath)) {
 481                 fos.write(getJextractProperties(args));
 482                 fos.flush();
 483             }
 484         } catch (UncheckedIOException uioe) {
 485             throw uioe.getCause();
 486         }
 487     }
 488 
 489     private void writeJar(AsmCodeFactory cf, JarOutputStream jar) {
 490         cf.collect().entrySet().stream().forEach(e -> {
 491             try {
 492                 String path = e.getKey().replace('.', File.separatorChar) + ".class";
 493                 logger.fine(() -> "Add " + path);
 494                 jar.putNextEntry(new ZipEntry(path));
 495                 jar.write(e.getValue());
 496                 jar.closeEntry();
 497             } catch (IOException ioe) {
 498                 throw new UncheckedIOException(ioe);
 499             }
 500         });
 501     }
 502 
 503     public void collectJarFile(final JarOutputStream jos, String[] args, String... pkgs) {
 504         final Map<String, List<AsmCodeFactory>> mapPkgCf = getPkgCfMap();
 505 
 506         for (String pkg_name : pkgs) {
 507             // convert '.' to '/' to use as a path
 508             String entryName = Utils.toInternalName(pkg_name, "");
 509             // package folder
 510             if (!entryName.isEmpty()) {
 511                 try {
 512                     jos.putNextEntry(new ZipEntry(entryName));
 513                 } catch (IOException ex) {
 514                     throw new UncheckedIOException(ex);
 515                 }
 516             }
 517             logger.fine(() -> "Produce for package " + pkg_name);
 518             mapPkgCf.getOrDefault(pkg_name, Collections.emptyList())
 519                     .forEach(cf -> writeJar(cf, jos));
 520         }
 521 
 522         try {
 523             jos.putNextEntry(new ZipEntry(JEXTRACT_MANIFEST));
 524             jos.write(getJextractProperties(args));
 525             jos.closeEntry();
 526         } catch (IOException ioe) {
 527             throw new UncheckedIOException(ioe);
 528         }
 529     }
 530 
 531     void collectJarFile(final Path jar, String[] args, String... pkgs) throws IOException {
 532         logger.info(() -> "Collecting jar file " + jar);
 533         try (OutputStream os = Files.newOutputStream(jar, CREATE, TRUNCATE_EXISTING, WRITE);
 534                 JarOutputStream jo = new JarOutputStream(os)) {
 535             collectJarFile(jo, args, pkgs);
 536         } catch (UncheckedIOException uioe) {
 537             throw uioe.getCause();
 538         }
 539     }
 540 
 541     /**
 542      * Perform a local lookup, any undefined type will cause a JType
 543      * be defined within origin scope.
 544      *
 545      * @param type   The libclang type
 546      * @param origin The path of the file where type is encountered
 547      * @return The JType
 548      */
 549     JType getJType(final Type type, Path origin) {
 550         Path p = origin.normalize().toAbsolutePath();
 551 
 552         HeaderFile hf = headerMap.get(p);
 553         // We should not encounter a type if the header file reference to it is not yet processed
 554         assert(null != hf);
 555         if (hf == null) {
 556             throw new IllegalArgumentException("Failed to lookup header for " + p + " (origin: " + origin + ")");
 557         }
 558 
 559         return hf.localLookup(type);
 560     }
 561 
 562     /**
 563      * Perform a global lookup
 564      *
 565      * @param c The cursor define or declare the type.
 566      * @return
 567      */
 568     JType getJType(final Cursor c) {
 569         if (c.isInvalid()) {
 570             throw new IllegalArgumentException();
 571         }
 572         SourceLocation loc = c.getSourceLocation();
 573         if (null == loc) {
 574             return null;
 575         }
 576         Path p = loc.getFileLocation().path();
 577         if (null == p) {
 578             return null;
 579         }
 580         return getJType(c.type(), p);
 581     }
 582 }
--- EOF ---