1 /*
   2  * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 package com.sun.tools.jextract;
  24 
  25 import jdk.internal.clang.*;
  26 
  27 import java.io.ByteArrayOutputStream;
  28 import java.io.File;
  29 import java.io.IOException;
  30 import java.io.OutputStream;
  31 import java.io.PrintWriter;
  32 import java.io.UncheckedIOException;
  33 import java.lang.invoke.MethodHandles;
  34 import java.lang.invoke.MethodHandles.Lookup;
  35 import java.foreign.Library;
  36 import java.foreign.Libraries;
  37 import java.nio.file.Files;
  38 import java.nio.file.Path;
  39 import java.nio.file.Paths;
  40 import java.util.ArrayList;
  41 import java.util.Arrays;
  42 import java.util.Collections;
  43 import java.util.HashMap;
  44 import java.util.List;
  45 import java.util.Map;
  46 import java.util.Optional;
  47 import java.util.Properties;
  48 import java.util.Set;
  49 import java.util.TreeSet;
  50 import java.util.function.Function;
  51 import java.util.function.Predicate;
  52 import java.util.jar.JarOutputStream;
  53 import java.util.logging.Logger;
  54 import java.util.regex.Pattern;
  55 import java.util.stream.Collectors;
  56 import java.util.zip.ZipEntry;
  57 import com.sun.tools.jextract.parser.Parser;
  58 import com.sun.tools.jextract.tree.FunctionTree;
  59 import com.sun.tools.jextract.tree.HeaderTree;
  60 import com.sun.tools.jextract.tree.Tree;
  61 
  62 import static java.nio.file.StandardOpenOption.CREATE;
  63 import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
  64 import static java.nio.file.StandardOpenOption.WRITE;
  65 
  66 /**
  67  * The setup for the tool execution
  68  */
  69 public final class Context {
  70     // package name to TypeDictionary
  71     private final Map<String, TypeDictionary> tdMap;
  72     // The folder path mapping to package name
  73     private final Map<Path, String> pkgMap;
  74     // The header file parsed
  75     private final Map<Path, HeaderFile> headerMap;
  76     // The args for parsing C
  77     private final List<String> clangArgs;
  78     // The set of source header files
  79     private final Set<Path>  sources;
  80     // The list of library names
  81     private final List<String> libraryNames;
  82     // The list of library paths
  83     private final List<String> libraryPaths;
  84     // The list of library paths for link checks
  85     private final List<String> linkCheckPaths;
  86     // Symbol patterns to be excluded
  87     private final List<Pattern> excludeSymbols;
  88 
  89     final PrintWriter out;
  90     final PrintWriter err;
  91 
  92     private Predicate<String> symChecker;
  93     private Predicate<String> symFilter;
  94 
  95     private final Parser parser;
  96 
  97     private final static String defaultPkg = "jextract.dump";
  98     final Logger logger = Logger.getLogger(getClass().getPackage().getName());
  99 
 100     public Context(PrintWriter out, PrintWriter err) {
 101         this.tdMap = new HashMap<>();
 102         this.pkgMap = new HashMap<>();
 103         this.headerMap = new HashMap<>();
 104         this.clangArgs = new ArrayList<>();
 105         this.sources = new TreeSet<>();
 106         this.libraryNames = new ArrayList<>();
 107         this.libraryPaths = new ArrayList<>();
 108         this.linkCheckPaths = new ArrayList<>();
 109         this.excludeSymbols = new ArrayList<>();
 110         this.parser = new Parser(out, err, Main.INCLUDE_MACROS);
 111         this.out = out;
 112         this.err = err;
 113     }
 114 
 115     public Context() {
 116         this(new PrintWriter(System.out, true), new PrintWriter(System.err, true));
 117     }
 118 
 119     TypeDictionary typeDictionaryFor(String pkg) {
 120         return tdMap.computeIfAbsent(pkg, p->new TypeDictionary(this, p));
 121     }
 122 
 123     void addClangArg(String arg) {
 124         clangArgs.add(arg);
 125     }
 126 
 127     public void addSource(Path path) {
 128         sources.add(path);
 129     }
 130 
 131     void addLibraryName(String name) {
 132         libraryNames.add(name);
 133     }
 134 
 135     void addLibraryPath(String path) {
 136         libraryPaths.add(path);
 137     }
 138 
 139     void addLinkCheckPath(String path) {
 140         linkCheckPaths.add(path);
 141     }
 142 
 143     void addExcludeSymbols(String pattern) {
 144         excludeSymbols.add(Pattern.compile(pattern));
 145     }
 146 
 147     // return the absolute path of the library of given name by searching
 148     // in the given array of paths.
 149     private static Optional<Path> findLibraryPath(Path[] paths, String libName) {
 150          return Arrays.stream(paths).
 151               map(p -> p.resolve(System.mapLibraryName(libName))).
 152               filter(Files::isRegularFile).map(Path::toAbsolutePath).findFirst();
 153     }
 154 
 155     /*
 156      * Load the specified shared libraries from the specified paths.
 157      *
 158      * @param lookup Lookup object of the caller.
 159      * @param pathStrs array of paths to load the shared libraries from.
 160      * @param names array of shared library names.
 161      */
 162     // used by jextract tool to load libraries for symbol checks.
 163     public static Library[] loadLibraries(Lookup lookup, String[] pathStrs, String[] names) {
 164         if (pathStrs == null || pathStrs.length == 0) {
 165             return Arrays.stream(names).map(
 166                 name -> Libraries.loadLibrary(lookup, name)).toArray(Library[]::new);
 167         } else {
 168             Path[] paths = Arrays.stream(pathStrs).map(Paths::get).toArray(Path[]::new);
 169             return Arrays.stream(names).map(libName -> {
 170                 Optional<Path> absPath = findLibraryPath(paths, libName);
 171                 return absPath.isPresent() ?
 172                     Libraries.load(lookup, absPath.get().toString()) :
 173                     Libraries.loadLibrary(lookup, libName);
 174             }).toArray(Library[]::new);
 175         }
 176     }
 177 
 178     private void initSymChecker() {
 179         if (!libraryNames.isEmpty() && !linkCheckPaths.isEmpty()) {
 180             try {
 181                 Library[] libs = loadLibraries(MethodHandles.lookup(),
 182                     linkCheckPaths.toArray(new String[0]),
 183                     libraryNames.toArray(new String[0]));
 184                 // check if the given symbol is found in any of the libraries or not.
 185                 // If not found, warn the user for the missing symbol.
 186                 symChecker = name -> {
 187                     if (Main.DEBUG) {
 188                         err.println("Searching symbol: " + name);
 189                     }
 190                     return (Arrays.stream(libs).filter(lib -> {
 191                             try {
 192                                 lib.lookup(name);
 193                                 if (Main.DEBUG) {
 194                                     err.println("Found symbol: " + name);
 195                                 }
 196                                 return true;
 197                             } catch (NoSuchMethodException nsme) {
 198                                 return false;
 199                             }
 200                         }).findFirst().isPresent());
 201                 };
 202             } catch (UnsatisfiedLinkError ex) {
 203                 err.println(Main.format("warn.lib.not.found"));
 204                 symChecker = null;
 205             }
 206         } else {
 207             symChecker = null;
 208         }
 209     }
 210 
 211     private boolean isSymbolFound(String name) {
 212         return symChecker == null? true : symChecker.test(name);
 213     }
 214 
 215     private void initSymFilter() {
 216         if (!excludeSymbols.isEmpty()) {
 217             Pattern[] pats = excludeSymbols.toArray(new Pattern[0]);
 218             symFilter = name -> {
 219                 return Arrays.stream(pats).filter(pat -> pat.matcher(name).matches()).
 220                     findFirst().isPresent();
 221             };
 222         } else {
 223             symFilter = null;
 224         }
 225     }
 226 
 227     private boolean isSymbolExcluded(String name) {
 228         return symFilter == null? false : symFilter.test(name);
 229     }
 230 
 231     /**
 232      * Setup a package name for a given folder.
 233      *
 234      * @param folder The path to the folder, use null to set catch-all.
 235      * @param pkg    The package name
 236      * @return True if the folder is setup successfully. False is a package
 237      * has been assigned for the folder.
 238      */
 239     public boolean usePackageForFolder(Path folder, String pkg) {
 240         if (folder != null) {
 241             folder = folder.toAbsolutePath();
 242             if (!Files.isDirectory(folder)) {
 243                 folder = folder.getParent();
 244             }
 245         }
 246         String existing = pkgMap.putIfAbsent(folder, pkg);
 247         final String finalFolder = (null == folder) ? "all folders not configured" : folder.toString();
 248         if (null == existing) {
 249             logger.config(() -> "Package " + pkg + " is selected for " + finalFolder);
 250             return true;
 251         } else {
 252             logger.warning(() -> "Package " + existing + " had been selected for " + finalFolder + ", request to use " + pkg + " is ignored.");
 253             return false;
 254         }
 255     }
 256 
 257     static class Entity {
 258         final String pkg;
 259         final String entity;
 260 
 261         Entity(String pkg, String entity) {
 262             this.pkg = pkg;
 263             this.entity = entity;
 264         }
 265     }
 266 
 267     /**
 268      * Determine package and interface name given a path. If the path is
 269      * a folder, then only package name is determined. The package name is
 270      * determined with the longest path matching the setup. If the path is not
 271      * setup for any package, the default package name is returned.
 272      *
 273      * @param origin The source path
 274      * @return The Entity
 275      * @see Context::usePackageForFolder(Path, String)
 276      */
 277     Entity whatis(Path origin) {
 278         // normalize to absolute path
 279         origin = origin.toAbsolutePath();
 280         String filename = null;
 281         if (!Files.isDirectory(origin)) {
 282             // ensure it's a folder name
 283             filename = origin.getFileName().toString();
 284             origin = origin.getParent();
 285         }
 286         Path path = origin;
 287 
 288         // search the map for a hit with longest path
 289         while (path != null && !pkgMap.containsKey(path)) {
 290             path = path.getParent();
 291         }
 292 
 293         int start;
 294         String pkg;
 295         if (path != null) {
 296             start = path.getNameCount();
 297             pkg = pkgMap.get(path);
 298         } else {
 299             pkg = pkgMap.get(null);
 300             if (pkg == null) {
 301                 start = 0;
 302                 pkg = defaultPkg;
 303             } else {
 304                 start = origin.getNameCount();
 305             }
 306         }
 307 
 308         if (filename == null) {
 309             // a folder, only pkg name matters
 310             return new Entity(pkg, null);
 311         }
 312 
 313         StringBuilder sb = new StringBuilder();
 314         while (start < origin.getNameCount()) {
 315             sb.append(Utils.toJavaIdentifier(origin.getName(start++).toString()));
 316             sb.append("_");
 317         }
 318 
 319         int ext = filename.lastIndexOf('.');
 320         if (ext != -1) {
 321             sb.append(filename.substring(0, ext));
 322         } else {
 323             sb.append(filename);
 324         }
 325         return new Entity(pkg, Utils.toClassName(sb.toString()));
 326     }
 327 
 328     HeaderFile getHeaderFile(Path header, HeaderFile main) {
 329         if (!Files.isRegularFile(header)) {
 330             logger.warning(() -> "Not a regular file: " + header.toString());
 331             throw new IllegalArgumentException(header.toString());
 332         }
 333 
 334         final Context.Entity e = whatis(header);
 335         HeaderFile headerFile = new HeaderFile(this, header, e.pkg, e.entity, main);
 336         headerFile.useLibraries(libraryNames, libraryPaths);
 337         return headerFile;
 338     }
 339 
 340     void processTree(Tree tree, HeaderFile main, Function<HeaderFile, AsmCodeFactory> fn) {
 341         SourceLocation loc = tree.location();
 342 
 343         HeaderFile header;
 344         boolean isBuiltIn = false;
 345 
 346         if (tree.isFromMain()) {
 347             header = main;
 348         } else {
 349             SourceLocation.Location src = loc.getFileLocation();
 350             if (src == null) {
 351                 logger.info(() -> "Tree " + tree.name() + "@" + tree.USR() + " has no FileLocation");
 352                 return;
 353             }
 354 
 355             Path p = src.path();
 356             if (p == null) {
 357                 logger.fine(() -> "Found built-in type: " + tree.name());
 358                 header = main;
 359                 isBuiltIn = true;
 360             } else {
 361                 p = p.normalize().toAbsolutePath();
 362                 header = headerMap.get(p);
 363                 if (header == null) {
 364                     final HeaderFile hf = header = getHeaderFile(p, main);
 365                     logger.config(() -> "First encounter of header file " + hf.path + ", assigned to package " + hf.pkgName);
 366                     // Only generate code for header files specified or in the same package
 367                     if (sources.contains(p) ||
 368                         (header.pkgName.equals(main.pkgName))) {
 369                         logger.config("Code gen for header " + p + " enabled in package " + header.pkgName);
 370                         header.useCodeFactory(fn.apply(header));
 371                     }
 372                     headerMap.put(p, header);
 373                 }
 374             }
 375         }
 376 
 377         header.processTree(tree, main, isBuiltIn);
 378     }
 379 
 380     public void parse() {
 381         parse(header -> new AsmCodeFactory(this, header));
 382     }
 383 
 384     private boolean symbolFilter(Tree tree) {
 385          String name = tree.name();
 386          if (isSymbolExcluded(name)) {
 387              return false;
 388          }
 389 
 390          // check for function symbols in libraries & warn missing symbols
 391          if (tree instanceof FunctionTree && !isSymbolFound(name)) {
 392              err.println(Main.format("warn.symbol.not.found", name));
 393              //auto-exclude symbols not found
 394              return false;
 395          }
 396 
 397          return true;
 398     }
 399 
 400     public void parse(Function<HeaderFile, AsmCodeFactory> fn) {
 401         initSymChecker();
 402         initSymFilter();
 403 
 404         List<HeaderTree> headers = parser.parse(sources, clangArgs);
 405         processHeaders(headers, fn);
 406     }
 407 
 408     private void processHeaders(List<HeaderTree> headers, Function<HeaderFile, AsmCodeFactory> fn) {
 409         headers.stream().
 410                 map(new TreeFilter(this::symbolFilter)).
 411                 map(new TypedefHandler()).
 412                 map(new EmptyNameHandler()).
 413                 forEach(header -> {
 414             HeaderFile hf = headerMap.computeIfAbsent(header.path(), p -> getHeaderFile(p, null));
 415             hf.useCodeFactory(fn.apply(hf));
 416             logger.info(() -> "Processing header file " + header.path());
 417 
 418             header.declarations().stream()
 419                     .peek(decl -> logger.finest(
 420                         () -> "Cursor: " + decl.name() + "@" + decl.USR() + "?" + decl.isDeclaration()))
 421                     .forEach(decl -> processTree(decl, hf, fn));
 422         });
 423     }
 424 
 425     private Map<String, List<AsmCodeFactory>> getPkgCfMap() {
 426         final Map<String, List<AsmCodeFactory>> mapPkgCf = new HashMap<>();
 427         // Build the pkg to CodeFactory map
 428         headerMap.values().forEach(header -> {
 429             AsmCodeFactory cf = header.getCodeFactory();
 430             String pkg = header.pkgName;
 431             logger.config(() -> "File " + header + " is in package: " + pkg);
 432             if (cf == null) {
 433                 logger.config(() -> "File " + header + " code generation is not activated!");
 434                 return;
 435             }
 436             List<AsmCodeFactory> l = mapPkgCf.computeIfAbsent(pkg, k -> new ArrayList<>());
 437             l.add(cf);
 438             logger.config(() -> "Add cf " + cf + " to pkg " + pkg + ", size is now " + l.size());
 439         });
 440         return Collections.unmodifiableMap(mapPkgCf);
 441     }
 442 
 443     public Map<String, byte[]> collectClasses(String... pkgs) {
 444         final Map<String, byte[]> rv = new HashMap<>();
 445         final Map<String, List<AsmCodeFactory>> mapPkgCf = getPkgCfMap();
 446         for (String pkg_name : pkgs) {
 447             mapPkgCf.getOrDefault(pkg_name, Collections.emptyList())
 448                     .forEach(cf -> rv.putAll(cf.collect()));
 449         }
 450         return Collections.unmodifiableMap(rv);
 451     }
 452 
 453     private static final String JEXTRACT_MANIFEST = "META-INFO" + File.separatorChar + "jextract.properties";
 454 
 455     @SuppressWarnings("deprecation")
 456     private byte[] getJextractProperties(String[] args) {
 457         Properties props = new Properties();
 458         props.setProperty("os.name", System.getProperty("os.name"));
 459         props.setProperty("os.version", System.getProperty("os.version"));
 460         props.setProperty("os.arch", System.getProperty("os.arch"));
 461         props.setProperty("jextract.args", Arrays.toString(args));
 462         ByteArrayOutputStream baos = new ByteArrayOutputStream();
 463         props.save(baos, "jextract meta data");
 464         return baos.toByteArray();
 465     }
 466 
 467     void collectClassFiles(Path destDir, String[] args, String... pkgs) throws IOException {
 468         try {
 469             collectClasses(pkgs).entrySet().stream().forEach(e -> {
 470                 try {
 471                     String path = e.getKey().replace('.', File.separatorChar) + ".class";
 472                     logger.fine(() -> "Writing " + path);
 473                     Path fullPath = destDir.resolve(path).normalize();
 474                     Files.createDirectories(fullPath.getParent());
 475                     try (OutputStream fos = Files.newOutputStream(fullPath)) {
 476                         fos.write(e.getValue());
 477                         fos.flush();
 478                     }
 479                 } catch (IOException ioe) {
 480                     throw new UncheckedIOException(ioe);
 481                 }
 482             });
 483 
 484             Path propsPath = destDir.resolve(JEXTRACT_MANIFEST).normalize();
 485             Files.createDirectories(propsPath.getParent());
 486             try (OutputStream fos = Files.newOutputStream(propsPath)) {
 487                 fos.write(getJextractProperties(args));
 488                 fos.flush();
 489             }
 490         } catch (UncheckedIOException uioe) {
 491             throw uioe.getCause();
 492         }
 493     }
 494 
 495     private void writeJar(AsmCodeFactory cf, JarOutputStream jar) {
 496         cf.collect().entrySet().stream().forEach(e -> {
 497             try {
 498                 String path = e.getKey().replace('.', File.separatorChar) + ".class";
 499                 logger.fine(() -> "Add " + path);
 500                 jar.putNextEntry(new ZipEntry(path));
 501                 jar.write(e.getValue());
 502                 jar.closeEntry();
 503             } catch (IOException ioe) {
 504                 throw new UncheckedIOException(ioe);
 505             }
 506         });
 507     }
 508 
 509     public void collectJarFile(final JarOutputStream jos, String[] args, String... pkgs) {
 510         final Map<String, List<AsmCodeFactory>> mapPkgCf = getPkgCfMap();
 511 
 512         for (String pkg_name : pkgs) {
 513             // convert '.' to '/' to use as a path
 514             String entryName = Utils.toInternalName(pkg_name, "");
 515             // package folder
 516             if (!entryName.isEmpty()) {
 517                 try {
 518                     jos.putNextEntry(new ZipEntry(entryName));
 519                 } catch (IOException ex) {
 520                     throw new UncheckedIOException(ex);
 521                 }
 522             }
 523             logger.fine(() -> "Produce for package " + pkg_name);
 524             mapPkgCf.getOrDefault(pkg_name, Collections.emptyList())
 525                     .forEach(cf -> writeJar(cf, jos));
 526         }
 527 
 528         try {
 529             jos.putNextEntry(new ZipEntry(JEXTRACT_MANIFEST));
 530             jos.write(getJextractProperties(args));
 531             jos.closeEntry();
 532         } catch (IOException ioe) {
 533             throw new UncheckedIOException(ioe);
 534         }
 535     }
 536 
 537     void collectJarFile(final Path jar, String[] args, String... pkgs) throws IOException {
 538         logger.info(() -> "Collecting jar file " + jar);
 539         try (OutputStream os = Files.newOutputStream(jar, CREATE, TRUNCATE_EXISTING, WRITE);
 540                 JarOutputStream jo = new JarOutputStream(os)) {
 541             collectJarFile(jo, args, pkgs);
 542         } catch (UncheckedIOException uioe) {
 543             throw uioe.getCause();
 544         }
 545     }
 546 
 547     /**
 548      * Perform a local lookup, any undefined type will cause a JType
 549      * be defined within origin scope.
 550      *
 551      * @param type   The libclang type
 552      * @param origin The path of the file where type is encountered
 553      * @return The JType
 554      */
 555     JType getJType(final Type type, Path origin) {
 556         Path p = origin.normalize().toAbsolutePath();
 557 
 558         HeaderFile hf = headerMap.get(p);
 559         // We should not encounter a type if the header file reference to it is not yet processed
 560         assert(null != hf);
 561         if (hf == null) {
 562             throw new IllegalArgumentException("Failed to lookup header for " + p + " (origin: " + origin + ")");
 563         }
 564 
 565         return hf.localLookup(type);
 566     }
 567 
 568     /**
 569      * Perform a global lookup
 570      *
 571      * @param c The cursor define or declare the type.
 572      * @return
 573      */
 574     JType getJType(final Cursor c) {
 575         if (c.isInvalid()) {
 576             throw new IllegalArgumentException();
 577         }
 578         SourceLocation loc = c.getSourceLocation();
 579         if (null == loc) {
 580             return null;
 581         }
 582         Path p = loc.getFileLocation().path();
 583         if (null == p) {
 584             return null;
 585         }
 586         return getJType(c.type(), p);
 587     }
 588 }