1 /*
   2  * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 package com.sun.tools.jextract;
  24 
  25 import jdk.internal.clang.*;
  26 
  27 import java.io.ByteArrayOutputStream;
  28 import java.io.File;
  29 import java.io.IOException;
  30 import java.io.OutputStream;
  31 import java.io.PrintWriter;
  32 import java.io.UncheckedIOException;
  33 import java.lang.invoke.MethodHandles;
  34 import java.lang.invoke.MethodHandles.Lookup;
  35 import java.foreign.Library;
  36 import java.foreign.Libraries;
  37 import java.nio.file.Files;
  38 import java.nio.file.Path;
  39 import java.nio.file.Paths;
  40 import java.util.ArrayList;
  41 import java.util.Arrays;
  42 import java.util.Collections;
  43 import java.util.HashMap;
  44 import java.util.List;
  45 import java.util.Map;
  46 import java.util.Optional;
  47 import java.util.Properties;
  48 import java.util.Set;
  49 import java.util.TreeSet;
  50 import java.util.function.Function;
  51 import java.util.function.Predicate;
  52 import java.util.jar.JarOutputStream;
  53 import java.util.logging.Logger;
  54 import java.util.regex.Pattern;
  55 import java.util.stream.Collectors;
  56 import java.util.zip.ZipEntry;
  57 import com.sun.tools.jextract.parser.Parser;
  58 import com.sun.tools.jextract.tree.FunctionTree;
  59 import com.sun.tools.jextract.tree.HeaderTree;
  60 import com.sun.tools.jextract.tree.Tree;
  61 
  62 import static java.nio.file.StandardOpenOption.CREATE;
  63 import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
  64 import static java.nio.file.StandardOpenOption.WRITE;
  65 
  66 /**
  67  * The setup for the tool execution
  68  */
  69 public final class Context {
  70     // package name to TypeDictionary
  71     private final Map<String, TypeDictionary> tdMap;
  72     // The folder path mapping to package name
  73     private final Map<Path, String> pkgMap;
  74     // The header file parsed
  75     private final Map<Path, HeaderFile> headerMap;
  76     // The args for parsing C
  77     private final List<String> clangArgs;
  78     // The set of source header files
  79     private final Set<Path>  sources;
  80     // The list of library names
  81     private final List<String> libraryNames;
  82     // The list of library paths
  83     private final List<String> libraryPaths;
  84     // The list of library paths for link checks
  85     private final List<String> linkCheckPaths;
  86     // Symbol patterns to be excluded
  87     private final List<Pattern> excludeSymbols;
  88     // generate static forwarder class or not?
  89     private boolean genStaticForwarder;
  90 
  91     final PrintWriter out;
  92     final PrintWriter err;
  93 
  94     private Predicate<String> symChecker;
  95     private Predicate<String> symFilter;
  96 
  97     private final Parser parser;
  98 
  99     private final static String defaultPkg = "jextract.dump";
 100     final Logger logger = Logger.getLogger(getClass().getPackage().getName());
 101 
 102     public Context(PrintWriter out, PrintWriter err) {
 103         this.tdMap = new HashMap<>();
 104         this.pkgMap = new HashMap<>();
 105         this.headerMap = new HashMap<>();
 106         this.clangArgs = new ArrayList<>();
 107         this.sources = new TreeSet<>();
 108         this.libraryNames = new ArrayList<>();
 109         this.libraryPaths = new ArrayList<>();
 110         this.linkCheckPaths = new ArrayList<>();
 111         this.excludeSymbols = new ArrayList<>();
 112         this.parser = new Parser(out, err, Main.INCLUDE_MACROS);
 113         this.out = out;
 114         this.err = err;
 115     }
 116 
 117     public Context() {
 118         this(new PrintWriter(System.out, true), new PrintWriter(System.err, true));
 119     }
 120 
 121     TypeDictionary typeDictionaryFor(String pkg) {
 122         return tdMap.computeIfAbsent(pkg, p->new TypeDictionary(this, p));
 123     }
 124 
 125     void addClangArg(String arg) {
 126         clangArgs.add(arg);
 127     }
 128 
 129     public void addSource(Path path) {
 130         sources.add(path);
 131     }
 132 
 133     void addLibraryName(String name) {
 134         libraryNames.add(name);
 135     }
 136 
 137     void addLibraryPath(String path) {
 138         libraryPaths.add(path);
 139     }
 140 
 141     void addLinkCheckPath(String path) {
 142         linkCheckPaths.add(path);
 143     }
 144 
 145     void addExcludeSymbols(String pattern) {
 146         excludeSymbols.add(Pattern.compile(pattern));
 147     }
 148 
 149     void setGenStaticForwarder(boolean flag) {
 150         this.genStaticForwarder = flag;
 151     }
 152 
 153     boolean getGenStaticForwarder() {
 154         return genStaticForwarder;
 155     }
 156 
 157     // return the absolute path of the library of given name by searching
 158     // in the given array of paths.
 159     private static Optional<Path> findLibraryPath(Path[] paths, String libName) {
 160          return Arrays.stream(paths).
 161               map(p -> p.resolve(System.mapLibraryName(libName))).
 162               filter(Files::isRegularFile).map(Path::toAbsolutePath).findFirst();
 163     }
 164 
 165     /*
 166      * Load the specified shared libraries from the specified paths.
 167      *
 168      * @param lookup Lookup object of the caller.
 169      * @param pathStrs array of paths to load the shared libraries from.
 170      * @param names array of shared library names.
 171      */
 172     // used by jextract tool to load libraries for symbol checks.
 173     public static Library[] loadLibraries(Lookup lookup, String[] pathStrs, String[] names) {
 174         if (pathStrs == null || pathStrs.length == 0) {
 175             return Arrays.stream(names).map(
 176                 name -> Libraries.loadLibrary(lookup, name)).toArray(Library[]::new);
 177         } else {
 178             Path[] paths = Arrays.stream(pathStrs).map(Paths::get).toArray(Path[]::new);
 179             return Arrays.stream(names).map(libName -> {
 180                 Optional<Path> absPath = findLibraryPath(paths, libName);
 181                 return absPath.isPresent() ?
 182                     Libraries.load(lookup, absPath.get().toString()) :
 183                     Libraries.loadLibrary(lookup, libName);
 184             }).toArray(Library[]::new);
 185         }
 186     }
 187 
 188     private void initSymChecker() {
 189         if (!libraryNames.isEmpty() && !linkCheckPaths.isEmpty()) {
 190             try {
 191                 Library[] libs = loadLibraries(MethodHandles.lookup(),
 192                     linkCheckPaths.toArray(new String[0]),
 193                     libraryNames.toArray(new String[0]));
 194                 // check if the given symbol is found in any of the libraries or not.
 195                 // If not found, warn the user for the missing symbol.
 196                 symChecker = name -> {
 197                     if (Main.DEBUG) {
 198                         err.println("Searching symbol: " + name);
 199                     }
 200                     return (Arrays.stream(libs).filter(lib -> {
 201                             try {
 202                                 lib.lookup(name);
 203                                 if (Main.DEBUG) {
 204                                     err.println("Found symbol: " + name);
 205                                 }
 206                                 return true;
 207                             } catch (NoSuchMethodException nsme) {
 208                                 return false;
 209                             }
 210                         }).findFirst().isPresent());
 211                 };
 212             } catch (UnsatisfiedLinkError ex) {
 213                 err.println(Main.format("warn.lib.not.found"));
 214                 symChecker = null;
 215             }
 216         } else {
 217             symChecker = null;
 218         }
 219     }
 220 
 221     private boolean isSymbolFound(String name) {
 222         return symChecker == null? true : symChecker.test(name);
 223     }
 224 
 225     private void initSymFilter() {
 226         if (!excludeSymbols.isEmpty()) {
 227             Pattern[] pats = excludeSymbols.toArray(new Pattern[0]);
 228             symFilter = name -> {
 229                 return Arrays.stream(pats).filter(pat -> pat.matcher(name).matches()).
 230                     findFirst().isPresent();
 231             };
 232         } else {
 233             symFilter = null;
 234         }
 235     }
 236 
 237     private boolean isSymbolExcluded(String name) {
 238         return symFilter == null? false : symFilter.test(name);
 239     }
 240 
 241     /**
 242      * Setup a package name for a given folder.
 243      *
 244      * @param folder The path to the folder, use null to set catch-all.
 245      * @param pkg    The package name
 246      * @return True if the folder is setup successfully. False is a package
 247      * has been assigned for the folder.
 248      */
 249     public boolean usePackageForFolder(Path folder, String pkg) {
 250         if (folder != null) {
 251             folder = folder.toAbsolutePath();
 252             if (!Files.isDirectory(folder)) {
 253                 folder = folder.getParent();
 254             }
 255         }
 256         String existing = pkgMap.putIfAbsent(folder, pkg);
 257         final String finalFolder = (null == folder) ? "all folders not configured" : folder.toString();
 258         if (null == existing) {
 259             logger.config(() -> "Package " + pkg + " is selected for " + finalFolder);
 260             return true;
 261         } else {
 262             logger.warning(() -> "Package " + existing + " had been selected for " + finalFolder + ", request to use " + pkg + " is ignored.");
 263             return false;
 264         }
 265     }
 266 
 267     static class Entity {
 268         final String pkg;
 269         final String entity;
 270 
 271         Entity(String pkg, String entity) {
 272             this.pkg = pkg;
 273             this.entity = entity;
 274         }
 275     }
 276 
 277     /**
 278      * Determine package and interface name given a path. If the path is
 279      * a folder, then only package name is determined. The package name is
 280      * determined with the longest path matching the setup. If the path is not
 281      * setup for any package, the default package name is returned.
 282      *
 283      * @param origin The source path
 284      * @return The Entity
 285      * @see Context::usePackageForFolder(Path, String)
 286      */
 287     Entity whatis(Path origin) {
 288         // normalize to absolute path
 289         origin = origin.toAbsolutePath();
 290         String filename = null;
 291         if (!Files.isDirectory(origin)) {
 292             // ensure it's a folder name
 293             filename = origin.getFileName().toString();
 294             origin = origin.getParent();
 295         }
 296         Path path = origin;
 297 
 298         // search the map for a hit with longest path
 299         while (path != null && !pkgMap.containsKey(path)) {
 300             path = path.getParent();
 301         }
 302 
 303         int start;
 304         String pkg;
 305         if (path != null) {
 306             start = path.getNameCount();
 307             pkg = pkgMap.get(path);
 308         } else {
 309             pkg = pkgMap.get(null);
 310             if (pkg == null) {
 311                 start = 0;
 312                 pkg = defaultPkg;
 313             } else {
 314                 start = origin.getNameCount();
 315             }
 316         }
 317 
 318         if (filename == null) {
 319             // a folder, only pkg name matters
 320             return new Entity(pkg, null);
 321         }
 322 
 323         StringBuilder sb = new StringBuilder();
 324         while (start < origin.getNameCount()) {
 325             sb.append(Utils.toJavaIdentifier(origin.getName(start++).toString()));
 326             sb.append("_");
 327         }
 328 
 329         int ext = filename.lastIndexOf('.');
 330         if (ext != -1) {
 331             sb.append(filename.substring(0, ext));
 332         } else {
 333             sb.append(filename);
 334         }
 335         return new Entity(pkg, Utils.toClassName(sb.toString()));
 336     }
 337 
 338     HeaderFile getHeaderFile(Path header, HeaderFile main) {
 339         if (!Files.isRegularFile(header)) {
 340             logger.warning(() -> "Not a regular file: " + header.toString());
 341             throw new IllegalArgumentException(header.toString());
 342         }
 343 
 344         final Context.Entity e = whatis(header);
 345         HeaderFile headerFile = new HeaderFile(this, header, e.pkg, e.entity, main);
 346         headerFile.useLibraries(libraryNames, libraryPaths);
 347         return headerFile;
 348     }
 349 
 350     void processTree(Tree tree, HeaderFile main, Function<HeaderFile, AsmCodeFactory> fn) {
 351         SourceLocation loc = tree.location();
 352 
 353         HeaderFile header;
 354         boolean isBuiltIn = false;
 355 
 356         if (tree.isFromMain()) {
 357             header = main;
 358         } else {
 359             SourceLocation.Location src = loc.getFileLocation();
 360             if (src == null) {
 361                 logger.info(() -> "Tree " + tree.name() + "@" + tree.USR() + " has no FileLocation");
 362                 return;
 363             }
 364 
 365             Path p = src.path();
 366             if (p == null) {
 367                 logger.fine(() -> "Found built-in type: " + tree.name());
 368                 header = main;
 369                 isBuiltIn = true;
 370             } else {
 371                 p = p.normalize().toAbsolutePath();
 372                 header = headerMap.get(p);
 373                 if (header == null) {
 374                     final HeaderFile hf = header = getHeaderFile(p, main);
 375                     logger.config(() -> "First encounter of header file " + hf.path + ", assigned to package " + hf.pkgName);
 376                     // Only generate code for header files specified or in the same package
 377                     if (sources.contains(p) ||
 378                         (header.pkgName.equals(main.pkgName))) {
 379                         logger.config("Code gen for header " + p + " enabled in package " + header.pkgName);
 380                         header.useCodeFactory(fn.apply(header));
 381                     }
 382                     headerMap.put(p, header);
 383                 }
 384             }
 385         }
 386 
 387         header.processTree(tree, main, isBuiltIn);
 388     }
 389 
 390     public void parse() {
 391         parse(header -> new AsmCodeFactory(this, header));
 392     }
 393 
 394     private boolean symbolFilter(Tree tree) {
 395          String name = tree.name();
 396          if (isSymbolExcluded(name)) {
 397              return false;
 398          }
 399 
 400          // check for function symbols in libraries & warn missing symbols
 401          if (tree instanceof FunctionTree && !isSymbolFound(name)) {
 402              err.println(Main.format("warn.symbol.not.found", name));
 403              //auto-exclude symbols not found
 404              return false;
 405          }
 406 
 407          return true;
 408     }
 409 
 410     public void parse(Function<HeaderFile, AsmCodeFactory> fn) {
 411         initSymChecker();
 412         initSymFilter();
 413 
 414         List<HeaderTree> headers = parser.parse(sources, clangArgs);
 415         processHeaders(headers, fn);
 416     }
 417 
 418     private void processHeaders(List<HeaderTree> headers, Function<HeaderFile, AsmCodeFactory> fn) {
 419         headers.stream().
 420                 map(new TreeFilter(this::symbolFilter)).
 421                 map(new TypedefHandler()).
 422                 map(new EmptyNameHandler()).
 423                 forEach(header -> {
 424             HeaderFile hf = headerMap.computeIfAbsent(header.path(), p -> getHeaderFile(p, null));
 425             hf.useCodeFactory(fn.apply(hf));
 426             logger.info(() -> "Processing header file " + header.path());
 427 
 428             header.declarations().stream()
 429                     .peek(decl -> logger.finest(
 430                         () -> "Cursor: " + decl.name() + "@" + decl.USR() + "?" + decl.isDeclaration()))
 431                     .forEach(decl -> processTree(decl, hf, fn));
 432         });
 433     }
 434 
 435     private Map<String, List<AsmCodeFactory>> getPkgCfMap() {
 436         final Map<String, List<AsmCodeFactory>> mapPkgCf = new HashMap<>();
 437         // Build the pkg to CodeFactory map
 438         headerMap.values().forEach(header -> {
 439             AsmCodeFactory cf = header.getCodeFactory();
 440             String pkg = header.pkgName;
 441             logger.config(() -> "File " + header + " is in package: " + pkg);
 442             if (cf == null) {
 443                 logger.config(() -> "File " + header + " code generation is not activated!");
 444                 return;
 445             }
 446             List<AsmCodeFactory> l = mapPkgCf.computeIfAbsent(pkg, k -> new ArrayList<>());
 447             l.add(cf);
 448             logger.config(() -> "Add cf " + cf + " to pkg " + pkg + ", size is now " + l.size());
 449         });
 450         return Collections.unmodifiableMap(mapPkgCf);
 451     }
 452 
 453     public Map<String, byte[]> collectClasses(String... pkgs) {
 454         final Map<String, byte[]> rv = new HashMap<>();
 455         final Map<String, List<AsmCodeFactory>> mapPkgCf = getPkgCfMap();
 456         for (String pkg_name : pkgs) {
 457             mapPkgCf.getOrDefault(pkg_name, Collections.emptyList())
 458                     .forEach(cf -> rv.putAll(cf.collect()));
 459         }
 460         return Collections.unmodifiableMap(rv);
 461     }
 462 
 463     private static final String JEXTRACT_MANIFEST = "META-INFO" + File.separatorChar + "jextract.properties";
 464 
 465     @SuppressWarnings("deprecation")
 466     private byte[] getJextractProperties(String[] args) {
 467         Properties props = new Properties();
 468         props.setProperty("os.name", System.getProperty("os.name"));
 469         props.setProperty("os.version", System.getProperty("os.version"));
 470         props.setProperty("os.arch", System.getProperty("os.arch"));
 471         props.setProperty("jextract.args", Arrays.toString(args));
 472         ByteArrayOutputStream baos = new ByteArrayOutputStream();
 473         props.save(baos, "jextract meta data");
 474         return baos.toByteArray();
 475     }
 476 
 477     void collectClassFiles(Path destDir, String[] args, String... pkgs) throws IOException {
 478         try {
 479             collectClasses(pkgs).entrySet().stream().forEach(e -> {
 480                 try {
 481                     String path = e.getKey().replace('.', File.separatorChar) + ".class";
 482                     logger.fine(() -> "Writing " + path);
 483                     Path fullPath = destDir.resolve(path).normalize();
 484                     Files.createDirectories(fullPath.getParent());
 485                     try (OutputStream fos = Files.newOutputStream(fullPath)) {
 486                         fos.write(e.getValue());
 487                         fos.flush();
 488                     }
 489                 } catch (IOException ioe) {
 490                     throw new UncheckedIOException(ioe);
 491                 }
 492             });
 493 
 494             Path propsPath = destDir.resolve(JEXTRACT_MANIFEST).normalize();
 495             Files.createDirectories(propsPath.getParent());
 496             try (OutputStream fos = Files.newOutputStream(propsPath)) {
 497                 fos.write(getJextractProperties(args));
 498                 fos.flush();
 499             }
 500         } catch (UncheckedIOException uioe) {
 501             throw uioe.getCause();
 502         }
 503     }
 504 
 505     private void writeJar(AsmCodeFactory cf, JarOutputStream jar) {
 506         cf.collect().entrySet().stream().forEach(e -> {
 507             try {
 508                 String path = e.getKey().replace('.', File.separatorChar) + ".class";
 509                 logger.fine(() -> "Add " + path);
 510                 jar.putNextEntry(new ZipEntry(path));
 511                 jar.write(e.getValue());
 512                 jar.closeEntry();
 513             } catch (IOException ioe) {
 514                 throw new UncheckedIOException(ioe);
 515             }
 516         });
 517     }
 518 
 519     public void collectJarFile(final JarOutputStream jos, String[] args, String... pkgs) {
 520         final Map<String, List<AsmCodeFactory>> mapPkgCf = getPkgCfMap();
 521 
 522         for (String pkg_name : pkgs) {
 523             // convert '.' to '/' to use as a path
 524             String entryName = Utils.toInternalName(pkg_name, "");
 525             // package folder
 526             if (!entryName.isEmpty()) {
 527                 try {
 528                     jos.putNextEntry(new ZipEntry(entryName));
 529                 } catch (IOException ex) {
 530                     throw new UncheckedIOException(ex);
 531                 }
 532             }
 533             logger.fine(() -> "Produce for package " + pkg_name);
 534             mapPkgCf.getOrDefault(pkg_name, Collections.emptyList())
 535                     .forEach(cf -> writeJar(cf, jos));
 536         }
 537 
 538         try {
 539             jos.putNextEntry(new ZipEntry(JEXTRACT_MANIFEST));
 540             jos.write(getJextractProperties(args));
 541             jos.closeEntry();
 542         } catch (IOException ioe) {
 543             throw new UncheckedIOException(ioe);
 544         }
 545     }
 546 
 547     void collectJarFile(final Path jar, String[] args, String... pkgs) throws IOException {
 548         logger.info(() -> "Collecting jar file " + jar);
 549         try (OutputStream os = Files.newOutputStream(jar, CREATE, TRUNCATE_EXISTING, WRITE);
 550                 JarOutputStream jo = new JarOutputStream(os)) {
 551             collectJarFile(jo, args, pkgs);
 552         } catch (UncheckedIOException uioe) {
 553             throw uioe.getCause();
 554         }
 555     }
 556 
 557     /**
 558      * Perform a local lookup, any undefined type will cause a JType
 559      * be defined within origin scope.
 560      *
 561      * @param type   The libclang type
 562      * @param origin The path of the file where type is encountered
 563      * @return The JType
 564      */
 565     JType getJType(final Type type, Path origin) {
 566         Path p = origin.normalize().toAbsolutePath();
 567 
 568         HeaderFile hf = headerMap.get(p);
 569         // We should not encounter a type if the header file reference to it is not yet processed
 570         assert(null != hf);
 571         if (hf == null) {
 572             throw new IllegalArgumentException("Failed to lookup header for " + p + " (origin: " + origin + ")");
 573         }
 574 
 575         return hf.localLookup(type);
 576     }
 577 
 578     /**
 579      * Perform a global lookup
 580      *
 581      * @param c The cursor define or declare the type.
 582      * @return
 583      */
 584     JType getJType(final Cursor c) {
 585         if (c.isInvalid()) {
 586             throw new IllegalArgumentException();
 587         }
 588         SourceLocation loc = c.getSourceLocation();
 589         if (null == loc) {
 590             return null;
 591         }
 592         Path p = loc.getFileLocation().path();
 593         if (null == p) {
 594             return null;
 595         }
 596         return getJType(c.type(), p);
 597     }
 598 }
--- EOF ---