1 /*
   2  * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 package com.sun.tools.jextract;
  24 
  25 import jdk.internal.clang.*;
  26 import jdk.internal.foreign.LibrariesHelper;
  27 
  28 import java.io.File;
  29 import java.io.IOException;
  30 import java.io.OutputStream;
  31 import java.io.PrintWriter;
  32 import java.io.UncheckedIOException;
  33 import java.lang.invoke.MethodHandles;
  34 import java.foreign.Library;
  35 import java.nio.file.Files;
  36 import java.nio.file.Path;
  37 import java.util.ArrayList;
  38 import java.util.Arrays;
  39 import java.util.Collections;
  40 import java.util.HashMap;
  41 import java.util.List;
  42 import java.util.Map;
  43 import java.util.Set;
  44 import java.util.TreeSet;
  45 import java.util.function.Function;
  46 import java.util.function.Predicate;
  47 import java.util.jar.JarOutputStream;
  48 import java.util.logging.Logger;
  49 import java.util.regex.Pattern;
  50 import java.util.stream.Collectors;
  51 import java.util.zip.ZipEntry;
  52 
  53 import static java.nio.file.StandardOpenOption.CREATE;
  54 import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
  55 import static java.nio.file.StandardOpenOption.WRITE;
  56 
  57 /**
  58  * The setup for the tool execution
  59  */
  60 public final class Context {
  61     // package name to TypeDictionary
  62     private final Map<String, TypeDictionary> tdMap;
  63     // The folder path mapping to package name
  64     private final Map<Path, String> pkgMap;
  65     // The header file parsed
  66     private final Map<Path, HeaderFile> headerMap;
  67     // The args for parsing C
  68     private final List<String> clangArgs;
  69     // The set of source header files
  70     private final Set<Path>  sources;
  71     // The list of library names
  72     private final List<String> libraryNames;
  73     // The list of library paths
  74     private final List<String> libraryPaths;
  75     // The list of library paths for link checks
  76     private final List<String> linkCheckPaths;
  77     // Symbol patterns to be excluded
  78     private final List<Pattern> excludeSymbols;
  79 
  80     final PrintWriter out;
  81     final PrintWriter err;
  82 
  83     private Predicate<String> symChecker;
  84     private Predicate<String> symFilter;
  85 
  86     private final MacroParser macroParser;
  87 
  88     private final static String defaultPkg = "jextract.dump";
  89     final Logger logger = Logger.getLogger(getClass().getPackage().getName());
  90 
  91     public Context(PrintWriter out, PrintWriter err) {
  92         this.tdMap = new HashMap<>();
  93         this.pkgMap = new HashMap<>();
  94         this.headerMap = new HashMap<>();
  95         this.clangArgs = new ArrayList<>();
  96         this.sources = new TreeSet<>();
  97         this.libraryNames = new ArrayList<>();
  98         this.libraryPaths = new ArrayList<>();
  99         this.linkCheckPaths = new ArrayList<>();
 100         this.excludeSymbols = new ArrayList<>();
 101         this.macroParser = new MacroParser();
 102         this.out = out;
 103         this.err = err;
 104     }
 105 
 106     public Context() {
 107         this(new PrintWriter(System.out, true), new PrintWriter(System.err, true));
 108     }
 109 
 110     TypeDictionary typeDictionaryFor(String pkg) {
 111         return tdMap.computeIfAbsent(pkg, p->new TypeDictionary(this, p));
 112     }
 113 
 114     void addClangArg(String arg) {
 115         clangArgs.add(arg);
 116     }
 117 
 118     public void addSource(Path path) {
 119         sources.add(path);
 120     }
 121 
 122     void addLibraryName(String name) {
 123         libraryNames.add(name);
 124     }
 125 
 126     void addLibraryPath(String path) {
 127         libraryPaths.add(path);
 128     }
 129 
 130     void addLinkCheckPath(String path) {
 131         linkCheckPaths.add(path);
 132     }
 133 
 134     void addExcludeSymbols(String pattern) {
 135         excludeSymbols.add(Pattern.compile(pattern));
 136     }
 137 
 138     boolean isSymbolFound(String name) {
 139         return symChecker == null? true : symChecker.test(name);
 140     }
 141 
 142     boolean isSymbolExcluded(String name) {
 143         return symFilter == null? false : symFilter.test(name);
 144     }
 145 
 146     /**
 147      * Setup a package name for a given folder.
 148      *
 149      * @param folder The path to the folder, use null to set catch-all.
 150      * @param pkg    The package name
 151      * @return True if the folder is setup successfully. False is a package
 152      * has been assigned for the folder.
 153      */
 154     public boolean usePackageForFolder(Path folder, String pkg) {
 155         if (folder != null) {
 156             folder = folder.toAbsolutePath();
 157             if (!Files.isDirectory(folder)) {
 158                 folder = folder.getParent();
 159             }
 160         }
 161         String existing = pkgMap.putIfAbsent(folder, pkg);
 162         final String finalFolder = (null == folder) ? "all folders not configured" : folder.toString();
 163         if (null == existing) {
 164             logger.config(() -> "Package " + pkg + " is selected for " + finalFolder);
 165             return true;
 166         } else {
 167             logger.warning(() -> "Package " + existing + " had been selected for " + finalFolder + ", request to use " + pkg + " is ignored.");
 168             return false;
 169         }
 170     }
 171 
 172     static class Entity {
 173         final String pkg;
 174         final String entity;
 175 
 176         Entity(String pkg, String entity) {
 177             this.pkg = pkg;
 178             this.entity = entity;
 179         }
 180     }
 181 
 182     /**
 183      * Determine package and interface name given a path. If the path is
 184      * a folder, then only package name is determined. The package name is
 185      * determined with the longest path matching the setup. If the path is not
 186      * setup for any package, the default package name is returned.
 187      *
 188      * @param origin The source path
 189      * @return The Entity
 190      * @see Context::usePackageForFolder(Path, String)
 191      */
 192     Entity whatis(Path origin) {
 193         // normalize to absolute path
 194         origin = origin.toAbsolutePath();
 195         String filename = null;
 196         if (!Files.isDirectory(origin)) {
 197             // ensure it's a folder name
 198             filename = origin.getFileName().toString();
 199             origin = origin.getParent();
 200         }
 201         Path path = origin;
 202 
 203         // search the map for a hit with longest path
 204         while (path != null && !pkgMap.containsKey(path)) {
 205             path = path.getParent();
 206         }
 207 
 208         int start;
 209         String pkg;
 210         if (path != null) {
 211             start = path.getNameCount();
 212             pkg = pkgMap.get(path);
 213         } else {
 214             pkg = pkgMap.get(null);
 215             if (pkg == null) {
 216                 start = 0;
 217                 pkg = defaultPkg;
 218             } else {
 219                 start = origin.getNameCount();
 220             }
 221         }
 222 
 223         if (filename == null) {
 224             // a folder, only pkg name matters
 225             return new Entity(pkg, null);
 226         }
 227 
 228         StringBuilder sb = new StringBuilder();
 229         while (start < origin.getNameCount()) {
 230             sb.append(Utils.toJavaIdentifier(origin.getName(start++).toString()));
 231             sb.append("_");
 232         }
 233 
 234         int ext = filename.lastIndexOf('.');
 235         if (ext != -1) {
 236             sb.append(filename.substring(0, ext));
 237         } else {
 238             sb.append(filename);
 239         }
 240         return new Entity(pkg, Utils.toClassName(sb.toString()));
 241     }
 242 
 243     HeaderFile getHeaderFile(Path header, HeaderFile main) {
 244         if (!Files.isRegularFile(header)) {
 245             logger.warning(() -> "Not a regular file: " + header.toString());
 246             throw new IllegalArgumentException(header.toString());
 247         }
 248 
 249         final Context.Entity e = whatis(header);
 250         return new HeaderFile(this, header, e.pkg, e.entity, main);
 251     }
 252 
 253     void defineMacro(Cursor cursor) {
 254         String macroName = cursor.spelling();
 255         if (cursor.isMacroFunctionLike()) {
 256             logger.fine(() -> "Skipping function-like macro " + macroName);
 257             return;
 258         }
 259 
 260 
 261         if (macroParser.isDefined(macroName)) {
 262             logger.fine(() -> "Macro " + macroName + " already handled");
 263             return;
 264         }
 265 
 266         logger.fine(() -> "Defining macro " + macroName);
 267 
 268         TranslationUnit tu = cursor.getTranslationUnit();
 269         SourceRange range = cursor.getExtent();
 270         String[] tokens = tu.tokens(range);
 271 
 272         macroParser.parse(cursor, tokens);
 273     }
 274 
 275     List<MacroParser.Macro> macros(HeaderFile header) {
 276         return macroParser.macros().stream()
 277                 .filter(m -> m.getFileLocation().path().equals(header.path))
 278                 .collect(Collectors.toList());
 279     }
 280 
 281     void processCursor(Cursor c, HeaderFile main, Function<HeaderFile, AsmCodeFactory> fn) {
 282         SourceLocation loc = c.getSourceLocation();
 283         if (loc == null) {
 284             logger.info(() -> "Ignore Cursor " + c.spelling() + "@" + c.USR() + " has no SourceLocation");
 285             return;
 286         }
 287         logger.fine(() -> "Do cursor: " + c.spelling() + "@" + c.USR());
 288 
 289         HeaderFile header;
 290         boolean isBuiltIn = false;
 291 
 292         if (loc.isFromMainFile()) {
 293             header = main;
 294         } else {
 295             SourceLocation.Location src = loc.getFileLocation();
 296             if (src == null) {
 297                 logger.info(() -> "Cursor " + c.spelling() + "@" + c.USR() + " has no FileLocation");
 298                 return;
 299             }
 300 
 301             Path p = src.path();
 302             if (p == null) {
 303                 logger.fine(() -> "Found built-in type: " + c.spelling());
 304                 header = main;
 305                 isBuiltIn = true;
 306             } else {
 307                 p = p.normalize().toAbsolutePath();
 308                 header = headerMap.get(p);
 309                 if (header == null) {
 310                     final HeaderFile hf = header = getHeaderFile(p, main);
 311                     logger.config(() -> "First encounter of header file " + hf.path + ", assigned to package " + hf.pkgName);
 312                     // Only generate code for header files sepcified or in the same package
 313                     // System headers are excluded, they need to be explicitly specified in jextract cmdline
 314                     if (sources.contains(p) ||
 315                         (!loc.isInSystemHeader()) && (header.pkgName.equals(main.pkgName))) {
 316                         logger.config("Code gen for header " + p + " enabled in package " + header.pkgName);
 317                         header.useCodeFactory(fn.apply(header));
 318                     }
 319                     headerMap.put(p, header);
 320                 }
 321             }
 322         }
 323 
 324         header.processCursor(c, main, isBuiltIn);
 325     }
 326 
 327     public void parse() {
 328         parse(header -> new AsmCodeFactory(this, header));
 329     }
 330 
 331     public void parse(Function<HeaderFile, AsmCodeFactory> fn) {
 332         if (!libraryNames.isEmpty() && !linkCheckPaths.isEmpty()) {
 333             Library[] libs = LibrariesHelper.loadLibraries(MethodHandles.lookup(),
 334                 linkCheckPaths.toArray(new String[0]),
 335                 libraryNames.toArray(new String[0]));
 336 
 337             // check if the given symbol is found in any of the libraries or not.
 338             // If not found, warn the user for the missing symbol.
 339             symChecker = name -> {
 340                 if (Main.DEBUG) {
 341                     err.println("Searching symbol: " + name);
 342                 }
 343                 return (Arrays.stream(libs).filter(lib -> {
 344                         try {
 345                             lib.lookup(name);
 346                             if (Main.DEBUG) {
 347                                 err.println("Found symbol: " + name);
 348                             }
 349                             return true;
 350                         } catch (NoSuchMethodException nsme) {
 351                             return false;
 352                         }
 353                     }).findFirst().isPresent());
 354             };
 355         } else {
 356             symChecker = null;
 357         }
 358 
 359         if (!excludeSymbols.isEmpty()) {
 360             Pattern[] pats = excludeSymbols.toArray(new Pattern[0]);
 361             symFilter = name -> {
 362                 return Arrays.stream(pats).filter(pat -> pat.matcher(name).matches()).
 363                     findFirst().isPresent();
 364             };
 365         } else {
 366             symFilter = null;
 367         }
 368 
 369         sources.forEach(path -> {
 370             if (headerMap.containsKey(path)) {
 371                 logger.info(() -> path.toString() + " seen earlier via #include");
 372                 return;
 373             }
 374 
 375             HeaderFile hf = headerMap.computeIfAbsent(path, p -> getHeaderFile(p, null));
 376             hf.useLibraries(libraryNames, libraryPaths);
 377             hf.useCodeFactory(fn.apply(hf));
 378             logger.info(() -> {
 379                 StringBuilder sb = new StringBuilder(
 380                         "Parsing header file " + path + " with following args:\n");
 381                 int i = 0;
 382                 for (String arg : clangArgs) {
 383                     sb.append("arg[");
 384                     sb.append(i++);
 385                     sb.append("] = ");
 386                     sb.append(arg);
 387                     sb.append("\n");
 388                 }
 389                 return sb.toString();
 390             });
 391 
 392             Index index = LibClang.createIndex();
 393             Cursor tuCursor = index.parse(path.toString(),
 394                     d -> {
 395                         err.println(d);
 396                         if (d.severity() >  Diagnostic.CXDiagnostic_Warning) {
 397                             throw new RuntimeException(d.toString());
 398                         }
 399                     },
 400                     Main.INCLUDE_MACROS,
 401                     clangArgs.toArray(new String[0]));
 402 
 403             tuCursor.children()
 404                     .peek(c -> logger.finest(
 405                         () -> "Cursor: " + c.spelling() + "@" + c.USR() + "?" + c.isDeclaration()))
 406                     .filter(c -> c.isDeclaration() || c.isPreprocessing())
 407                     .forEach(c -> processCursor(c, hf, fn));
 408         });
 409     }
 410 
 411     private Map<String, List<AsmCodeFactory>> getPkgCfMap() {
 412         final Map<String, List<AsmCodeFactory>> mapPkgCf = new HashMap<>();
 413         // Build the pkg to CodeFactory map
 414         headerMap.values().forEach(header -> {
 415             AsmCodeFactory cf = header.getCodeFactory();
 416             String pkg = header.pkgName;
 417             logger.config(() -> "File " + header + " is in package: " + pkg);
 418             if (cf == null) {
 419                 logger.config(() -> "File " + header + " code generation is not activated!");
 420                 return;
 421             }
 422             List<AsmCodeFactory> l = mapPkgCf.computeIfAbsent(pkg, k -> new ArrayList<>());
 423             l.add(cf);
 424             logger.config(() -> "Add cf " + cf + " to pkg " + pkg + ", size is now " + l.size());
 425         });
 426         return Collections.unmodifiableMap(mapPkgCf);
 427     }
 428 
 429     public Map<String, byte[]> collectClasses(String... pkgs) {
 430         final Map<String, byte[]> rv = new HashMap<>();
 431         final Map<String, List<AsmCodeFactory>> mapPkgCf = getPkgCfMap();
 432         for (String pkg_name : pkgs) {
 433             mapPkgCf.getOrDefault(pkg_name, Collections.emptyList())
 434                     .forEach(cf -> rv.putAll(cf.collect()));
 435         }
 436         return Collections.unmodifiableMap(rv);
 437     }
 438 
 439     void collectClassFiles(Path destDir, String... pkgs) throws IOException {
 440         try {
 441             collectClasses(pkgs).entrySet().stream().forEach(e -> {
 442                 try {
 443                     String path = e.getKey().replace('.', File.separatorChar) + ".class";
 444                     logger.fine(() -> "Writing " + path);
 445                     Path fullPath = destDir.resolve(path).normalize();
 446                     Files.createDirectories(fullPath.getParent());
 447                     try (OutputStream fos = Files.newOutputStream(fullPath)) {
 448                         fos.write(e.getValue());
 449                         fos.flush();
 450                     }
 451                 } catch (IOException ioe) {
 452                     throw new UncheckedIOException(ioe);
 453                 }
 454             });
 455         } catch (UncheckedIOException uioe) {
 456             throw uioe.getCause();
 457         }
 458     }
 459 
 460     private void writeJar(AsmCodeFactory cf, JarOutputStream jar) {
 461         cf.collect().entrySet().stream().forEach(e -> {
 462             try {
 463                 String path = e.getKey().replace('.', File.separatorChar) + ".class";
 464                 logger.fine(() -> "Add " + path);
 465                 jar.putNextEntry(new ZipEntry(path));
 466                 jar.write(e.getValue());
 467                 jar.closeEntry();
 468             } catch (IOException ioe) {
 469                 throw new UncheckedIOException(ioe);
 470             }
 471         });
 472     }
 473 
 474     public void collectJarFile(final JarOutputStream jos, String... pkgs) {
 475         final Map<String, List<AsmCodeFactory>> mapPkgCf = getPkgCfMap();
 476 
 477         for (String pkg_name : pkgs) {
 478             // convert '.' to '/' to use as a path
 479             String entryName = Utils.toInternalName(pkg_name, "");
 480             // package folder
 481             if (!entryName.isEmpty()) {
 482                 try {
 483                     jos.putNextEntry(new ZipEntry(entryName));
 484                 } catch (IOException ex) {
 485                     throw new UncheckedIOException(ex);
 486                 }
 487             }
 488             logger.fine(() -> "Produce for package " + pkg_name);
 489             mapPkgCf.getOrDefault(pkg_name, Collections.emptyList())
 490                     .forEach(cf -> writeJar(cf, jos));
 491         }
 492     }
 493 
 494     void collectJarFile(final Path jar, String... pkgs) throws IOException {
 495         logger.info(() -> "Collecting jar file " + jar);
 496         try (OutputStream os = Files.newOutputStream(jar, CREATE, TRUNCATE_EXISTING, WRITE);
 497                 JarOutputStream jo = new JarOutputStream(os)) {
 498             collectJarFile(jo, pkgs);
 499         } catch (UncheckedIOException uioe) {
 500             throw uioe.getCause();
 501         }
 502     }
 503 
 504     /**
 505      * Perform a local lookup, any undefined type will cause a JType
 506      * be defined within origin scope.
 507      *
 508      * @param type   The libclang type
 509      * @param origin The path of the file where type is encountered
 510      * @return The JType
 511      */
 512     JType getJType(final Type type, Path origin) {
 513         Path p = origin.normalize().toAbsolutePath();
 514 
 515         HeaderFile hf = headerMap.get(p);
 516         // We should not encounter a type if the header file reference to it is not yet processed
 517         assert(null != hf);
 518         if (hf == null) {
 519             throw new IllegalArgumentException("Failed to lookup header for " + p + " (origin: " + origin + ")");
 520         }
 521 
 522         return hf.localLookup(type);
 523     }
 524 
 525     /**
 526      * Perform a global lookup
 527      *
 528      * @param c The cursor define or declare the type.
 529      * @return
 530      */
 531     JType getJType(final Cursor c) {
 532         if (c.isInvalid()) {
 533             throw new IllegalArgumentException();
 534         }
 535         SourceLocation loc = c.getSourceLocation();
 536         if (null == loc) {
 537             return null;
 538         }
 539         Path p = loc.getFileLocation().path();
 540         if (null == p) {
 541             return null;
 542         }
 543         return getJType(c.type(), p);
 544     }
 545 }