1 /*
   2  * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  */
  23 package com.sun.tools.jextract;
  24 
  25 import jdk.internal.clang.*;
  26 import jdk.internal.foreign.LibrariesHelper;
  27 
  28 import java.io.ByteArrayOutputStream;
  29 import java.io.File;
  30 import java.io.IOException;
  31 import java.io.OutputStream;
  32 import java.io.PrintWriter;
  33 import java.io.UncheckedIOException;
  34 import java.lang.invoke.MethodHandles;
  35 import java.foreign.Library;
  36 import java.nio.file.Files;
  37 import java.nio.file.Path;
  38 import java.util.ArrayList;
  39 import java.util.Arrays;
  40 import java.util.Collections;
  41 import java.util.HashMap;
  42 import java.util.List;
  43 import java.util.Map;
  44 import java.util.Properties;
  45 import java.util.Set;
  46 import java.util.TreeSet;
  47 import java.util.function.Function;
  48 import java.util.function.Predicate;
  49 import java.util.jar.JarOutputStream;
  50 import java.util.logging.Logger;
  51 import java.util.regex.Pattern;
  52 import java.util.stream.Collectors;
  53 import java.util.zip.ZipEntry;
  54 
  55 import static java.nio.file.StandardOpenOption.CREATE;
  56 import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
  57 import static java.nio.file.StandardOpenOption.WRITE;
  58 
  59 /**
  60  * The setup for the tool execution
  61  */
  62 public final class Context {
  63     // package name to TypeDictionary
  64     private final Map<String, TypeDictionary> tdMap;
  65     // The folder path mapping to package name
  66     private final Map<Path, String> pkgMap;
  67     // The header file parsed
  68     private final Map<Path, HeaderFile> headerMap;
  69     // The args for parsing C
  70     private final List<String> clangArgs;
  71     // The set of source header files
  72     private final Set<Path>  sources;
  73     // The list of library names
  74     private final List<String> libraryNames;
  75     // The list of library paths
  76     private final List<String> libraryPaths;
  77     // The list of library paths for link checks
  78     private final List<String> linkCheckPaths;
  79     // Symbol patterns to be excluded
  80     private final List<Pattern> excludeSymbols;
  81 
  82     final PrintWriter out;
  83     final PrintWriter err;
  84 
  85     private Predicate<String> symChecker;
  86     private Predicate<String> symFilter;
  87 
  88     private final MacroParser macroParser;
  89 
  90     private final static String defaultPkg = "jextract.dump";
  91     final Logger logger = Logger.getLogger(getClass().getPackage().getName());
  92 
  93     public Context(PrintWriter out, PrintWriter err) {
  94         this.tdMap = new HashMap<>();
  95         this.pkgMap = new HashMap<>();
  96         this.headerMap = new HashMap<>();
  97         this.clangArgs = new ArrayList<>();
  98         this.sources = new TreeSet<>();
  99         this.libraryNames = new ArrayList<>();
 100         this.libraryPaths = new ArrayList<>();
 101         this.linkCheckPaths = new ArrayList<>();
 102         this.excludeSymbols = new ArrayList<>();
 103         this.macroParser = new MacroParser();
 104         this.out = out;
 105         this.err = err;
 106     }
 107 
 108     public Context() {
 109         this(new PrintWriter(System.out, true), new PrintWriter(System.err, true));
 110     }
 111 
 112     TypeDictionary typeDictionaryFor(String pkg) {
 113         return tdMap.computeIfAbsent(pkg, p->new TypeDictionary(this, p));
 114     }
 115 
 116     void addClangArg(String arg) {
 117         clangArgs.add(arg);
 118     }
 119 
 120     public void addSource(Path path) {
 121         sources.add(path);
 122     }
 123 
 124     void addLibraryName(String name) {
 125         libraryNames.add(name);
 126     }
 127 
 128     void addLibraryPath(String path) {
 129         libraryPaths.add(path);
 130     }
 131 
 132     void addLinkCheckPath(String path) {
 133         linkCheckPaths.add(path);
 134     }
 135 
 136     void addExcludeSymbols(String pattern) {
 137         excludeSymbols.add(Pattern.compile(pattern));
 138     }
 139 
 140     boolean isSymbolFound(String name) {
 141         return symChecker == null? true : symChecker.test(name);
 142     }
 143 
 144     boolean isSymbolExcluded(String name) {
 145         return symFilter == null? false : symFilter.test(name);
 146     }
 147 
 148     /**
 149      * Setup a package name for a given folder.
 150      *
 151      * @param folder The path to the folder, use null to set catch-all.
 152      * @param pkg    The package name
 153      * @return True if the folder is setup successfully. False is a package
 154      * has been assigned for the folder.
 155      */
 156     public boolean usePackageForFolder(Path folder, String pkg) {
 157         if (folder != null) {
 158             folder = folder.toAbsolutePath();
 159             if (!Files.isDirectory(folder)) {
 160                 folder = folder.getParent();
 161             }
 162         }
 163         String existing = pkgMap.putIfAbsent(folder, pkg);
 164         final String finalFolder = (null == folder) ? "all folders not configured" : folder.toString();
 165         if (null == existing) {
 166             logger.config(() -> "Package " + pkg + " is selected for " + finalFolder);
 167             return true;
 168         } else {
 169             logger.warning(() -> "Package " + existing + " had been selected for " + finalFolder + ", request to use " + pkg + " is ignored.");
 170             return false;
 171         }
 172     }
 173 
 174     static class Entity {
 175         final String pkg;
 176         final String entity;
 177 
 178         Entity(String pkg, String entity) {
 179             this.pkg = pkg;
 180             this.entity = entity;
 181         }
 182     }
 183 
 184     /**
 185      * Determine package and interface name given a path. If the path is
 186      * a folder, then only package name is determined. The package name is
 187      * determined with the longest path matching the setup. If the path is not
 188      * setup for any package, the default package name is returned.
 189      *
 190      * @param origin The source path
 191      * @return The Entity
 192      * @see Context::usePackageForFolder(Path, String)
 193      */
 194     Entity whatis(Path origin) {
 195         // normalize to absolute path
 196         origin = origin.toAbsolutePath();
 197         String filename = null;
 198         if (!Files.isDirectory(origin)) {
 199             // ensure it's a folder name
 200             filename = origin.getFileName().toString();
 201             origin = origin.getParent();
 202         }
 203         Path path = origin;
 204 
 205         // search the map for a hit with longest path
 206         while (path != null && !pkgMap.containsKey(path)) {
 207             path = path.getParent();
 208         }
 209 
 210         int start;
 211         String pkg;
 212         if (path != null) {
 213             start = path.getNameCount();
 214             pkg = pkgMap.get(path);
 215         } else {
 216             pkg = pkgMap.get(null);
 217             if (pkg == null) {
 218                 start = 0;
 219                 pkg = defaultPkg;
 220             } else {
 221                 start = origin.getNameCount();
 222             }
 223         }
 224 
 225         if (filename == null) {
 226             // a folder, only pkg name matters
 227             return new Entity(pkg, null);
 228         }
 229 
 230         StringBuilder sb = new StringBuilder();
 231         while (start < origin.getNameCount()) {
 232             sb.append(Utils.toJavaIdentifier(origin.getName(start++).toString()));
 233             sb.append("_");
 234         }
 235 
 236         int ext = filename.lastIndexOf('.');
 237         if (ext != -1) {
 238             sb.append(filename.substring(0, ext));
 239         } else {
 240             sb.append(filename);
 241         }
 242         return new Entity(pkg, Utils.toClassName(sb.toString()));
 243     }
 244 
 245     HeaderFile getHeaderFile(Path header, HeaderFile main) {
 246         if (!Files.isRegularFile(header)) {
 247             logger.warning(() -> "Not a regular file: " + header.toString());
 248             throw new IllegalArgumentException(header.toString());
 249         }
 250 
 251         final Context.Entity e = whatis(header);
 252         return new HeaderFile(this, header, e.pkg, e.entity, main);
 253     }
 254 
 255     void defineMacro(Cursor cursor) {
 256         String macroName = cursor.spelling();
 257         if (cursor.isMacroFunctionLike()) {
 258             logger.fine(() -> "Skipping function-like macro " + macroName);
 259             return;
 260         }
 261 
 262 
 263         if (macroParser.isDefined(macroName)) {
 264             logger.fine(() -> "Macro " + macroName + " already handled");
 265             return;
 266         }
 267 
 268         logger.fine(() -> "Defining macro " + macroName);
 269 
 270         TranslationUnit tu = cursor.getTranslationUnit();
 271         SourceRange range = cursor.getExtent();
 272         String[] tokens = tu.tokens(range);
 273 
 274         macroParser.parse(cursor, tokens);
 275     }
 276 
 277     List<MacroParser.Macro> macros(HeaderFile header) {
 278         return macroParser.macros().stream()
 279                 .filter(m -> m.getFileLocation().path().equals(header.path))
 280                 .collect(Collectors.toList());
 281     }
 282 
 283     void processCursor(Cursor c, HeaderFile main, Function<HeaderFile, AsmCodeFactory> fn) {
 284         SourceLocation loc = c.getSourceLocation();
 285         if (loc == null) {
 286             logger.info(() -> "Ignore Cursor " + c.spelling() + "@" + c.USR() + " has no SourceLocation");
 287             return;
 288         }
 289         logger.fine(() -> "Do cursor: " + c.spelling() + "@" + c.USR());
 290 
 291         HeaderFile header;
 292         boolean isBuiltIn = false;
 293 
 294         if (loc.isFromMainFile()) {
 295             header = main;
 296         } else {
 297             SourceLocation.Location src = loc.getFileLocation();
 298             if (src == null) {
 299                 logger.info(() -> "Cursor " + c.spelling() + "@" + c.USR() + " has no FileLocation");
 300                 return;
 301             }
 302 
 303             Path p = src.path();
 304             if (p == null) {
 305                 logger.fine(() -> "Found built-in type: " + c.spelling());
 306                 header = main;
 307                 isBuiltIn = true;
 308             } else {
 309                 p = p.normalize().toAbsolutePath();
 310                 header = headerMap.get(p);
 311                 if (header == null) {
 312                     final HeaderFile hf = header = getHeaderFile(p, main);
 313                     logger.config(() -> "First encounter of header file " + hf.path + ", assigned to package " + hf.pkgName);
 314                     // Only generate code for header files sepcified or in the same package
 315                     // System headers are excluded, they need to be explicitly specified in jextract cmdline
 316                     if (sources.contains(p) ||
 317                         (!loc.isInSystemHeader()) && (header.pkgName.equals(main.pkgName))) {
 318                         logger.config("Code gen for header " + p + " enabled in package " + header.pkgName);
 319                         header.useCodeFactory(fn.apply(header));
 320                     }
 321                     headerMap.put(p, header);
 322                 }
 323             }
 324         }
 325 
 326         header.processCursor(c, main, isBuiltIn);
 327     }
 328 
 329     public void parse() {
 330         parse(header -> new AsmCodeFactory(this, header));
 331     }
 332 
 333     public void parse(Function<HeaderFile, AsmCodeFactory> fn) {
 334         if (!libraryNames.isEmpty() && !linkCheckPaths.isEmpty()) {
 335             Library[] libs = LibrariesHelper.loadLibraries(MethodHandles.lookup(),
 336                 linkCheckPaths.toArray(new String[0]),
 337                 libraryNames.toArray(new String[0]));
 338 
 339             // check if the given symbol is found in any of the libraries or not.
 340             // If not found, warn the user for the missing symbol.
 341             symChecker = name -> {
 342                 if (Main.DEBUG) {
 343                     err.println("Searching symbol: " + name);
 344                 }
 345                 return (Arrays.stream(libs).filter(lib -> {
 346                         try {
 347                             lib.lookup(name);
 348                             if (Main.DEBUG) {
 349                                 err.println("Found symbol: " + name);
 350                             }
 351                             return true;
 352                         } catch (NoSuchMethodException nsme) {
 353                             return false;
 354                         }
 355                     }).findFirst().isPresent());
 356             };
 357         } else {
 358             symChecker = null;
 359         }
 360 
 361         if (!excludeSymbols.isEmpty()) {
 362             Pattern[] pats = excludeSymbols.toArray(new Pattern[0]);
 363             symFilter = name -> {
 364                 return Arrays.stream(pats).filter(pat -> pat.matcher(name).matches()).
 365                     findFirst().isPresent();
 366             };
 367         } else {
 368             symFilter = null;
 369         }
 370 
 371         sources.forEach(path -> {
 372             if (headerMap.containsKey(path)) {
 373                 logger.info(() -> path.toString() + " seen earlier via #include");
 374                 return;
 375             }
 376 
 377             HeaderFile hf = headerMap.computeIfAbsent(path, p -> getHeaderFile(p, null));
 378             hf.useLibraries(libraryNames, libraryPaths);
 379             hf.useCodeFactory(fn.apply(hf));
 380             logger.info(() -> {
 381                 StringBuilder sb = new StringBuilder(
 382                         "Parsing header file " + path + " with following args:\n");
 383                 int i = 0;
 384                 for (String arg : clangArgs) {
 385                     sb.append("arg[");
 386                     sb.append(i++);
 387                     sb.append("] = ");
 388                     sb.append(arg);
 389                     sb.append("\n");
 390                 }
 391                 return sb.toString();
 392             });
 393 
 394             Index index = LibClang.createIndex();
 395             Cursor tuCursor = index.parse(path.toString(),
 396                     d -> {
 397                         err.println(d);
 398                         if (d.severity() >  Diagnostic.CXDiagnostic_Warning) {
 399                             throw new RuntimeException(d.toString());
 400                         }
 401                     },
 402                     Main.INCLUDE_MACROS,
 403                     clangArgs.toArray(new String[0]));
 404 
 405             tuCursor.children()
 406                     .peek(c -> logger.finest(
 407                         () -> "Cursor: " + c.spelling() + "@" + c.USR() + "?" + c.isDeclaration()))
 408                     .filter(c -> c.isDeclaration() || c.isPreprocessing())
 409                     .forEach(c -> processCursor(c, hf, fn));
 410         });
 411     }
 412 
 413     private Map<String, List<AsmCodeFactory>> getPkgCfMap() {
 414         final Map<String, List<AsmCodeFactory>> mapPkgCf = new HashMap<>();
 415         // Build the pkg to CodeFactory map
 416         headerMap.values().forEach(header -> {
 417             AsmCodeFactory cf = header.getCodeFactory();
 418             String pkg = header.pkgName;
 419             logger.config(() -> "File " + header + " is in package: " + pkg);
 420             if (cf == null) {
 421                 logger.config(() -> "File " + header + " code generation is not activated!");
 422                 return;
 423             }
 424             List<AsmCodeFactory> l = mapPkgCf.computeIfAbsent(pkg, k -> new ArrayList<>());
 425             l.add(cf);
 426             logger.config(() -> "Add cf " + cf + " to pkg " + pkg + ", size is now " + l.size());
 427         });
 428         return Collections.unmodifiableMap(mapPkgCf);
 429     }
 430 
 431     public Map<String, byte[]> collectClasses(String... pkgs) {
 432         final Map<String, byte[]> rv = new HashMap<>();
 433         final Map<String, List<AsmCodeFactory>> mapPkgCf = getPkgCfMap();
 434         for (String pkg_name : pkgs) {
 435             mapPkgCf.getOrDefault(pkg_name, Collections.emptyList())
 436                     .forEach(cf -> rv.putAll(cf.collect()));
 437         }
 438         return Collections.unmodifiableMap(rv);
 439     }
 440 
 441     private static final String JEXTRACT_MANIFEST = "META-INFO" + File.separatorChar + "jextract.properties";
 442 
 443     @SuppressWarnings("deprecation")
 444     private byte[] getJextractProperties(String[] args) {
 445         Properties props = new Properties();
 446         props.setProperty("os.name", System.getProperty("os.name"));
 447         props.setProperty("os.version", System.getProperty("os.version"));
 448         props.setProperty("os.arch", System.getProperty("os.arch"));
 449         props.setProperty("jextract.args", Arrays.toString(args));
 450         ByteArrayOutputStream baos = new ByteArrayOutputStream();
 451         props.save(baos, "jextract meta data");
 452         return baos.toByteArray();
 453     }
 454 
 455     void collectClassFiles(Path destDir, String[] args, String... pkgs) throws IOException {
 456         try {
 457             collectClasses(pkgs).entrySet().stream().forEach(e -> {
 458                 try {
 459                     String path = e.getKey().replace('.', File.separatorChar) + ".class";
 460                     logger.fine(() -> "Writing " + path);
 461                     Path fullPath = destDir.resolve(path).normalize();
 462                     Files.createDirectories(fullPath.getParent());
 463                     try (OutputStream fos = Files.newOutputStream(fullPath)) {
 464                         fos.write(e.getValue());
 465                         fos.flush();
 466                     }
 467                 } catch (IOException ioe) {
 468                     throw new UncheckedIOException(ioe);
 469                 }
 470             });
 471 
 472             Path propsPath = destDir.resolve(JEXTRACT_MANIFEST).normalize();
 473             Files.createDirectories(propsPath.getParent());
 474             try (OutputStream fos = Files.newOutputStream(propsPath)) {
 475                 fos.write(getJextractProperties(args));
 476                 fos.flush();
 477             }
 478         } catch (UncheckedIOException uioe) {
 479             throw uioe.getCause();
 480         }
 481     }
 482 
 483     private void writeJar(AsmCodeFactory cf, JarOutputStream jar, String[] args) {
 484         cf.collect().entrySet().stream().forEach(e -> {
 485             try {
 486                 String path = e.getKey().replace('.', File.separatorChar) + ".class";
 487                 logger.fine(() -> "Add " + path);
 488                 jar.putNextEntry(new ZipEntry(path));
 489                 jar.write(e.getValue());
 490                 jar.closeEntry();
 491             } catch (IOException ioe) {
 492                 throw new UncheckedIOException(ioe);
 493             }
 494         });
 495     }
 496 
 497     public void collectJarFile(final JarOutputStream jos, String[] args, String... pkgs) {
 498         final Map<String, List<AsmCodeFactory>> mapPkgCf = getPkgCfMap();
 499 
 500         for (String pkg_name : pkgs) {
 501             // convert '.' to '/' to use as a path
 502             String entryName = Utils.toInternalName(pkg_name, "");
 503             // package folder
 504             if (!entryName.isEmpty()) {
 505                 try {
 506                     jos.putNextEntry(new ZipEntry(entryName));
 507                 } catch (IOException ex) {
 508                     throw new UncheckedIOException(ex);
 509                 }
 510             }
 511             logger.fine(() -> "Produce for package " + pkg_name);
 512             mapPkgCf.getOrDefault(pkg_name, Collections.emptyList())
 513                     .forEach(cf -> writeJar(cf, jos, args));
 514         }
 515 
 516         try {
 517             jos.putNextEntry(new ZipEntry(JEXTRACT_MANIFEST));
 518             jos.write(getJextractProperties(args));
 519             jos.closeEntry();
 520         } catch (IOException ioe) {
 521             throw new UncheckedIOException(ioe);
 522         }
 523     }
 524 
 525     void collectJarFile(final Path jar, String[] args, String... pkgs) throws IOException {
 526         logger.info(() -> "Collecting jar file " + jar);
 527         try (OutputStream os = Files.newOutputStream(jar, CREATE, TRUNCATE_EXISTING, WRITE);
 528                 JarOutputStream jo = new JarOutputStream(os)) {
 529             collectJarFile(jo, args, pkgs);
 530         } catch (UncheckedIOException uioe) {
 531             throw uioe.getCause();
 532         }
 533     }
 534 
 535     /**
 536      * Perform a local lookup, any undefined type will cause a JType
 537      * be defined within origin scope.
 538      *
 539      * @param type   The libclang type
 540      * @param origin The path of the file where type is encountered
 541      * @return The JType
 542      */
 543     JType getJType(final Type type, Path origin) {
 544         Path p = origin.normalize().toAbsolutePath();
 545 
 546         HeaderFile hf = headerMap.get(p);
 547         // We should not encounter a type if the header file reference to it is not yet processed
 548         assert(null != hf);
 549         if (hf == null) {
 550             throw new IllegalArgumentException("Failed to lookup header for " + p + " (origin: " + origin + ")");
 551         }
 552 
 553         return hf.localLookup(type);
 554     }
 555 
 556     /**
 557      * Perform a global lookup
 558      *
 559      * @param c The cursor define or declare the type.
 560      * @return
 561      */
 562     JType getJType(final Cursor c) {
 563         if (c.isInvalid()) {
 564             throw new IllegalArgumentException();
 565         }
 566         SourceLocation loc = c.getSourceLocation();
 567         if (null == loc) {
 568             return null;
 569         }
 570         Path p = loc.getFileLocation().path();
 571         if (null == p) {
 572             return null;
 573         }
 574         return getJType(c.type(), p);
 575     }
 576 }
--- EOF ---