1 /* 2 * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 package com.sun.tools.jextract; 24 25 import jdk.internal.clang.*; 26 27 import java.io.ByteArrayOutputStream; 28 import java.io.File; 29 import java.io.IOException; 30 import java.io.OutputStream; 31 import java.io.PrintWriter; 32 import java.io.UncheckedIOException; 33 import java.lang.invoke.MethodHandles; 34 import java.lang.invoke.MethodHandles.Lookup; 35 import java.foreign.Library; 36 import java.foreign.Libraries; 37 import java.nio.file.Files; 38 import java.nio.file.Path; 39 import java.nio.file.Paths; 40 import java.util.ArrayList; 41 import java.util.Arrays; 42 import java.util.Collections; 43 import java.util.HashMap; 44 import java.util.List; 45 import java.util.Map; 46 import java.util.Optional; 47 import java.util.Properties; 48 import java.util.Set; 49 import java.util.TreeSet; 50 import java.util.function.Function; 51 import java.util.function.Predicate; 52 import java.util.jar.JarOutputStream; 53 import java.util.logging.Logger; 54 import java.util.regex.Pattern; 55 import java.util.stream.Collectors; 56 import java.util.zip.ZipEntry; 57 import com.sun.tools.jextract.parser.Parser; 58 import com.sun.tools.jextract.tree.FunctionTree; 59 import com.sun.tools.jextract.tree.HeaderTree; 60 import com.sun.tools.jextract.tree.Tree; 61 62 import static java.nio.file.StandardOpenOption.CREATE; 63 import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; 64 import static java.nio.file.StandardOpenOption.WRITE; 65 66 /** 67 * The setup for the tool execution 68 */ 69 public final class Context { 70 // package name to TypeDictionary 71 private final Map<String, TypeDictionary> tdMap; 72 // The folder path mapping to package name 73 private final Map<Path, String> pkgMap; 74 // The header file parsed 75 private final Map<Path, HeaderFile> headerMap; 76 // The args for parsing C 77 private final List<String> clangArgs; 78 // The set of source header files 79 private final Set<Path> sources; 80 // The list of library names 81 private final List<String> libraryNames; 82 // The list of library paths 83 private final List<String> libraryPaths; 84 // The list of library paths for link checks 85 private final List<String> linkCheckPaths; 86 // Symbol patterns to be excluded 87 private final List<Pattern> excludeSymbols; 88 89 final PrintWriter out; 90 final PrintWriter err; 91 92 private Predicate<String> symChecker; 93 private Predicate<String> symFilter; 94 95 private final Parser parser; 96 97 private final static String defaultPkg = "jextract.dump"; 98 final Logger logger = Logger.getLogger(getClass().getPackage().getName()); 99 100 public Context(PrintWriter out, PrintWriter err) { 101 this.tdMap = new HashMap<>(); 102 this.pkgMap = new HashMap<>(); 103 this.headerMap = new HashMap<>(); 104 this.clangArgs = new ArrayList<>(); 105 this.sources = new TreeSet<>(); 106 this.libraryNames = new ArrayList<>(); 107 this.libraryPaths = new ArrayList<>(); 108 this.linkCheckPaths = new ArrayList<>(); 109 this.excludeSymbols = new ArrayList<>(); 110 this.parser = new Parser(out, err, Main.INCLUDE_MACROS); 111 this.out = out; 112 this.err = err; 113 } 114 115 public Context() { 116 this(new PrintWriter(System.out, true), new PrintWriter(System.err, true)); 117 } 118 119 TypeDictionary typeDictionaryFor(String pkg) { 120 return tdMap.computeIfAbsent(pkg, p->new TypeDictionary(this, p)); 121 } 122 123 void addClangArg(String arg) { 124 clangArgs.add(arg); 125 } 126 127 public void addSource(Path path) { 128 sources.add(path); 129 } 130 131 void addLibraryName(String name) { 132 libraryNames.add(name); 133 } 134 135 void addLibraryPath(String path) { 136 libraryPaths.add(path); 137 } 138 139 void addLinkCheckPath(String path) { 140 linkCheckPaths.add(path); 141 } 142 143 void addExcludeSymbols(String pattern) { 144 excludeSymbols.add(Pattern.compile(pattern)); 145 } 146 147 // return the absolute path of the library of given name by searching 148 // in the given array of paths. 149 private static Optional<Path> findLibraryPath(Path[] paths, String libName) { 150 return Arrays.stream(paths). 151 map(p -> p.resolve(System.mapLibraryName(libName))). 152 filter(Files::isRegularFile).map(Path::toAbsolutePath).findFirst(); 153 } 154 155 /* 156 * Load the specified shared libraries from the specified paths. 157 * 158 * @param lookup Lookup object of the caller. 159 * @param pathStrs array of paths to load the shared libraries from. 160 * @param names array of shared library names. 161 */ 162 // used by jextract tool to load libraries for symbol checks. 163 public static Library[] loadLibraries(Lookup lookup, String[] pathStrs, String[] names) { 164 if (pathStrs == null || pathStrs.length == 0) { 165 return Arrays.stream(names).map( 166 name -> Libraries.loadLibrary(lookup, name)).toArray(Library[]::new); 167 } else { 168 Path[] paths = Arrays.stream(pathStrs).map(Paths::get).toArray(Path[]::new); 169 return Arrays.stream(names).map(libName -> { 170 Optional<Path> absPath = findLibraryPath(paths, libName); 171 return absPath.isPresent() ? 172 Libraries.load(lookup, absPath.get().toString()) : 173 Libraries.loadLibrary(lookup, libName); 174 }).toArray(Library[]::new); 175 } 176 } 177 178 private void initSymChecker() { 179 if (!libraryNames.isEmpty() && !linkCheckPaths.isEmpty()) { 180 Library[] libs = loadLibraries(MethodHandles.lookup(), 181 linkCheckPaths.toArray(new String[0]), 182 libraryNames.toArray(new String[0])); 183 184 // check if the given symbol is found in any of the libraries or not. 185 // If not found, warn the user for the missing symbol. 186 symChecker = name -> { 187 if (Main.DEBUG) { 188 err.println("Searching symbol: " + name); 189 } 190 return (Arrays.stream(libs).filter(lib -> { 191 try { 192 lib.lookup(name); 193 if (Main.DEBUG) { 194 err.println("Found symbol: " + name); 195 } 196 return true; 197 } catch (NoSuchMethodException nsme) { 198 return false; 199 } 200 }).findFirst().isPresent()); 201 }; 202 } else { 203 symChecker = null; 204 } 205 } 206 207 private boolean isSymbolFound(String name) { 208 return symChecker == null? true : symChecker.test(name); 209 } 210 211 private void initSymFilter() { 212 if (!excludeSymbols.isEmpty()) { 213 Pattern[] pats = excludeSymbols.toArray(new Pattern[0]); 214 symFilter = name -> { 215 return Arrays.stream(pats).filter(pat -> pat.matcher(name).matches()). 216 findFirst().isPresent(); 217 }; 218 } else { 219 symFilter = null; 220 } 221 } 222 223 private boolean isSymbolExcluded(String name) { 224 return symFilter == null? false : symFilter.test(name); 225 } 226 227 /** 228 * Setup a package name for a given folder. 229 * 230 * @param folder The path to the folder, use null to set catch-all. 231 * @param pkg The package name 232 * @return True if the folder is setup successfully. False is a package 233 * has been assigned for the folder. 234 */ 235 public boolean usePackageForFolder(Path folder, String pkg) { 236 if (folder != null) { 237 folder = folder.toAbsolutePath(); 238 if (!Files.isDirectory(folder)) { 239 folder = folder.getParent(); 240 } 241 } 242 String existing = pkgMap.putIfAbsent(folder, pkg); 243 final String finalFolder = (null == folder) ? "all folders not configured" : folder.toString(); 244 if (null == existing) { 245 logger.config(() -> "Package " + pkg + " is selected for " + finalFolder); 246 return true; 247 } else { 248 logger.warning(() -> "Package " + existing + " had been selected for " + finalFolder + ", request to use " + pkg + " is ignored."); 249 return false; 250 } 251 } 252 253 static class Entity { 254 final String pkg; 255 final String entity; 256 257 Entity(String pkg, String entity) { 258 this.pkg = pkg; 259 this.entity = entity; 260 } 261 } 262 263 /** 264 * Determine package and interface name given a path. If the path is 265 * a folder, then only package name is determined. The package name is 266 * determined with the longest path matching the setup. If the path is not 267 * setup for any package, the default package name is returned. 268 * 269 * @param origin The source path 270 * @return The Entity 271 * @see Context::usePackageForFolder(Path, String) 272 */ 273 Entity whatis(Path origin) { 274 // normalize to absolute path 275 origin = origin.toAbsolutePath(); 276 String filename = null; 277 if (!Files.isDirectory(origin)) { 278 // ensure it's a folder name 279 filename = origin.getFileName().toString(); 280 origin = origin.getParent(); 281 } 282 Path path = origin; 283 284 // search the map for a hit with longest path 285 while (path != null && !pkgMap.containsKey(path)) { 286 path = path.getParent(); 287 } 288 289 int start; 290 String pkg; 291 if (path != null) { 292 start = path.getNameCount(); 293 pkg = pkgMap.get(path); 294 } else { 295 pkg = pkgMap.get(null); 296 if (pkg == null) { 297 start = 0; 298 pkg = defaultPkg; 299 } else { 300 start = origin.getNameCount(); 301 } 302 } 303 304 if (filename == null) { 305 // a folder, only pkg name matters 306 return new Entity(pkg, null); 307 } 308 309 StringBuilder sb = new StringBuilder(); 310 while (start < origin.getNameCount()) { 311 sb.append(Utils.toJavaIdentifier(origin.getName(start++).toString())); 312 sb.append("_"); 313 } 314 315 int ext = filename.lastIndexOf('.'); 316 if (ext != -1) { 317 sb.append(filename.substring(0, ext)); 318 } else { 319 sb.append(filename); 320 } 321 return new Entity(pkg, Utils.toClassName(sb.toString())); 322 } 323 324 HeaderFile getHeaderFile(Path header, HeaderFile main) { 325 if (!Files.isRegularFile(header)) { 326 logger.warning(() -> "Not a regular file: " + header.toString()); 327 throw new IllegalArgumentException(header.toString()); 328 } 329 330 final Context.Entity e = whatis(header); 331 return new HeaderFile(this, header, e.pkg, e.entity, main); 332 } 333 334 void processTree(Tree tree, HeaderFile main, Function<HeaderFile, AsmCodeFactory> fn) { 335 SourceLocation loc = tree.location(); 336 337 HeaderFile header; 338 boolean isBuiltIn = false; 339 340 if (tree.isFromMain()) { 341 header = main; 342 } else { 343 SourceLocation.Location src = loc.getFileLocation(); 344 if (src == null) { 345 logger.info(() -> "Tree " + tree.name() + "@" + tree.USR() + " has no FileLocation"); 346 return; 347 } 348 349 Path p = src.path(); 350 if (p == null) { 351 logger.fine(() -> "Found built-in type: " + tree.name()); 352 header = main; 353 isBuiltIn = true; 354 } else { 355 p = p.normalize().toAbsolutePath(); 356 header = headerMap.get(p); 357 if (header == null) { 358 final HeaderFile hf = header = getHeaderFile(p, main); 359 logger.config(() -> "First encounter of header file " + hf.path + ", assigned to package " + hf.pkgName); 360 // Only generate code for header files sepcified or in the same package 361 // System headers are excluded, they need to be explicitly specified in jextract cmdline 362 if (sources.contains(p) || 363 (!loc.isInSystemHeader()) && (header.pkgName.equals(main.pkgName))) { 364 logger.config("Code gen for header " + p + " enabled in package " + header.pkgName); 365 header.useCodeFactory(fn.apply(header)); 366 } 367 headerMap.put(p, header); 368 } 369 } 370 } 371 372 header.processTree(tree, main, isBuiltIn); 373 } 374 375 public void parse() { 376 parse(header -> new AsmCodeFactory(this, header)); 377 } 378 379 private boolean symbolFilter(Tree tree) { 380 String name = tree.name(); 381 if (isSymbolExcluded(name)) { 382 return false; 383 } 384 385 // check for function symbols in libraries & warn missing symbols 386 if (tree instanceof FunctionTree && !isSymbolFound(name)) { 387 err.println(Main.format("warn.symbol.not.found", name)); 388 } 389 390 return true; 391 } 392 393 public void parse(Function<HeaderFile, AsmCodeFactory> fn) { 394 initSymChecker(); 395 initSymFilter(); 396 397 List<HeaderTree> headers = parser.parse(sources, clangArgs); 398 processHeaders(headers, fn); 399 } 400 401 private void processHeaders(List<HeaderTree> headers, Function<HeaderFile, AsmCodeFactory> fn) { 402 headers.stream(). 403 map(new TreeFilter(this::symbolFilter)). 404 map(new TypedefHandler()). 405 map(new EmptyNameHandler()). 406 forEach(header -> { 407 HeaderFile hf = headerMap.computeIfAbsent(header.path(), p -> getHeaderFile(p, null)); 408 hf.useLibraries(libraryNames, libraryPaths); 409 hf.useCodeFactory(fn.apply(hf)); 410 logger.info(() -> "Processing header file " + header.path()); 411 412 header.declarations().stream() 413 .peek(decl -> logger.finest( 414 () -> "Cursor: " + decl.name() + "@" + decl.USR() + "?" + decl.isDeclaration())) 415 .forEach(decl -> processTree(decl, hf, fn)); 416 }); 417 } 418 419 private Map<String, List<AsmCodeFactory>> getPkgCfMap() { 420 final Map<String, List<AsmCodeFactory>> mapPkgCf = new HashMap<>(); 421 // Build the pkg to CodeFactory map 422 headerMap.values().forEach(header -> { 423 AsmCodeFactory cf = header.getCodeFactory(); 424 String pkg = header.pkgName; 425 logger.config(() -> "File " + header + " is in package: " + pkg); 426 if (cf == null) { 427 logger.config(() -> "File " + header + " code generation is not activated!"); 428 return; 429 } 430 List<AsmCodeFactory> l = mapPkgCf.computeIfAbsent(pkg, k -> new ArrayList<>()); 431 l.add(cf); 432 logger.config(() -> "Add cf " + cf + " to pkg " + pkg + ", size is now " + l.size()); 433 }); 434 return Collections.unmodifiableMap(mapPkgCf); 435 } 436 437 public Map<String, byte[]> collectClasses(String... pkgs) { 438 final Map<String, byte[]> rv = new HashMap<>(); 439 final Map<String, List<AsmCodeFactory>> mapPkgCf = getPkgCfMap(); 440 for (String pkg_name : pkgs) { 441 mapPkgCf.getOrDefault(pkg_name, Collections.emptyList()) 442 .forEach(cf -> rv.putAll(cf.collect())); 443 } 444 return Collections.unmodifiableMap(rv); 445 } 446 447 private static final String JEXTRACT_MANIFEST = "META-INFO" + File.separatorChar + "jextract.properties"; 448 449 @SuppressWarnings("deprecation") 450 private byte[] getJextractProperties(String[] args) { 451 Properties props = new Properties(); 452 props.setProperty("os.name", System.getProperty("os.name")); 453 props.setProperty("os.version", System.getProperty("os.version")); 454 props.setProperty("os.arch", System.getProperty("os.arch")); 455 props.setProperty("jextract.args", Arrays.toString(args)); 456 ByteArrayOutputStream baos = new ByteArrayOutputStream(); 457 props.save(baos, "jextract meta data"); 458 return baos.toByteArray(); 459 } 460 461 void collectClassFiles(Path destDir, String[] args, String... pkgs) throws IOException { 462 try { 463 collectClasses(pkgs).entrySet().stream().forEach(e -> { 464 try { 465 String path = e.getKey().replace('.', File.separatorChar) + ".class"; 466 logger.fine(() -> "Writing " + path); 467 Path fullPath = destDir.resolve(path).normalize(); 468 Files.createDirectories(fullPath.getParent()); 469 try (OutputStream fos = Files.newOutputStream(fullPath)) { 470 fos.write(e.getValue()); 471 fos.flush(); 472 } 473 } catch (IOException ioe) { 474 throw new UncheckedIOException(ioe); 475 } 476 }); 477 478 Path propsPath = destDir.resolve(JEXTRACT_MANIFEST).normalize(); 479 Files.createDirectories(propsPath.getParent()); 480 try (OutputStream fos = Files.newOutputStream(propsPath)) { 481 fos.write(getJextractProperties(args)); 482 fos.flush(); 483 } 484 } catch (UncheckedIOException uioe) { 485 throw uioe.getCause(); 486 } 487 } 488 489 private void writeJar(AsmCodeFactory cf, JarOutputStream jar) { 490 cf.collect().entrySet().stream().forEach(e -> { 491 try { 492 String path = e.getKey().replace('.', File.separatorChar) + ".class"; 493 logger.fine(() -> "Add " + path); 494 jar.putNextEntry(new ZipEntry(path)); 495 jar.write(e.getValue()); 496 jar.closeEntry(); 497 } catch (IOException ioe) { 498 throw new UncheckedIOException(ioe); 499 } 500 }); 501 } 502 503 public void collectJarFile(final JarOutputStream jos, String[] args, String... pkgs) { 504 final Map<String, List<AsmCodeFactory>> mapPkgCf = getPkgCfMap(); 505 506 for (String pkg_name : pkgs) { 507 // convert '.' to '/' to use as a path 508 String entryName = Utils.toInternalName(pkg_name, ""); 509 // package folder 510 if (!entryName.isEmpty()) { 511 try { 512 jos.putNextEntry(new ZipEntry(entryName)); 513 } catch (IOException ex) { 514 throw new UncheckedIOException(ex); 515 } 516 } 517 logger.fine(() -> "Produce for package " + pkg_name); 518 mapPkgCf.getOrDefault(pkg_name, Collections.emptyList()) 519 .forEach(cf -> writeJar(cf, jos)); 520 } 521 522 try { 523 jos.putNextEntry(new ZipEntry(JEXTRACT_MANIFEST)); 524 jos.write(getJextractProperties(args)); 525 jos.closeEntry(); 526 } catch (IOException ioe) { 527 throw new UncheckedIOException(ioe); 528 } 529 } 530 531 void collectJarFile(final Path jar, String[] args, String... pkgs) throws IOException { 532 logger.info(() -> "Collecting jar file " + jar); 533 try (OutputStream os = Files.newOutputStream(jar, CREATE, TRUNCATE_EXISTING, WRITE); 534 JarOutputStream jo = new JarOutputStream(os)) { 535 collectJarFile(jo, args, pkgs); 536 } catch (UncheckedIOException uioe) { 537 throw uioe.getCause(); 538 } 539 } 540 541 /** 542 * Perform a local lookup, any undefined type will cause a JType 543 * be defined within origin scope. 544 * 545 * @param type The libclang type 546 * @param origin The path of the file where type is encountered 547 * @return The JType 548 */ 549 JType getJType(final Type type, Path origin) { 550 Path p = origin.normalize().toAbsolutePath(); 551 552 HeaderFile hf = headerMap.get(p); 553 // We should not encounter a type if the header file reference to it is not yet processed 554 assert(null != hf); 555 if (hf == null) { 556 throw new IllegalArgumentException("Failed to lookup header for " + p + " (origin: " + origin + ")"); 557 } 558 559 return hf.localLookup(type); 560 } 561 562 /** 563 * Perform a global lookup 564 * 565 * @param c The cursor define or declare the type. 566 * @return 567 */ 568 JType getJType(final Cursor c) { 569 if (c.isInvalid()) { 570 throw new IllegalArgumentException(); 571 } 572 SourceLocation loc = c.getSourceLocation(); 573 if (null == loc) { 574 return null; 575 } 576 Path p = loc.getFileLocation().path(); 577 if (null == p) { 578 return null; 579 } 580 return getJType(c.type(), p); 581 } 582 } --- EOF ---