1 /* 2 * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 package com.sun.tools.jextract; 24 25 import jdk.internal.clang.*; 26 import jdk.internal.foreign.LibrariesHelper; 27 28 import java.io.ByteArrayOutputStream; 29 import java.io.File; 30 import java.io.IOException; 31 import java.io.OutputStream; 32 import java.io.PrintWriter; 33 import java.io.UncheckedIOException; 34 import java.lang.invoke.MethodHandles; 35 import java.foreign.Library; 36 import java.nio.file.Files; 37 import java.nio.file.Path; 38 import java.util.ArrayList; 39 import java.util.Arrays; 40 import java.util.Collections; 41 import java.util.HashMap; 42 import java.util.List; 43 import java.util.Map; 44 import java.util.Properties; 45 import java.util.Set; 46 import java.util.TreeSet; 47 import java.util.function.Function; 48 import java.util.function.Predicate; 49 import java.util.jar.JarOutputStream; 50 import java.util.logging.Logger; 51 import java.util.regex.Pattern; 52 import java.util.stream.Collectors; 53 import java.util.zip.ZipEntry; 54 55 import static java.nio.file.StandardOpenOption.CREATE; 56 import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; 57 import static java.nio.file.StandardOpenOption.WRITE; 58 59 /** 60 * The setup for the tool execution 61 */ 62 public final class Context { 63 // package name to TypeDictionary 64 private final Map<String, TypeDictionary> tdMap; 65 // The folder path mapping to package name 66 private final Map<Path, String> pkgMap; 67 // The header file parsed 68 private final Map<Path, HeaderFile> headerMap; 69 // The args for parsing C 70 private final List<String> clangArgs; 71 // The set of source header files 72 private final Set<Path> sources; 73 // The list of library names 74 private final List<String> libraryNames; 75 // The list of library paths 76 private final List<String> libraryPaths; 77 // The list of library paths for link checks 78 private final List<String> linkCheckPaths; 79 // Symbol patterns to be excluded 80 private final List<Pattern> excludeSymbols; 81 82 final PrintWriter out; 83 final PrintWriter err; 84 85 private Predicate<String> symChecker; 86 private Predicate<String> symFilter; 87 88 private final MacroParser macroParser; 89 90 private final static String defaultPkg = "jextract.dump"; 91 final Logger logger = Logger.getLogger(getClass().getPackage().getName()); 92 93 public Context(PrintWriter out, PrintWriter err) { 94 this.tdMap = new HashMap<>(); 95 this.pkgMap = new HashMap<>(); 96 this.headerMap = new HashMap<>(); 97 this.clangArgs = new ArrayList<>(); 98 this.sources = new TreeSet<>(); 99 this.libraryNames = new ArrayList<>(); 100 this.libraryPaths = new ArrayList<>(); 101 this.linkCheckPaths = new ArrayList<>(); 102 this.excludeSymbols = new ArrayList<>(); 103 this.macroParser = new MacroParser(); 104 this.out = out; 105 this.err = err; 106 } 107 108 public Context() { 109 this(new PrintWriter(System.out, true), new PrintWriter(System.err, true)); 110 } 111 112 TypeDictionary typeDictionaryFor(String pkg) { 113 return tdMap.computeIfAbsent(pkg, p->new TypeDictionary(this, p)); 114 } 115 116 void addClangArg(String arg) { 117 clangArgs.add(arg); 118 } 119 120 public void addSource(Path path) { 121 sources.add(path); 122 } 123 124 void addLibraryName(String name) { 125 libraryNames.add(name); 126 } 127 128 void addLibraryPath(String path) { 129 libraryPaths.add(path); 130 } 131 132 void addLinkCheckPath(String path) { 133 linkCheckPaths.add(path); 134 } 135 136 void addExcludeSymbols(String pattern) { 137 excludeSymbols.add(Pattern.compile(pattern)); 138 } 139 140 boolean isSymbolFound(String name) { 141 return symChecker == null? true : symChecker.test(name); 142 } 143 144 boolean isSymbolExcluded(String name) { 145 return symFilter == null? false : symFilter.test(name); 146 } 147 148 /** 149 * Setup a package name for a given folder. 150 * 151 * @param folder The path to the folder, use null to set catch-all. 152 * @param pkg The package name 153 * @return True if the folder is setup successfully. False is a package 154 * has been assigned for the folder. 155 */ 156 public boolean usePackageForFolder(Path folder, String pkg) { 157 if (folder != null) { 158 folder = folder.toAbsolutePath(); 159 if (!Files.isDirectory(folder)) { 160 folder = folder.getParent(); 161 } 162 } 163 String existing = pkgMap.putIfAbsent(folder, pkg); 164 final String finalFolder = (null == folder) ? "all folders not configured" : folder.toString(); 165 if (null == existing) { 166 logger.config(() -> "Package " + pkg + " is selected for " + finalFolder); 167 return true; 168 } else { 169 logger.warning(() -> "Package " + existing + " had been selected for " + finalFolder + ", request to use " + pkg + " is ignored."); 170 return false; 171 } 172 } 173 174 static class Entity { 175 final String pkg; 176 final String entity; 177 178 Entity(String pkg, String entity) { 179 this.pkg = pkg; 180 this.entity = entity; 181 } 182 } 183 184 /** 185 * Determine package and interface name given a path. If the path is 186 * a folder, then only package name is determined. The package name is 187 * determined with the longest path matching the setup. If the path is not 188 * setup for any package, the default package name is returned. 189 * 190 * @param origin The source path 191 * @return The Entity 192 * @see Context::usePackageForFolder(Path, String) 193 */ 194 Entity whatis(Path origin) { 195 // normalize to absolute path 196 origin = origin.toAbsolutePath(); 197 String filename = null; 198 if (!Files.isDirectory(origin)) { 199 // ensure it's a folder name 200 filename = origin.getFileName().toString(); 201 origin = origin.getParent(); 202 } 203 Path path = origin; 204 205 // search the map for a hit with longest path 206 while (path != null && !pkgMap.containsKey(path)) { 207 path = path.getParent(); 208 } 209 210 int start; 211 String pkg; 212 if (path != null) { 213 start = path.getNameCount(); 214 pkg = pkgMap.get(path); 215 } else { 216 pkg = pkgMap.get(null); 217 if (pkg == null) { 218 start = 0; 219 pkg = defaultPkg; 220 } else { 221 start = origin.getNameCount(); 222 } 223 } 224 225 if (filename == null) { 226 // a folder, only pkg name matters 227 return new Entity(pkg, null); 228 } 229 230 StringBuilder sb = new StringBuilder(); 231 while (start < origin.getNameCount()) { 232 sb.append(Utils.toJavaIdentifier(origin.getName(start++).toString())); 233 sb.append("_"); 234 } 235 236 int ext = filename.lastIndexOf('.'); 237 if (ext != -1) { 238 sb.append(filename.substring(0, ext)); 239 } else { 240 sb.append(filename); 241 } 242 return new Entity(pkg, Utils.toClassName(sb.toString())); 243 } 244 245 HeaderFile getHeaderFile(Path header, HeaderFile main) { 246 if (!Files.isRegularFile(header)) { 247 logger.warning(() -> "Not a regular file: " + header.toString()); 248 throw new IllegalArgumentException(header.toString()); 249 } 250 251 final Context.Entity e = whatis(header); 252 return new HeaderFile(this, header, e.pkg, e.entity, main); 253 } 254 255 void defineMacro(Cursor cursor) { 256 String macroName = cursor.spelling(); 257 if (cursor.isMacroFunctionLike()) { 258 logger.fine(() -> "Skipping function-like macro " + macroName); 259 return; 260 } 261 262 263 if (macroParser.isDefined(macroName)) { 264 logger.fine(() -> "Macro " + macroName + " already handled"); 265 return; 266 } 267 268 logger.fine(() -> "Defining macro " + macroName); 269 270 TranslationUnit tu = cursor.getTranslationUnit(); 271 SourceRange range = cursor.getExtent(); 272 String[] tokens = tu.tokens(range); 273 274 macroParser.parse(cursor, tokens); 275 } 276 277 List<MacroParser.Macro> macros(HeaderFile header) { 278 return macroParser.macros().stream() 279 .filter(m -> m.getFileLocation().path().equals(header.path)) 280 .collect(Collectors.toList()); 281 } 282 283 void processCursor(Cursor c, HeaderFile main, Function<HeaderFile, AsmCodeFactory> fn) { 284 SourceLocation loc = c.getSourceLocation(); 285 if (loc == null) { 286 logger.info(() -> "Ignore Cursor " + c.spelling() + "@" + c.USR() + " has no SourceLocation"); 287 return; 288 } 289 logger.fine(() -> "Do cursor: " + c.spelling() + "@" + c.USR()); 290 291 HeaderFile header; 292 boolean isBuiltIn = false; 293 294 if (loc.isFromMainFile()) { 295 header = main; 296 } else { 297 SourceLocation.Location src = loc.getFileLocation(); 298 if (src == null) { 299 logger.info(() -> "Cursor " + c.spelling() + "@" + c.USR() + " has no FileLocation"); 300 return; 301 } 302 303 Path p = src.path(); 304 if (p == null) { 305 logger.fine(() -> "Found built-in type: " + c.spelling()); 306 header = main; 307 isBuiltIn = true; 308 } else { 309 p = p.normalize().toAbsolutePath(); 310 header = headerMap.get(p); 311 if (header == null) { 312 final HeaderFile hf = header = getHeaderFile(p, main); 313 logger.config(() -> "First encounter of header file " + hf.path + ", assigned to package " + hf.pkgName); 314 // Only generate code for header files sepcified or in the same package 315 // System headers are excluded, they need to be explicitly specified in jextract cmdline 316 if (sources.contains(p) || 317 (!loc.isInSystemHeader()) && (header.pkgName.equals(main.pkgName))) { 318 logger.config("Code gen for header " + p + " enabled in package " + header.pkgName); 319 header.useCodeFactory(fn.apply(header)); 320 } 321 headerMap.put(p, header); 322 } 323 } 324 } 325 326 header.processCursor(c, main, isBuiltIn); 327 } 328 329 public void parse() { 330 parse(header -> new AsmCodeFactory(this, header)); 331 } 332 333 public void parse(Function<HeaderFile, AsmCodeFactory> fn) { 334 if (!libraryNames.isEmpty() && !linkCheckPaths.isEmpty()) { 335 Library[] libs = LibrariesHelper.loadLibraries(MethodHandles.lookup(), 336 linkCheckPaths.toArray(new String[0]), 337 libraryNames.toArray(new String[0])); 338 339 // check if the given symbol is found in any of the libraries or not. 340 // If not found, warn the user for the missing symbol. 341 symChecker = name -> { 342 if (Main.DEBUG) { 343 err.println("Searching symbol: " + name); 344 } 345 return (Arrays.stream(libs).filter(lib -> { 346 try { 347 lib.lookup(name); 348 if (Main.DEBUG) { 349 err.println("Found symbol: " + name); 350 } 351 return true; 352 } catch (NoSuchMethodException nsme) { 353 return false; 354 } 355 }).findFirst().isPresent()); 356 }; 357 } else { 358 symChecker = null; 359 } 360 361 if (!excludeSymbols.isEmpty()) { 362 Pattern[] pats = excludeSymbols.toArray(new Pattern[0]); 363 symFilter = name -> { 364 return Arrays.stream(pats).filter(pat -> pat.matcher(name).matches()). 365 findFirst().isPresent(); 366 }; 367 } else { 368 symFilter = null; 369 } 370 371 sources.forEach(path -> { 372 if (headerMap.containsKey(path)) { 373 logger.info(() -> path.toString() + " seen earlier via #include"); 374 return; 375 } 376 377 HeaderFile hf = headerMap.computeIfAbsent(path, p -> getHeaderFile(p, null)); 378 hf.useLibraries(libraryNames, libraryPaths); 379 hf.useCodeFactory(fn.apply(hf)); 380 logger.info(() -> { 381 StringBuilder sb = new StringBuilder( 382 "Parsing header file " + path + " with following args:\n"); 383 int i = 0; 384 for (String arg : clangArgs) { 385 sb.append("arg["); 386 sb.append(i++); 387 sb.append("] = "); 388 sb.append(arg); 389 sb.append("\n"); 390 } 391 return sb.toString(); 392 }); 393 394 Index index = LibClang.createIndex(); 395 Cursor tuCursor = index.parse(path.toString(), 396 d -> { 397 err.println(d); 398 if (d.severity() > Diagnostic.CXDiagnostic_Warning) { 399 throw new RuntimeException(d.toString()); 400 } 401 }, 402 Main.INCLUDE_MACROS, 403 clangArgs.toArray(new String[0])); 404 405 tuCursor.children() 406 .peek(c -> logger.finest( 407 () -> "Cursor: " + c.spelling() + "@" + c.USR() + "?" + c.isDeclaration())) 408 .filter(c -> c.isDeclaration() || c.isPreprocessing()) 409 .forEach(c -> processCursor(c, hf, fn)); 410 }); 411 } 412 413 private Map<String, List<AsmCodeFactory>> getPkgCfMap() { 414 final Map<String, List<AsmCodeFactory>> mapPkgCf = new HashMap<>(); 415 // Build the pkg to CodeFactory map 416 headerMap.values().forEach(header -> { 417 AsmCodeFactory cf = header.getCodeFactory(); 418 String pkg = header.pkgName; 419 logger.config(() -> "File " + header + " is in package: " + pkg); 420 if (cf == null) { 421 logger.config(() -> "File " + header + " code generation is not activated!"); 422 return; 423 } 424 List<AsmCodeFactory> l = mapPkgCf.computeIfAbsent(pkg, k -> new ArrayList<>()); 425 l.add(cf); 426 logger.config(() -> "Add cf " + cf + " to pkg " + pkg + ", size is now " + l.size()); 427 }); 428 return Collections.unmodifiableMap(mapPkgCf); 429 } 430 431 public Map<String, byte[]> collectClasses(String... pkgs) { 432 final Map<String, byte[]> rv = new HashMap<>(); 433 final Map<String, List<AsmCodeFactory>> mapPkgCf = getPkgCfMap(); 434 for (String pkg_name : pkgs) { 435 mapPkgCf.getOrDefault(pkg_name, Collections.emptyList()) 436 .forEach(cf -> rv.putAll(cf.collect())); 437 } 438 return Collections.unmodifiableMap(rv); 439 } 440 441 private static final String JEXTRACT_MANIFEST = "META-INFO" + File.separatorChar + "jextract.properties"; 442 443 @SuppressWarnings("deprecation") 444 private byte[] getJextractProperties(String[] args) { 445 Properties props = new Properties(); 446 props.setProperty("os.name", System.getProperty("os.name")); 447 props.setProperty("os.version", System.getProperty("os.version")); 448 props.setProperty("os.arch", System.getProperty("os.arch")); 449 props.setProperty("jextract.args", Arrays.toString(args)); 450 ByteArrayOutputStream baos = new ByteArrayOutputStream(); 451 props.save(baos, "jextract meta data"); 452 return baos.toByteArray(); 453 } 454 455 void collectClassFiles(Path destDir, String[] args, String... pkgs) throws IOException { 456 try { 457 collectClasses(pkgs).entrySet().stream().forEach(e -> { 458 try { 459 String path = e.getKey().replace('.', File.separatorChar) + ".class"; 460 logger.fine(() -> "Writing " + path); 461 Path fullPath = destDir.resolve(path).normalize(); 462 Files.createDirectories(fullPath.getParent()); 463 try (OutputStream fos = Files.newOutputStream(fullPath)) { 464 fos.write(e.getValue()); 465 fos.flush(); 466 } 467 } catch (IOException ioe) { 468 throw new UncheckedIOException(ioe); 469 } 470 }); 471 472 Path propsPath = destDir.resolve(JEXTRACT_MANIFEST).normalize(); 473 Files.createDirectories(propsPath.getParent()); 474 try (OutputStream fos = Files.newOutputStream(propsPath)) { 475 fos.write(getJextractProperties(args)); 476 fos.flush(); 477 } 478 } catch (UncheckedIOException uioe) { 479 throw uioe.getCause(); 480 } 481 } 482 483 private void writeJar(AsmCodeFactory cf, JarOutputStream jar, String[] args) { 484 cf.collect().entrySet().stream().forEach(e -> { 485 try { 486 String path = e.getKey().replace('.', File.separatorChar) + ".class"; 487 logger.fine(() -> "Add " + path); 488 jar.putNextEntry(new ZipEntry(path)); 489 jar.write(e.getValue()); 490 jar.closeEntry(); 491 } catch (IOException ioe) { 492 throw new UncheckedIOException(ioe); 493 } 494 }); 495 } 496 497 public void collectJarFile(final JarOutputStream jos, String[] args, String... pkgs) { 498 final Map<String, List<AsmCodeFactory>> mapPkgCf = getPkgCfMap(); 499 500 for (String pkg_name : pkgs) { 501 // convert '.' to '/' to use as a path 502 String entryName = Utils.toInternalName(pkg_name, ""); 503 // package folder 504 if (!entryName.isEmpty()) { 505 try { 506 jos.putNextEntry(new ZipEntry(entryName)); 507 } catch (IOException ex) { 508 throw new UncheckedIOException(ex); 509 } 510 } 511 logger.fine(() -> "Produce for package " + pkg_name); 512 mapPkgCf.getOrDefault(pkg_name, Collections.emptyList()) 513 .forEach(cf -> writeJar(cf, jos, args)); 514 } 515 516 try { 517 jos.putNextEntry(new ZipEntry(JEXTRACT_MANIFEST)); 518 jos.write(getJextractProperties(args)); 519 jos.closeEntry(); 520 } catch (IOException ioe) { 521 throw new UncheckedIOException(ioe); 522 } 523 } 524 525 void collectJarFile(final Path jar, String[] args, String... pkgs) throws IOException { 526 logger.info(() -> "Collecting jar file " + jar); 527 try (OutputStream os = Files.newOutputStream(jar, CREATE, TRUNCATE_EXISTING, WRITE); 528 JarOutputStream jo = new JarOutputStream(os)) { 529 collectJarFile(jo, args, pkgs); 530 } catch (UncheckedIOException uioe) { 531 throw uioe.getCause(); 532 } 533 } 534 535 /** 536 * Perform a local lookup, any undefined type will cause a JType 537 * be defined within origin scope. 538 * 539 * @param type The libclang type 540 * @param origin The path of the file where type is encountered 541 * @return The JType 542 */ 543 JType getJType(final Type type, Path origin) { 544 Path p = origin.normalize().toAbsolutePath(); 545 546 HeaderFile hf = headerMap.get(p); 547 // We should not encounter a type if the header file reference to it is not yet processed 548 assert(null != hf); 549 if (hf == null) { 550 throw new IllegalArgumentException("Failed to lookup header for " + p + " (origin: " + origin + ")"); 551 } 552 553 return hf.localLookup(type); 554 } 555 556 /** 557 * Perform a global lookup 558 * 559 * @param c The cursor define or declare the type. 560 * @return 561 */ 562 JType getJType(final Cursor c) { 563 if (c.isInvalid()) { 564 throw new IllegalArgumentException(); 565 } 566 SourceLocation loc = c.getSourceLocation(); 567 if (null == loc) { 568 return null; 569 } 570 Path p = loc.getFileLocation().path(); 571 if (null == p) { 572 return null; 573 } 574 return getJType(c.type(), p); 575 } 576 } --- EOF ---