1 /* 2 * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 package com.sun.tools.jextract; 24 25 import jdk.internal.clang.*; 26 import jdk.internal.foreign.LibrariesHelper; 27 28 import java.io.File; 29 import java.io.IOException; 30 import java.io.OutputStream; 31 import java.io.PrintWriter; 32 import java.io.UncheckedIOException; 33 import java.lang.invoke.MethodHandles; 34 import java.foreign.Library; 35 import java.nio.file.Files; 36 import java.nio.file.Path; 37 import java.util.ArrayList; 38 import java.util.Arrays; 39 import java.util.Collections; 40 import java.util.HashMap; 41 import java.util.List; 42 import java.util.Map; 43 import java.util.Set; 44 import java.util.TreeSet; 45 import java.util.function.Function; 46 import java.util.function.Predicate; 47 import java.util.jar.JarOutputStream; 48 import java.util.logging.Logger; 49 import java.util.regex.Pattern; 50 import java.util.stream.Collectors; 51 import java.util.zip.ZipEntry; 52 53 import static java.nio.file.StandardOpenOption.CREATE; 54 import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; 55 import static java.nio.file.StandardOpenOption.WRITE; 56 57 /** 58 * The setup for the tool execution 59 */ 60 public final class Context { 61 // package name to TypeDictionary 62 private final Map<String, TypeDictionary> tdMap; 63 // The folder path mapping to package name 64 private final Map<Path, String> pkgMap; 65 // The header file parsed 66 private final Map<Path, HeaderFile> headerMap; 67 // The args for parsing C 68 private final List<String> clangArgs; 69 // The set of source header files 70 private final Set<Path> sources; 71 // The list of library names 72 private final List<String> libraryNames; 73 // The list of library paths 74 private final List<String> libraryPaths; 75 // The list of library paths for link checks 76 private final List<String> linkCheckPaths; 77 // Symbol patterns to be excluded 78 private final List<Pattern> excludeSymbols; 79 80 final PrintWriter out; 81 final PrintWriter err; 82 83 private Predicate<String> symChecker; 84 private Predicate<String> symFilter; 85 86 private final MacroParser macroParser; 87 88 private final static String defaultPkg = "jextract.dump"; 89 final Logger logger = Logger.getLogger(getClass().getPackage().getName()); 90 91 public Context(PrintWriter out, PrintWriter err) { 92 this.tdMap = new HashMap<>(); 93 this.pkgMap = new HashMap<>(); 94 this.headerMap = new HashMap<>(); 95 this.clangArgs = new ArrayList<>(); 96 this.sources = new TreeSet<>(); 97 this.libraryNames = new ArrayList<>(); 98 this.libraryPaths = new ArrayList<>(); 99 this.linkCheckPaths = new ArrayList<>(); 100 this.excludeSymbols = new ArrayList<>(); 101 this.macroParser = new MacroParser(); 102 this.out = out; 103 this.err = err; 104 } 105 106 public Context() { 107 this(new PrintWriter(System.out, true), new PrintWriter(System.err, true)); 108 } 109 110 TypeDictionary typeDictionaryFor(String pkg) { 111 return tdMap.computeIfAbsent(pkg, p->new TypeDictionary(this, p)); 112 } 113 114 void addClangArg(String arg) { 115 clangArgs.add(arg); 116 } 117 118 public void addSource(Path path) { 119 sources.add(path); 120 } 121 122 void addLibraryName(String name) { 123 libraryNames.add(name); 124 } 125 126 void addLibraryPath(String path) { 127 libraryPaths.add(path); 128 } 129 130 void addLinkCheckPath(String path) { 131 linkCheckPaths.add(path); 132 } 133 134 void addExcludeSymbols(String pattern) { 135 excludeSymbols.add(Pattern.compile(pattern)); 136 } 137 138 boolean isSymbolFound(String name) { 139 return symChecker == null? true : symChecker.test(name); 140 } 141 142 boolean isSymbolExcluded(String name) { 143 return symFilter == null? false : symFilter.test(name); 144 } 145 146 /** 147 * Setup a package name for a given folder. 148 * 149 * @param folder The path to the folder, use null to set catch-all. 150 * @param pkg The package name 151 * @return True if the folder is setup successfully. False is a package 152 * has been assigned for the folder. 153 */ 154 public boolean usePackageForFolder(Path folder, String pkg) { 155 if (folder != null) { 156 folder = folder.toAbsolutePath(); 157 if (!Files.isDirectory(folder)) { 158 folder = folder.getParent(); 159 } 160 } 161 String existing = pkgMap.putIfAbsent(folder, pkg); 162 final String finalFolder = (null == folder) ? "all folders not configured" : folder.toString(); 163 if (null == existing) { 164 logger.config(() -> "Package " + pkg + " is selected for " + finalFolder); 165 return true; 166 } else { 167 logger.warning(() -> "Package " + existing + " had been selected for " + finalFolder + ", request to use " + pkg + " is ignored."); 168 return false; 169 } 170 } 171 172 static class Entity { 173 final String pkg; 174 final String entity; 175 176 Entity(String pkg, String entity) { 177 this.pkg = pkg; 178 this.entity = entity; 179 } 180 } 181 182 /** 183 * Determine package and interface name given a path. If the path is 184 * a folder, then only package name is determined. The package name is 185 * determined with the longest path matching the setup. If the path is not 186 * setup for any package, the default package name is returned. 187 * 188 * @param origin The source path 189 * @return The Entity 190 * @see Context::usePackageForFolder(Path, String) 191 */ 192 Entity whatis(Path origin) { 193 // normalize to absolute path 194 origin = origin.toAbsolutePath(); 195 String filename = null; 196 if (!Files.isDirectory(origin)) { 197 // ensure it's a folder name 198 filename = origin.getFileName().toString(); 199 origin = origin.getParent(); 200 } 201 Path path = origin; 202 203 // search the map for a hit with longest path 204 while (path != null && !pkgMap.containsKey(path)) { 205 path = path.getParent(); 206 } 207 208 int start; 209 String pkg; 210 if (path != null) { 211 start = path.getNameCount(); 212 pkg = pkgMap.get(path); 213 } else { 214 pkg = pkgMap.get(null); 215 if (pkg == null) { 216 start = 0; 217 pkg = defaultPkg; 218 } else { 219 start = origin.getNameCount(); 220 } 221 } 222 223 if (filename == null) { 224 // a folder, only pkg name matters 225 return new Entity(pkg, null); 226 } 227 228 StringBuilder sb = new StringBuilder(); 229 while (start < origin.getNameCount()) { 230 sb.append(Utils.toJavaIdentifier(origin.getName(start++).toString())); 231 sb.append("_"); 232 } 233 234 int ext = filename.lastIndexOf('.'); 235 if (ext != -1) { 236 sb.append(filename.substring(0, ext)); 237 } else { 238 sb.append(filename); 239 } 240 return new Entity(pkg, Utils.toClassName(sb.toString())); 241 } 242 243 HeaderFile getHeaderFile(Path header, HeaderFile main) { 244 if (!Files.isRegularFile(header)) { 245 logger.warning(() -> "Not a regular file: " + header.toString()); 246 throw new IllegalArgumentException(header.toString()); 247 } 248 249 final Context.Entity e = whatis(header); 250 return new HeaderFile(this, header, e.pkg, e.entity, main); 251 } 252 253 void defineMacro(Cursor cursor) { 254 String macroName = cursor.spelling(); 255 if (cursor.isMacroFunctionLike()) { 256 logger.fine(() -> "Skipping function-like macro " + macroName); 257 return; 258 } 259 260 261 if (macroParser.isDefined(macroName)) { 262 logger.fine(() -> "Macro " + macroName + " already handled"); 263 return; 264 } 265 266 logger.fine(() -> "Defining macro " + macroName); 267 268 TranslationUnit tu = cursor.getTranslationUnit(); 269 SourceRange range = cursor.getExtent(); 270 String[] tokens = tu.tokens(range); 271 272 macroParser.parse(cursor, tokens); 273 } 274 275 List<MacroParser.Macro> macros(HeaderFile header) { 276 return macroParser.macros().stream() 277 .filter(m -> m.getFileLocation().path().equals(header.path)) 278 .collect(Collectors.toList()); 279 } 280 281 void processCursor(Cursor c, HeaderFile main, Function<HeaderFile, AsmCodeFactory> fn) { 282 SourceLocation loc = c.getSourceLocation(); 283 if (loc == null) { 284 logger.info(() -> "Ignore Cursor " + c.spelling() + "@" + c.USR() + " has no SourceLocation"); 285 return; 286 } 287 logger.fine(() -> "Do cursor: " + c.spelling() + "@" + c.USR()); 288 289 HeaderFile header; 290 boolean isBuiltIn = false; 291 292 if (loc.isFromMainFile()) { 293 header = main; 294 } else { 295 SourceLocation.Location src = loc.getFileLocation(); 296 if (src == null) { 297 logger.info(() -> "Cursor " + c.spelling() + "@" + c.USR() + " has no FileLocation"); 298 return; 299 } 300 301 Path p = src.path(); 302 if (p == null) { 303 logger.fine(() -> "Found built-in type: " + c.spelling()); 304 header = main; 305 isBuiltIn = true; 306 } else { 307 p = p.normalize().toAbsolutePath(); 308 header = headerMap.get(p); 309 if (header == null) { 310 final HeaderFile hf = header = getHeaderFile(p, main); 311 logger.config(() -> "First encounter of header file " + hf.path + ", assigned to package " + hf.pkgName); 312 // Only generate code for header files sepcified or in the same package 313 // System headers are excluded, they need to be explicitly specified in jextract cmdline 314 if (sources.contains(p) || 315 (!loc.isInSystemHeader()) && (header.pkgName.equals(main.pkgName))) { 316 logger.config("Code gen for header " + p + " enabled in package " + header.pkgName); 317 header.useCodeFactory(fn.apply(header)); 318 } 319 headerMap.put(p, header); 320 } 321 } 322 } 323 324 header.processCursor(c, main, isBuiltIn); 325 } 326 327 public void parse() { 328 parse(header -> new AsmCodeFactory(this, header)); 329 } 330 331 public void parse(Function<HeaderFile, AsmCodeFactory> fn) { 332 if (!libraryNames.isEmpty() && !linkCheckPaths.isEmpty()) { 333 Library[] libs = LibrariesHelper.loadLibraries(MethodHandles.lookup(), 334 linkCheckPaths.toArray(new String[0]), 335 libraryNames.toArray(new String[0])); 336 337 // check if the given symbol is found in any of the libraries or not. 338 // If not found, warn the user for the missing symbol. 339 symChecker = name -> { 340 if (Main.DEBUG) { 341 err.println("Searching symbol: " + name); 342 } 343 return (Arrays.stream(libs).filter(lib -> { 344 try { 345 lib.lookup(name); 346 if (Main.DEBUG) { 347 err.println("Found symbol: " + name); 348 } 349 return true; 350 } catch (NoSuchMethodException nsme) { 351 return false; 352 } 353 }).findFirst().isPresent()); 354 }; 355 } else { 356 symChecker = null; 357 } 358 359 if (!excludeSymbols.isEmpty()) { 360 Pattern[] pats = excludeSymbols.toArray(new Pattern[0]); 361 symFilter = name -> { 362 return Arrays.stream(pats).filter(pat -> pat.matcher(name).matches()). 363 findFirst().isPresent(); 364 }; 365 } else { 366 symFilter = null; 367 } 368 369 sources.forEach(path -> { 370 if (headerMap.containsKey(path)) { 371 logger.info(() -> path.toString() + " seen earlier via #include"); 372 return; 373 } 374 375 HeaderFile hf = headerMap.computeIfAbsent(path, p -> getHeaderFile(p, null)); 376 hf.useLibraries(libraryNames, libraryPaths); 377 hf.useCodeFactory(fn.apply(hf)); 378 logger.info(() -> { 379 StringBuilder sb = new StringBuilder( 380 "Parsing header file " + path + " with following args:\n"); 381 int i = 0; 382 for (String arg : clangArgs) { 383 sb.append("arg["); 384 sb.append(i++); 385 sb.append("] = "); 386 sb.append(arg); 387 sb.append("\n"); 388 } 389 return sb.toString(); 390 }); 391 392 Index index = LibClang.createIndex(); 393 Cursor tuCursor = index.parse(path.toString(), 394 d -> { 395 err.println(d); 396 if (d.severity() > Diagnostic.CXDiagnostic_Warning) { 397 throw new RuntimeException(d.toString()); 398 } 399 }, 400 Main.INCLUDE_MACROS, 401 clangArgs.toArray(new String[0])); 402 403 tuCursor.children() 404 .peek(c -> logger.finest( 405 () -> "Cursor: " + c.spelling() + "@" + c.USR() + "?" + c.isDeclaration())) 406 .filter(c -> c.isDeclaration() || c.isPreprocessing()) 407 .forEach(c -> processCursor(c, hf, fn)); 408 }); 409 } 410 411 private Map<String, List<AsmCodeFactory>> getPkgCfMap() { 412 final Map<String, List<AsmCodeFactory>> mapPkgCf = new HashMap<>(); 413 // Build the pkg to CodeFactory map 414 headerMap.values().forEach(header -> { 415 AsmCodeFactory cf = header.getCodeFactory(); 416 String pkg = header.pkgName; 417 logger.config(() -> "File " + header + " is in package: " + pkg); 418 if (cf == null) { 419 logger.config(() -> "File " + header + " code generation is not activated!"); 420 return; 421 } 422 List<AsmCodeFactory> l = mapPkgCf.computeIfAbsent(pkg, k -> new ArrayList<>()); 423 l.add(cf); 424 logger.config(() -> "Add cf " + cf + " to pkg " + pkg + ", size is now " + l.size()); 425 }); 426 return Collections.unmodifiableMap(mapPkgCf); 427 } 428 429 public Map<String, byte[]> collectClasses(String... pkgs) { 430 final Map<String, byte[]> rv = new HashMap<>(); 431 final Map<String, List<AsmCodeFactory>> mapPkgCf = getPkgCfMap(); 432 for (String pkg_name : pkgs) { 433 mapPkgCf.getOrDefault(pkg_name, Collections.emptyList()) 434 .forEach(cf -> rv.putAll(cf.collect())); 435 } 436 return Collections.unmodifiableMap(rv); 437 } 438 439 void collectClassFiles(Path destDir, String... pkgs) throws IOException { 440 try { 441 collectClasses(pkgs).entrySet().stream().forEach(e -> { 442 try { 443 String path = e.getKey().replace('.', File.separatorChar) + ".class"; 444 logger.fine(() -> "Writing " + path); 445 Path fullPath = destDir.resolve(path).normalize(); 446 Files.createDirectories(fullPath.getParent()); 447 try (OutputStream fos = Files.newOutputStream(fullPath)) { 448 fos.write(e.getValue()); 449 fos.flush(); 450 } 451 } catch (IOException ioe) { 452 throw new UncheckedIOException(ioe); 453 } 454 }); 455 } catch (UncheckedIOException uioe) { 456 throw uioe.getCause(); 457 } 458 } 459 460 private void writeJar(AsmCodeFactory cf, JarOutputStream jar) { 461 cf.collect().entrySet().stream().forEach(e -> { 462 try { 463 String path = e.getKey().replace('.', File.separatorChar) + ".class"; 464 logger.fine(() -> "Add " + path); 465 jar.putNextEntry(new ZipEntry(path)); 466 jar.write(e.getValue()); 467 jar.closeEntry(); 468 } catch (IOException ioe) { 469 throw new UncheckedIOException(ioe); 470 } 471 }); 472 } 473 474 public void collectJarFile(final JarOutputStream jos, String... pkgs) { 475 final Map<String, List<AsmCodeFactory>> mapPkgCf = getPkgCfMap(); 476 477 for (String pkg_name : pkgs) { 478 // convert '.' to '/' to use as a path 479 String entryName = Utils.toInternalName(pkg_name, ""); 480 // package folder 481 if (!entryName.isEmpty()) { 482 try { 483 jos.putNextEntry(new ZipEntry(entryName)); 484 } catch (IOException ex) { 485 throw new UncheckedIOException(ex); 486 } 487 } 488 logger.fine(() -> "Produce for package " + pkg_name); 489 mapPkgCf.getOrDefault(pkg_name, Collections.emptyList()) 490 .forEach(cf -> writeJar(cf, jos)); 491 } 492 } 493 494 void collectJarFile(final Path jar, String... pkgs) throws IOException { 495 logger.info(() -> "Collecting jar file " + jar); 496 try (OutputStream os = Files.newOutputStream(jar, CREATE, TRUNCATE_EXISTING, WRITE); 497 JarOutputStream jo = new JarOutputStream(os)) { 498 collectJarFile(jo, pkgs); 499 } catch (UncheckedIOException uioe) { 500 throw uioe.getCause(); 501 } 502 } 503 504 /** 505 * Perform a local lookup, any undefined type will cause a JType 506 * be defined within origin scope. 507 * 508 * @param type The libclang type 509 * @param origin The path of the file where type is encountered 510 * @return The JType 511 */ 512 JType getJType(final Type type, Path origin) { 513 Path p = origin.normalize().toAbsolutePath(); 514 515 HeaderFile hf = headerMap.get(p); 516 // We should not encounter a type if the header file reference to it is not yet processed 517 assert(null != hf); 518 if (hf == null) { 519 throw new IllegalArgumentException("Failed to lookup header for " + p + " (origin: " + origin + ")"); 520 } 521 522 return hf.localLookup(type); 523 } 524 525 /** 526 * Perform a global lookup 527 * 528 * @param c The cursor define or declare the type. 529 * @return 530 */ 531 JType getJType(final Cursor c) { 532 if (c.isInvalid()) { 533 throw new IllegalArgumentException(); 534 } 535 SourceLocation loc = c.getSourceLocation(); 536 if (null == loc) { 537 return null; 538 } 539 Path p = loc.getFileLocation().path(); 540 if (null == p) { 541 return null; 542 } 543 return getJType(c.type(), p); 544 } 545 }