1 /* 2 * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 package com.sun.tools.jextract; 24 25 import jdk.internal.clang.*; 26 import jdk.internal.foreign.LibrariesHelper; 27 28 import java.io.ByteArrayOutputStream; 29 import java.io.File; 30 import java.io.IOException; 31 import java.io.OutputStream; 32 import java.io.PrintWriter; 33 import java.io.UncheckedIOException; 34 import java.lang.invoke.MethodHandles; 35 import java.foreign.Library; 36 import java.nio.file.Files; 37 import java.nio.file.Path; 38 import java.util.ArrayList; 39 import java.util.Arrays; 40 import java.util.Collections; 41 import java.util.HashMap; 42 import java.util.List; 43 import java.util.Map; 44 import java.util.Properties; 45 import java.util.Set; 46 import java.util.TreeSet; 47 import java.util.function.Function; 48 import java.util.function.Predicate; 49 import java.util.jar.JarOutputStream; 50 import java.util.logging.Logger; 51 import java.util.regex.Pattern; 52 import java.util.stream.Collectors; 53 import java.util.zip.ZipEntry; 54 55 import static java.nio.file.StandardOpenOption.CREATE; 56 import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; 57 import static java.nio.file.StandardOpenOption.WRITE; 58 59 /** 60 * The setup for the tool execution 61 */ 62 public final class Context { 63 // package name to TypeDictionary 64 private final Map<String, TypeDictionary> tdMap; 65 // The folder path mapping to package name 66 private final Map<Path, String> pkgMap; 67 // The header file parsed 68 private final Map<Path, HeaderFile> headerMap; 69 // The args for parsing C 70 private final List<String> clangArgs; 71 // The set of source header files 72 private final Set<Path> sources; 73 // The list of library names 74 private final List<String> libraryNames; 75 // The list of library paths 76 private final List<String> libraryPaths; 77 // The list of library paths for link checks 78 private final List<String> linkCheckPaths; 79 // Symbol patterns to be excluded 80 private final List<Pattern> excludeSymbols; 81 82 final PrintWriter out; 83 final PrintWriter err; 84 final String[] args; 85 86 private Predicate<String> symChecker; 87 private Predicate<String> symFilter; 88 89 private final MacroParser macroParser; 90 91 private final static String defaultPkg = "jextract.dump"; 92 final Logger logger = Logger.getLogger(getClass().getPackage().getName()); 93 94 public Context(PrintWriter out, PrintWriter err, String[] args) { 95 this.tdMap = new HashMap<>(); 96 this.pkgMap = new HashMap<>(); 97 this.headerMap = new HashMap<>(); 98 this.clangArgs = new ArrayList<>(); 99 this.sources = new TreeSet<>(); 100 this.libraryNames = new ArrayList<>(); 101 this.libraryPaths = new ArrayList<>(); 102 this.linkCheckPaths = new ArrayList<>(); 103 this.excludeSymbols = new ArrayList<>(); 104 this.macroParser = new MacroParser(); 105 this.out = out; 106 this.err = err; 107 this.args = args; 108 } 109 110 public Context(String[] args) { 111 this(new PrintWriter(System.out, true), new PrintWriter(System.err, true), args); 112 } 113 114 TypeDictionary typeDictionaryFor(String pkg) { 115 return tdMap.computeIfAbsent(pkg, p->new TypeDictionary(this, p)); 116 } 117 118 void addClangArg(String arg) { 119 clangArgs.add(arg); 120 } 121 122 public void addSource(Path path) { 123 sources.add(path); 124 } 125 126 void addLibraryName(String name) { 127 libraryNames.add(name); 128 } 129 130 void addLibraryPath(String path) { 131 libraryPaths.add(path); 132 } 133 134 void addLinkCheckPath(String path) { 135 linkCheckPaths.add(path); 136 } 137 138 void addExcludeSymbols(String pattern) { 139 excludeSymbols.add(Pattern.compile(pattern)); 140 } 141 142 boolean isSymbolFound(String name) { 143 return symChecker == null? true : symChecker.test(name); 144 } 145 146 boolean isSymbolExcluded(String name) { 147 return symFilter == null? false : symFilter.test(name); 148 } 149 150 /** 151 * Setup a package name for a given folder. 152 * 153 * @param folder The path to the folder, use null to set catch-all. 154 * @param pkg The package name 155 * @return True if the folder is setup successfully. False is a package 156 * has been assigned for the folder. 157 */ 158 public boolean usePackageForFolder(Path folder, String pkg) { 159 if (folder != null) { 160 folder = folder.toAbsolutePath(); 161 if (!Files.isDirectory(folder)) { 162 folder = folder.getParent(); 163 } 164 } 165 String existing = pkgMap.putIfAbsent(folder, pkg); 166 final String finalFolder = (null == folder) ? "all folders not configured" : folder.toString(); 167 if (null == existing) { 168 logger.config(() -> "Package " + pkg + " is selected for " + finalFolder); 169 return true; 170 } else { 171 logger.warning(() -> "Package " + existing + " had been selected for " + finalFolder + ", request to use " + pkg + " is ignored."); 172 return false; 173 } 174 } 175 176 static class Entity { 177 final String pkg; 178 final String entity; 179 180 Entity(String pkg, String entity) { 181 this.pkg = pkg; 182 this.entity = entity; 183 } 184 } 185 186 /** 187 * Determine package and interface name given a path. If the path is 188 * a folder, then only package name is determined. The package name is 189 * determined with the longest path matching the setup. If the path is not 190 * setup for any package, the default package name is returned. 191 * 192 * @param origin The source path 193 * @return The Entity 194 * @see Context::usePackageForFolder(Path, String) 195 */ 196 Entity whatis(Path origin) { 197 // normalize to absolute path 198 origin = origin.toAbsolutePath(); 199 String filename = null; 200 if (!Files.isDirectory(origin)) { 201 // ensure it's a folder name 202 filename = origin.getFileName().toString(); 203 origin = origin.getParent(); 204 } 205 Path path = origin; 206 207 // search the map for a hit with longest path 208 while (path != null && !pkgMap.containsKey(path)) { 209 path = path.getParent(); 210 } 211 212 int start; 213 String pkg; 214 if (path != null) { 215 start = path.getNameCount(); 216 pkg = pkgMap.get(path); 217 } else { 218 pkg = pkgMap.get(null); 219 if (pkg == null) { 220 start = 0; 221 pkg = defaultPkg; 222 } else { 223 start = origin.getNameCount(); 224 } 225 } 226 227 if (filename == null) { 228 // a folder, only pkg name matters 229 return new Entity(pkg, null); 230 } 231 232 StringBuilder sb = new StringBuilder(); 233 while (start < origin.getNameCount()) { 234 sb.append(Utils.toJavaIdentifier(origin.getName(start++).toString())); 235 sb.append("_"); 236 } 237 238 int ext = filename.lastIndexOf('.'); 239 if (ext != -1) { 240 sb.append(filename.substring(0, ext)); 241 } else { 242 sb.append(filename); 243 } 244 return new Entity(pkg, Utils.toClassName(sb.toString())); 245 } 246 247 HeaderFile getHeaderFile(Path header, HeaderFile main) { 248 if (!Files.isRegularFile(header)) { 249 logger.warning(() -> "Not a regular file: " + header.toString()); 250 throw new IllegalArgumentException(header.toString()); 251 } 252 253 final Context.Entity e = whatis(header); 254 return new HeaderFile(this, header, e.pkg, e.entity, main); 255 } 256 257 void defineMacro(Cursor cursor) { 258 String macroName = cursor.spelling(); 259 if (cursor.isMacroFunctionLike()) { 260 logger.fine(() -> "Skipping function-like macro " + macroName); 261 return; 262 } 263 264 265 if (macroParser.isDefined(macroName)) { 266 logger.fine(() -> "Macro " + macroName + " already handled"); 267 return; 268 } 269 270 logger.fine(() -> "Defining macro " + macroName); 271 272 TranslationUnit tu = cursor.getTranslationUnit(); 273 SourceRange range = cursor.getExtent(); 274 String[] tokens = tu.tokens(range); 275 276 macroParser.parse(cursor, tokens); 277 } 278 279 List<MacroParser.Macro> macros(HeaderFile header) { 280 return macroParser.macros().stream() 281 .filter(m -> m.getFileLocation().path().equals(header.path)) 282 .collect(Collectors.toList()); 283 } 284 285 void processCursor(Cursor c, HeaderFile main, Function<HeaderFile, AsmCodeFactory> fn) { 286 SourceLocation loc = c.getSourceLocation(); 287 if (loc == null) { 288 logger.info(() -> "Ignore Cursor " + c.spelling() + "@" + c.USR() + " has no SourceLocation"); 289 return; 290 } 291 logger.fine(() -> "Do cursor: " + c.spelling() + "@" + c.USR()); 292 293 HeaderFile header; 294 boolean isBuiltIn = false; 295 296 if (loc.isFromMainFile()) { 297 header = main; 298 } else { 299 SourceLocation.Location src = loc.getFileLocation(); 300 if (src == null) { 301 logger.info(() -> "Cursor " + c.spelling() + "@" + c.USR() + " has no FileLocation"); 302 return; 303 } 304 305 Path p = src.path(); 306 if (p == null) { 307 logger.fine(() -> "Found built-in type: " + c.spelling()); 308 header = main; 309 isBuiltIn = true; 310 } else { 311 p = p.normalize().toAbsolutePath(); 312 header = headerMap.get(p); 313 if (header == null) { 314 final HeaderFile hf = header = getHeaderFile(p, main); 315 logger.config(() -> "First encounter of header file " + hf.path + ", assigned to package " + hf.pkgName); 316 // Only generate code for header files sepcified or in the same package 317 // System headers are excluded, they need to be explicitly specified in jextract cmdline 318 if (sources.contains(p) || 319 (!loc.isInSystemHeader()) && (header.pkgName.equals(main.pkgName))) { 320 logger.config("Code gen for header " + p + " enabled in package " + header.pkgName); 321 header.useCodeFactory(fn.apply(header)); 322 } 323 headerMap.put(p, header); 324 } 325 } 326 } 327 328 header.processCursor(c, main, isBuiltIn); 329 } 330 331 public void parse() { 332 parse(header -> new AsmCodeFactory(this, header)); 333 } 334 335 public void parse(Function<HeaderFile, AsmCodeFactory> fn) { 336 if (!libraryNames.isEmpty() && !linkCheckPaths.isEmpty()) { 337 Library[] libs = LibrariesHelper.loadLibraries(MethodHandles.lookup(), 338 linkCheckPaths.toArray(new String[0]), 339 libraryNames.toArray(new String[0])); 340 341 // check if the given symbol is found in any of the libraries or not. 342 // If not found, warn the user for the missing symbol. 343 symChecker = name -> { 344 if (Main.DEBUG) { 345 err.println("Searching symbol: " + name); 346 } 347 return (Arrays.stream(libs).filter(lib -> { 348 try { 349 lib.lookup(name); 350 if (Main.DEBUG) { 351 err.println("Found symbol: " + name); 352 } 353 return true; 354 } catch (NoSuchMethodException nsme) { 355 return false; 356 } 357 }).findFirst().isPresent()); 358 }; 359 } else { 360 symChecker = null; 361 } 362 363 if (!excludeSymbols.isEmpty()) { 364 Pattern[] pats = excludeSymbols.toArray(new Pattern[0]); 365 symFilter = name -> { 366 return Arrays.stream(pats).filter(pat -> pat.matcher(name).matches()). 367 findFirst().isPresent(); 368 }; 369 } else { 370 symFilter = null; 371 } 372 373 sources.forEach(path -> { 374 if (headerMap.containsKey(path)) { 375 logger.info(() -> path.toString() + " seen earlier via #include"); 376 return; 377 } 378 379 HeaderFile hf = headerMap.computeIfAbsent(path, p -> getHeaderFile(p, null)); 380 hf.useLibraries(libraryNames, libraryPaths); 381 hf.useCodeFactory(fn.apply(hf)); 382 logger.info(() -> { 383 StringBuilder sb = new StringBuilder( 384 "Parsing header file " + path + " with following args:\n"); 385 int i = 0; 386 for (String arg : clangArgs) { 387 sb.append("arg["); 388 sb.append(i++); 389 sb.append("] = "); 390 sb.append(arg); 391 sb.append("\n"); 392 } 393 return sb.toString(); 394 }); 395 396 Index index = LibClang.createIndex(); 397 Cursor tuCursor = index.parse(path.toString(), 398 d -> { 399 err.println(d); 400 if (d.severity() > Diagnostic.CXDiagnostic_Warning) { 401 throw new RuntimeException(d.toString()); 402 } 403 }, 404 Main.INCLUDE_MACROS, 405 clangArgs.toArray(new String[0])); 406 407 tuCursor.children() 408 .peek(c -> logger.finest( 409 () -> "Cursor: " + c.spelling() + "@" + c.USR() + "?" + c.isDeclaration())) 410 .filter(c -> c.isDeclaration() || c.isPreprocessing()) 411 .forEach(c -> processCursor(c, hf, fn)); 412 }); 413 } 414 415 private Map<String, List<AsmCodeFactory>> getPkgCfMap() { 416 final Map<String, List<AsmCodeFactory>> mapPkgCf = new HashMap<>(); 417 // Build the pkg to CodeFactory map 418 headerMap.values().forEach(header -> { 419 AsmCodeFactory cf = header.getCodeFactory(); 420 String pkg = header.pkgName; 421 logger.config(() -> "File " + header + " is in package: " + pkg); 422 if (cf == null) { 423 logger.config(() -> "File " + header + " code generation is not activated!"); 424 return; 425 } 426 List<AsmCodeFactory> l = mapPkgCf.computeIfAbsent(pkg, k -> new ArrayList<>()); 427 l.add(cf); 428 logger.config(() -> "Add cf " + cf + " to pkg " + pkg + ", size is now " + l.size()); 429 }); 430 return Collections.unmodifiableMap(mapPkgCf); 431 } 432 433 public Map<String, byte[]> collectClasses(String... pkgs) { 434 final Map<String, byte[]> rv = new HashMap<>(); 435 final Map<String, List<AsmCodeFactory>> mapPkgCf = getPkgCfMap(); 436 for (String pkg_name : pkgs) { 437 mapPkgCf.getOrDefault(pkg_name, Collections.emptyList()) 438 .forEach(cf -> rv.putAll(cf.collect())); 439 } 440 return Collections.unmodifiableMap(rv); 441 } 442 443 private static final String JEXTRACT_MANIFEST = "META-INFO" + File.separatorChar + "jextract.properties"; 444 445 @SuppressWarnings("deprecation") 446 private byte[] getJextractProperties() { 447 Properties props = new Properties(); 448 props.setProperty("os.name", System.getProperty("os.name")); 449 props.setProperty("os.version", System.getProperty("os.version")); 450 props.setProperty("os.arch", System.getProperty("os.arch")); 451 props.setProperty("jextract.args", Arrays.toString(args)); 452 ByteArrayOutputStream baos = new ByteArrayOutputStream(); 453 props.save(baos, "jextract meta data"); 454 return baos.toByteArray(); 455 } 456 457 void collectClassFiles(Path destDir, String... pkgs) throws IOException { 458 try { 459 collectClasses(pkgs).entrySet().stream().forEach(e -> { 460 try { 461 String path = e.getKey().replace('.', File.separatorChar) + ".class"; 462 logger.fine(() -> "Writing " + path); 463 Path fullPath = destDir.resolve(path).normalize(); 464 Files.createDirectories(fullPath.getParent()); 465 try (OutputStream fos = Files.newOutputStream(fullPath)) { 466 fos.write(e.getValue()); 467 fos.flush(); 468 } 469 } catch (IOException ioe) { 470 throw new UncheckedIOException(ioe); 471 } 472 }); 473 474 Path propsPath = destDir.resolve(JEXTRACT_MANIFEST).normalize(); 475 Files.createDirectories(propsPath.getParent()); 476 try (OutputStream fos = Files.newOutputStream(propsPath)) { 477 fos.write(getJextractProperties()); 478 fos.flush(); 479 } 480 } catch (UncheckedIOException uioe) { 481 throw uioe.getCause(); 482 } 483 } 484 485 private void writeJar(AsmCodeFactory cf, JarOutputStream jar) { 486 cf.collect().entrySet().stream().forEach(e -> { 487 try { 488 String path = e.getKey().replace('.', File.separatorChar) + ".class"; 489 logger.fine(() -> "Add " + path); 490 jar.putNextEntry(new ZipEntry(path)); 491 jar.write(e.getValue()); 492 jar.closeEntry(); 493 } catch (IOException ioe) { 494 throw new UncheckedIOException(ioe); 495 } 496 }); 497 } 498 499 public void collectJarFile(final JarOutputStream jos, String... pkgs) { 500 final Map<String, List<AsmCodeFactory>> mapPkgCf = getPkgCfMap(); 501 502 for (String pkg_name : pkgs) { 503 // convert '.' to '/' to use as a path 504 String entryName = Utils.toInternalName(pkg_name, ""); 505 // package folder 506 if (!entryName.isEmpty()) { 507 try { 508 jos.putNextEntry(new ZipEntry(entryName)); 509 } catch (IOException ex) { 510 throw new UncheckedIOException(ex); 511 } 512 } 513 logger.fine(() -> "Produce for package " + pkg_name); 514 mapPkgCf.getOrDefault(pkg_name, Collections.emptyList()) 515 .forEach(cf -> writeJar(cf, jos)); 516 } 517 } 518 519 void collectJarFile(final Path jar, String... pkgs) throws IOException { 520 logger.info(() -> "Collecting jar file " + jar); 521 try (OutputStream os = Files.newOutputStream(jar, CREATE, TRUNCATE_EXISTING, WRITE); 522 JarOutputStream jo = new JarOutputStream(os)) { 523 collectJarFile(jo, pkgs); 524 jo.putNextEntry(new ZipEntry(JEXTRACT_MANIFEST)); 525 jo.write(getJextractProperties()); 526 jo.closeEntry(); 527 } catch (UncheckedIOException uioe) { 528 throw uioe.getCause(); 529 } 530 } 531 532 /** 533 * Perform a local lookup, any undefined type will cause a JType 534 * be defined within origin scope. 535 * 536 * @param type The libclang type 537 * @param origin The path of the file where type is encountered 538 * @return The JType 539 */ 540 JType getJType(final Type type, Path origin) { 541 Path p = origin.normalize().toAbsolutePath(); 542 543 HeaderFile hf = headerMap.get(p); 544 // We should not encounter a type if the header file reference to it is not yet processed 545 assert(null != hf); 546 if (hf == null) { 547 throw new IllegalArgumentException("Failed to lookup header for " + p + " (origin: " + origin + ")"); 548 } 549 550 return hf.localLookup(type); 551 } 552 553 /** 554 * Perform a global lookup 555 * 556 * @param c The cursor define or declare the type. 557 * @return 558 */ 559 JType getJType(final Cursor c) { 560 if (c.isInvalid()) { 561 throw new IllegalArgumentException(); 562 } 563 SourceLocation loc = c.getSourceLocation(); 564 if (null == loc) { 565 return null; 566 } 567 Path p = loc.getFileLocation().path(); 568 if (null == p) { 569 return null; 570 } 571 return getJType(c.type(), p); 572 } 573 } --- EOF ---