1 /*
2 * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23 package com.sun.tools.jextract;
24
25 import jdk.internal.clang.*;
26
27 import java.io.ByteArrayOutputStream;
28 import java.io.File;
29 import java.io.IOException;
30 import java.io.OutputStream;
31 import java.io.PrintWriter;
32 import java.io.UncheckedIOException;
33 import java.lang.invoke.MethodHandles;
34 import java.lang.invoke.MethodHandles.Lookup;
35 import java.foreign.Library;
36 import java.foreign.Libraries;
37 import java.nio.file.Files;
38 import java.nio.file.Path;
39 import java.nio.file.Paths;
40 import java.util.ArrayList;
41 import java.util.Arrays;
42 import java.util.Collections;
43 import java.util.HashMap;
44 import java.util.List;
45 import java.util.Map;
46 import java.util.Optional;
47 import java.util.Properties;
48 import java.util.Set;
49 import java.util.TreeSet;
50 import java.util.function.Function;
51 import java.util.function.Predicate;
52 import java.util.jar.JarOutputStream;
53 import java.util.logging.Logger;
54 import java.util.regex.Pattern;
55 import java.util.stream.Collectors;
56 import java.util.zip.ZipEntry;
57 import com.sun.tools.jextract.parser.Parser;
58 import com.sun.tools.jextract.tree.FunctionTree;
59 import com.sun.tools.jextract.tree.HeaderTree;
60 import com.sun.tools.jextract.tree.Tree;
61
62 import static java.nio.file.StandardOpenOption.CREATE;
63 import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
64 import static java.nio.file.StandardOpenOption.WRITE;
65
66 /**
67 * The setup for the tool execution
68 */
69 public final class Context {
70 // package name to TypeDictionary
71 private final Map<String, TypeDictionary> tdMap;
72 // The folder path mapping to package name
73 private final Map<Path, String> pkgMap;
74 // The header file parsed
75 private final Map<Path, HeaderFile> headerMap;
76 // The args for parsing C
77 private final List<String> clangArgs;
78 // The set of source header files
79 private final Set<Path> sources;
80 // The list of library names
81 private final List<String> libraryNames;
82 // The list of library paths
83 private final List<String> libraryPaths;
84 // The list of library paths for link checks
85 private final List<String> linkCheckPaths;
86 // Symbol patterns to be included
87 private final List<Pattern> includeSymbols;
88 // Symbol patterns to be excluded
89 private final List<Pattern> excludeSymbols;
90 // generate static forwarder class or not?
91 private boolean genStaticForwarder;
92
93 final PrintWriter out;
94 final PrintWriter err;
95
96 private Predicate<String> symChecker;
97 private Predicate<String> includeSymFilter;
98 private Predicate<String> excludeSymFilter;
99
100 private final Parser parser;
101
102 private final static String defaultPkg = "jextract.dump";
103 final Logger logger = Logger.getLogger(getClass().getPackage().getName());
104
105 public Context(PrintWriter out, PrintWriter err) {
106 this.tdMap = new HashMap<>();
107 this.pkgMap = new HashMap<>();
108 this.headerMap = new HashMap<>();
109 this.clangArgs = new ArrayList<>();
110 this.sources = new TreeSet<>();
111 this.libraryNames = new ArrayList<>();
112 this.libraryPaths = new ArrayList<>();
113 this.linkCheckPaths = new ArrayList<>();
114 this.includeSymbols = new ArrayList<>();
115 this.excludeSymbols = new ArrayList<>();
116 this.parser = new Parser(out, err, Main.INCLUDE_MACROS);
117 this.out = out;
118 this.err = err;
119 }
120
121 public Context() {
122 this(new PrintWriter(System.out, true), new PrintWriter(System.err, true));
123 }
124
125 TypeDictionary typeDictionaryFor(String pkg) {
126 return tdMap.computeIfAbsent(pkg, p->new TypeDictionary(this, p));
127 }
128
129 void addClangArg(String arg) {
130 clangArgs.add(arg);
131 }
132
133 public void addSource(Path path) {
134 sources.add(path);
135 }
136
137 void addLibraryName(String name) {
138 libraryNames.add(name);
139 }
140
141 void addLibraryPath(String path) {
142 libraryPaths.add(path);
143 }
144
145 void addLinkCheckPath(String path) {
146 linkCheckPaths.add(path);
147 }
148
149 void addIncludeSymbols(String pattern) {
150 includeSymbols.add(Pattern.compile(pattern));
151 }
152
153 void addExcludeSymbols(String pattern) {
154 excludeSymbols.add(Pattern.compile(pattern));
155 }
156
157 void setGenStaticForwarder(boolean flag) {
158 this.genStaticForwarder = flag;
159 }
160
161 // return the absolute path of the library of given name by searching
162 // in the given array of paths.
163 private static Optional<Path> findLibraryPath(Path[] paths, String libName) {
164 return Arrays.stream(paths).
165 map(p -> p.resolve(System.mapLibraryName(libName))).
166 filter(Files::isRegularFile).map(Path::toAbsolutePath).findFirst();
167 }
168
169 /*
170 * Load the specified shared libraries from the specified paths.
171 *
172 * @param lookup Lookup object of the caller.
173 * @param pathStrs array of paths to load the shared libraries from.
174 * @param names array of shared library names.
175 */
176 // used by jextract tool to load libraries for symbol checks.
177 public static Library[] loadLibraries(Lookup lookup, String[] pathStrs, String[] names) {
178 if (pathStrs == null || pathStrs.length == 0) {
179 return Arrays.stream(names).map(
180 name -> Libraries.loadLibrary(lookup, name)).toArray(Library[]::new);
181 } else {
182 Path[] paths = Arrays.stream(pathStrs).map(Paths::get).toArray(Path[]::new);
183 return Arrays.stream(names).map(libName -> {
184 Optional<Path> absPath = findLibraryPath(paths, libName);
185 return absPath.isPresent() ?
186 Libraries.load(lookup, absPath.get().toString()) :
187 Libraries.loadLibrary(lookup, libName);
188 }).toArray(Library[]::new);
189 }
190 }
191
192 private void initSymChecker() {
193 if (!libraryNames.isEmpty() && !linkCheckPaths.isEmpty()) {
194 try {
195 Library[] libs = loadLibraries(MethodHandles.lookup(),
196 linkCheckPaths.toArray(new String[0]),
197 libraryNames.toArray(new String[0]));
198 // check if the given symbol is found in any of the libraries or not.
199 // If not found, warn the user for the missing symbol.
200 symChecker = name -> {
201 if (Main.DEBUG) {
202 err.println("Searching symbol: " + name);
203 }
204 return (Arrays.stream(libs).filter(lib -> {
205 try {
206 lib.lookup(name);
207 if (Main.DEBUG) {
208 err.println("Found symbol: " + name);
209 }
210 return true;
211 } catch (NoSuchMethodException nsme) {
212 return false;
213 }
214 }).findFirst().isPresent());
215 };
216 } catch (UnsatisfiedLinkError ex) {
217 err.println(Main.format("warn.lib.not.found"));
218 symChecker = null;
219 }
220 } else {
221 symChecker = null;
222 }
223 }
224
225 private boolean isSymbolFound(String name) {
226 return symChecker == null? true : symChecker.test(name);
227 }
228
229 private void initSymFilters() {
230 if (!includeSymbols.isEmpty()) {
231 Pattern[] pats = includeSymbols.toArray(new Pattern[0]);
232 includeSymFilter = name -> {
233 return Arrays.stream(pats).filter(pat -> pat.matcher(name).matches()).
234 findFirst().isPresent();
235 };
236 } else {
237 includeSymFilter = null;
238 }
239
240 if (!excludeSymbols.isEmpty()) {
241 Pattern[] pats = excludeSymbols.toArray(new Pattern[0]);
242 excludeSymFilter = name -> {
243 return Arrays.stream(pats).filter(pat -> pat.matcher(name).matches()).
244 findFirst().isPresent();
245 };
246 } else {
247 excludeSymFilter = null;
248 }
249 }
250
251 private boolean isSymbolIncluded(String name) {
252 return includeSymFilter == null? true : includeSymFilter.test(name);
253 }
254
255 private boolean isSymbolExcluded(String name) {
256 return excludeSymFilter == null? false : excludeSymFilter.test(name);
257 }
258
259 /**
260 * Setup a package name for a given folder.
261 *
262 * @param folder The path to the folder, use null to set catch-all.
263 * @param pkg The package name
264 * @return True if the folder is setup successfully. False is a package
265 * has been assigned for the folder.
266 */
267 public boolean usePackageForFolder(Path folder, String pkg) {
268 if (folder != null) {
269 folder = folder.toAbsolutePath();
270 if (!Files.isDirectory(folder)) {
271 folder = folder.getParent();
272 }
273 }
274 String existing = pkgMap.putIfAbsent(folder, pkg);
275 final String finalFolder = (null == folder) ? "all folders not configured" : folder.toString();
276 if (null == existing) {
277 logger.config(() -> "Package " + pkg + " is selected for " + finalFolder);
278 return true;
279 } else {
280 logger.warning(() -> "Package " + existing + " had been selected for " + finalFolder + ", request to use " + pkg + " is ignored.");
281 return false;
282 }
283 }
284
285 static class Entity {
286 final String pkg;
287 final String entity;
288
289 Entity(String pkg, String entity) {
290 this.pkg = pkg;
291 this.entity = entity;
292 }
293 }
294
295 /**
296 * Determine package and interface name given a path. If the path is
297 * a folder, then only package name is determined. The package name is
298 * determined with the longest path matching the setup. If the path is not
299 * setup for any package, the default package name is returned.
300 *
301 * @param origin The source path
302 * @return The Entity
303 * @see Context::usePackageForFolder(Path, String)
304 */
305 Entity whatis(Path origin) {
306 // normalize to absolute path
307 origin = origin.toAbsolutePath();
308 String filename = null;
309 if (!Files.isDirectory(origin)) {
310 // ensure it's a folder name
311 filename = origin.getFileName().toString();
312 origin = origin.getParent();
313 }
314 Path path = origin;
315
316 // search the map for a hit with longest path
317 while (path != null && !pkgMap.containsKey(path)) {
318 path = path.getParent();
319 }
320
321 int start;
322 String pkg;
323 if (path != null) {
324 start = path.getNameCount();
325 pkg = pkgMap.get(path);
326 } else {
327 pkg = pkgMap.get(null);
328 if (pkg == null) {
329 start = 0;
330 pkg = defaultPkg;
331 } else {
332 start = origin.getNameCount();
333 }
334 }
335
336 if (filename == null) {
337 // a folder, only pkg name matters
338 return new Entity(pkg, null);
339 }
340
341 StringBuilder sb = new StringBuilder();
342 while (start < origin.getNameCount()) {
343 sb.append(Utils.toJavaIdentifier(origin.getName(start++).toString()));
344 sb.append("_");
345 }
346
347 int ext = filename.lastIndexOf('.');
348 if (ext != -1) {
349 sb.append(filename.substring(0, ext));
350 } else {
351 sb.append(filename);
352 }
353 return new Entity(pkg, Utils.toClassName(sb.toString()));
354 }
355
356 HeaderFile getHeaderFile(Path header, HeaderFile main) {
357 if (!Files.isRegularFile(header)) {
358 logger.warning(() -> "Not a regular file: " + header.toString());
359 throw new IllegalArgumentException(header.toString());
360 }
361
362 final Context.Entity e = whatis(header);
363 HeaderFile headerFile = new HeaderFile(this, header, e.pkg, e.entity, main);
364 headerFile.useLibraries(libraryNames, libraryPaths);
365 return headerFile;
366 }
367
368 void processTree(Tree tree, HeaderFile main, Function<HeaderFile, AsmCodeFactory> fn) {
369 SourceLocation loc = tree.location();
370
371 HeaderFile header;
372 boolean isBuiltIn = false;
373
374 if (tree.isFromMain()) {
375 header = main;
376 } else {
377 SourceLocation.Location src = loc.getFileLocation();
378 if (src == null) {
379 logger.info(() -> "Tree " + tree.name() + "@" + tree.USR() + " has no FileLocation");
380 return;
381 }
382
383 Path p = src.path();
384 if (p == null) {
385 logger.fine(() -> "Found built-in type: " + tree.name());
386 header = main;
387 isBuiltIn = true;
388 } else {
389 p = p.normalize().toAbsolutePath();
390 header = headerMap.get(p);
391 if (header == null) {
392 final HeaderFile hf = header = getHeaderFile(p, main);
393 logger.config(() -> "First encounter of header file " + hf.path + ", assigned to package " + hf.pkgName);
394 // Only generate code for header files specified or in the same package
395 if (sources.contains(p) ||
396 (header.pkgName.equals(main.pkgName))) {
397 logger.config("Code gen for header " + p + " enabled in package " + header.pkgName);
398 header.useCodeFactory(fn.apply(header));
399 }
400 headerMap.put(p, header);
401 }
402 }
403 }
404
405 header.processTree(tree, main, isBuiltIn);
406 }
407
408 public void parse() {
409 parse(header -> genStaticForwarder?
410 new AsmCodeFactoryExt(this, header) : new AsmCodeFactory(this, header));
411 }
412
413 private boolean symbolFilter(Tree tree) {
414 String name = tree.name();
415 if (!isSymbolIncluded(name) || isSymbolExcluded(name)) {
416 return false;
417 }
418
419 // check for function symbols in libraries & warn missing symbols
420 if (tree instanceof FunctionTree && !isSymbolFound(name)) {
421 err.println(Main.format("warn.symbol.not.found", name));
422 //auto-exclude symbols not found
423 return false;
424 }
425
426 return true;
427 }
428
429 public void parse(Function<HeaderFile, AsmCodeFactory> fn) {
430 initSymChecker();
431 initSymFilters();
432
433 List<HeaderTree> headers = parser.parse(sources, clangArgs);
434 processHeaders(headers, fn);
435 }
436
437 private void processHeaders(List<HeaderTree> headers, Function<HeaderFile, AsmCodeFactory> fn) {
438 headers.stream().
439 map(new TreeFilter(this::symbolFilter)).
440 map(new TypedefHandler()).
441 map(new EmptyNameHandler()).
442 forEach(header -> {
443 HeaderFile hf = headerMap.computeIfAbsent(header.path(), p -> getHeaderFile(p, null));
444 hf.useCodeFactory(fn.apply(hf));
445 logger.info(() -> "Processing header file " + header.path());
446
447 header.declarations().stream()
448 .peek(decl -> logger.finest(
449 () -> "Cursor: " + decl.name() + "@" + decl.USR() + "?" + decl.isDeclaration()))
450 .forEach(decl -> processTree(decl, hf, fn));
451 });
452 }
453
454 private Map<String, List<AsmCodeFactory>> getPkgCfMap() {
455 final Map<String, List<AsmCodeFactory>> mapPkgCf = new HashMap<>();
456 // Build the pkg to CodeFactory map
457 headerMap.values().forEach(header -> {
458 AsmCodeFactory cf = header.getCodeFactory();
459 String pkg = header.pkgName;
460 logger.config(() -> "File " + header + " is in package: " + pkg);
461 if (cf == null) {
462 logger.config(() -> "File " + header + " code generation is not activated!");
463 return;
464 }
465 List<AsmCodeFactory> l = mapPkgCf.computeIfAbsent(pkg, k -> new ArrayList<>());
466 l.add(cf);
467 logger.config(() -> "Add cf " + cf + " to pkg " + pkg + ", size is now " + l.size());
468 });
469 return Collections.unmodifiableMap(mapPkgCf);
470 }
471
472 public Map<String, byte[]> collectClasses(String... pkgs) {
473 final Map<String, byte[]> rv = new HashMap<>();
474 final Map<String, List<AsmCodeFactory>> mapPkgCf = getPkgCfMap();
475 for (String pkg_name : pkgs) {
476 mapPkgCf.getOrDefault(pkg_name, Collections.emptyList())
477 .forEach(cf -> rv.putAll(cf.collect()));
478 }
479 return Collections.unmodifiableMap(rv);
480 }
481
482 private static final String JEXTRACT_MANIFEST = "META-INFO" + File.separatorChar + "jextract.properties";
483
484 @SuppressWarnings("deprecation")
485 private byte[] getJextractProperties(String[] args) {
486 Properties props = new Properties();
487 props.setProperty("os.name", System.getProperty("os.name"));
488 props.setProperty("os.version", System.getProperty("os.version"));
489 props.setProperty("os.arch", System.getProperty("os.arch"));
490 props.setProperty("jextract.args", Arrays.toString(args));
491 ByteArrayOutputStream baos = new ByteArrayOutputStream();
492 props.save(baos, "jextract meta data");
493 return baos.toByteArray();
494 }
495
496 void collectClassFiles(Path destDir, String[] args, String... pkgs) throws IOException {
497 try {
498 collectClasses(pkgs).entrySet().stream().forEach(e -> {
499 try {
500 String path = e.getKey().replace('.', File.separatorChar) + ".class";
501 logger.fine(() -> "Writing " + path);
502 Path fullPath = destDir.resolve(path).normalize();
503 Files.createDirectories(fullPath.getParent());
504 try (OutputStream fos = Files.newOutputStream(fullPath)) {
505 fos.write(e.getValue());
506 fos.flush();
507 }
508 } catch (IOException ioe) {
509 throw new UncheckedIOException(ioe);
510 }
511 });
512
513 Path propsPath = destDir.resolve(JEXTRACT_MANIFEST).normalize();
514 Files.createDirectories(propsPath.getParent());
515 try (OutputStream fos = Files.newOutputStream(propsPath)) {
516 fos.write(getJextractProperties(args));
517 fos.flush();
518 }
519 } catch (UncheckedIOException uioe) {
520 throw uioe.getCause();
521 }
522 }
523
524 private void writeJar(AsmCodeFactory cf, JarOutputStream jar) {
525 cf.collect().entrySet().stream().forEach(e -> {
526 try {
527 String path = e.getKey().replace('.', File.separatorChar) + ".class";
528 logger.fine(() -> "Add " + path);
529 jar.putNextEntry(new ZipEntry(path));
530 jar.write(e.getValue());
531 jar.closeEntry();
532 } catch (IOException ioe) {
533 throw new UncheckedIOException(ioe);
534 }
535 });
536 }
537
538 public void collectJarFile(final JarOutputStream jos, String[] args, String... pkgs) {
539 final Map<String, List<AsmCodeFactory>> mapPkgCf = getPkgCfMap();
540
541 for (String pkg_name : pkgs) {
542 // convert '.' to '/' to use as a path
543 String entryName = Utils.toInternalName(pkg_name, "");
544 // package folder
545 if (!entryName.isEmpty()) {
546 try {
547 jos.putNextEntry(new ZipEntry(entryName));
548 } catch (IOException ex) {
549 throw new UncheckedIOException(ex);
550 }
551 }
552 logger.fine(() -> "Produce for package " + pkg_name);
553 mapPkgCf.getOrDefault(pkg_name, Collections.emptyList())
554 .forEach(cf -> writeJar(cf, jos));
555 }
556
557 try {
558 jos.putNextEntry(new ZipEntry(JEXTRACT_MANIFEST));
559 jos.write(getJextractProperties(args));
560 jos.closeEntry();
561 } catch (IOException ioe) {
562 throw new UncheckedIOException(ioe);
563 }
564 }
565
566 void collectJarFile(final Path jar, String[] args, String... pkgs) throws IOException {
567 logger.info(() -> "Collecting jar file " + jar);
568 try (OutputStream os = Files.newOutputStream(jar, CREATE, TRUNCATE_EXISTING, WRITE);
569 JarOutputStream jo = new JarOutputStream(os)) {
570 collectJarFile(jo, args, pkgs);
571 } catch (UncheckedIOException uioe) {
572 throw uioe.getCause();
573 }
574 }
575
576 /**
577 * Perform a local lookup, any undefined type will cause a JType
578 * be defined within origin scope.
579 *
580 * @param type The libclang type
581 * @param origin The path of the file where type is encountered
582 * @return The JType
583 */
584 JType getJType(final Type type, Path origin) {
585 Path p = origin.normalize().toAbsolutePath();
586
587 HeaderFile hf = headerMap.get(p);
588 // We should not encounter a type if the header file reference to it is not yet processed
589 assert(null != hf);
590 if (hf == null) {
591 throw new IllegalArgumentException("Failed to lookup header for " + p + " (origin: " + origin + ")");
592 }
593
594 return hf.localLookup(type);
595 }
596
597 /**
598 * Perform a global lookup
599 *
600 * @param c The cursor define or declare the type.
601 * @return
602 */
603 JType getJType(final Cursor c) {
604 if (c.isInvalid()) {
605 throw new IllegalArgumentException();
606 }
607 SourceLocation loc = c.getSourceLocation();
608 if (null == loc) {
609 return null;
610 }
611 Path p = loc.getFileLocation().path();
612 if (null == p) {
613 return null;
614 }
615 return getJType(c.type(), p);
616 }
617 }