1 /*
2 * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23 package com.sun.tools.jextract;
24
25 import jdk.internal.clang.*;
26
27 import java.io.ByteArrayOutputStream;
28 import java.io.File;
29 import java.io.IOException;
30 import java.io.OutputStream;
31 import java.io.PrintWriter;
32 import java.io.UncheckedIOException;
33 import java.lang.invoke.MethodHandles;
34 import java.lang.invoke.MethodHandles.Lookup;
35 import java.foreign.Library;
36 import java.foreign.Libraries;
37 import java.nio.file.Files;
38 import java.nio.file.Path;
39 import java.nio.file.Paths;
40 import java.util.ArrayList;
41 import java.util.Arrays;
42 import java.util.Collections;
43 import java.util.HashMap;
44 import java.util.List;
45 import java.util.Map;
46 import java.util.Optional;
47 import java.util.Properties;
48 import java.util.Set;
49 import java.util.TreeSet;
50 import java.util.function.Function;
51 import java.util.function.Predicate;
52 import java.util.jar.JarOutputStream;
53 import java.util.logging.Logger;
54 import java.util.regex.Pattern;
55 import java.util.stream.Collectors;
56 import java.util.zip.ZipEntry;
57 import com.sun.tools.jextract.parser.Parser;
58 import com.sun.tools.jextract.tree.FunctionTree;
59 import com.sun.tools.jextract.tree.HeaderTree;
60 import com.sun.tools.jextract.tree.Tree;
61
62 import static java.nio.file.StandardOpenOption.CREATE;
63 import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
64 import static java.nio.file.StandardOpenOption.WRITE;
65
66 /**
67 * The setup for the tool execution
68 */
69 public final class Context {
70 // package name to TypeDictionary
71 private final Map<String, TypeDictionary> tdMap;
72 // The folder path mapping to package name
73 private final Map<Path, String> pkgMap;
74 // The header file parsed
75 private final Map<Path, HeaderFile> headerMap;
76 // The args for parsing C
77 private final List<String> clangArgs;
78 // The set of source header files
79 private final Set<Path> sources;
80 // The list of library names
81 private final List<String> libraryNames;
82 // The list of library paths
83 private final List<String> libraryPaths;
84 // The list of library paths for link checks
85 private final List<String> linkCheckPaths;
86 // Symbol patterns to be excluded
87 private final List<Pattern> excludeSymbols;
88 // generate static forwarder class or not?
89 private boolean genStaticForwarder;
90
91 final PrintWriter out;
92 final PrintWriter err;
93
94 private Predicate<String> symChecker;
95 private Predicate<String> symFilter;
96
97 private final Parser parser;
98
99 private final static String defaultPkg = "jextract.dump";
100 final Logger logger = Logger.getLogger(getClass().getPackage().getName());
101
102 public Context(PrintWriter out, PrintWriter err) {
103 this.tdMap = new HashMap<>();
104 this.pkgMap = new HashMap<>();
105 this.headerMap = new HashMap<>();
106 this.clangArgs = new ArrayList<>();
107 this.sources = new TreeSet<>();
108 this.libraryNames = new ArrayList<>();
109 this.libraryPaths = new ArrayList<>();
110 this.linkCheckPaths = new ArrayList<>();
111 this.excludeSymbols = new ArrayList<>();
112 this.parser = new Parser(out, err, Main.INCLUDE_MACROS);
113 this.out = out;
114 this.err = err;
115 }
116
117 public Context() {
118 this(new PrintWriter(System.out, true), new PrintWriter(System.err, true));
119 }
120
121 TypeDictionary typeDictionaryFor(String pkg) {
122 return tdMap.computeIfAbsent(pkg, p->new TypeDictionary(this, p));
123 }
124
125 void addClangArg(String arg) {
126 clangArgs.add(arg);
127 }
128
129 public void addSource(Path path) {
130 sources.add(path);
131 }
132
133 void addLibraryName(String name) {
134 libraryNames.add(name);
135 }
136
137 void addLibraryPath(String path) {
138 libraryPaths.add(path);
139 }
140
141 void addLinkCheckPath(String path) {
142 linkCheckPaths.add(path);
143 }
144
145 void addExcludeSymbols(String pattern) {
146 excludeSymbols.add(Pattern.compile(pattern));
147 }
148
149 void setGenStaticForwarder(boolean flag) {
150 this.genStaticForwarder = flag;
151 }
152
153 // return the absolute path of the library of given name by searching
154 // in the given array of paths.
155 private static Optional<Path> findLibraryPath(Path[] paths, String libName) {
156 return Arrays.stream(paths).
157 map(p -> p.resolve(System.mapLibraryName(libName))).
158 filter(Files::isRegularFile).map(Path::toAbsolutePath).findFirst();
159 }
160
161 /*
162 * Load the specified shared libraries from the specified paths.
163 *
164 * @param lookup Lookup object of the caller.
165 * @param pathStrs array of paths to load the shared libraries from.
166 * @param names array of shared library names.
167 */
168 // used by jextract tool to load libraries for symbol checks.
169 public static Library[] loadLibraries(Lookup lookup, String[] pathStrs, String[] names) {
170 if (pathStrs == null || pathStrs.length == 0) {
171 return Arrays.stream(names).map(
172 name -> Libraries.loadLibrary(lookup, name)).toArray(Library[]::new);
173 } else {
174 Path[] paths = Arrays.stream(pathStrs).map(Paths::get).toArray(Path[]::new);
175 return Arrays.stream(names).map(libName -> {
176 Optional<Path> absPath = findLibraryPath(paths, libName);
177 return absPath.isPresent() ?
178 Libraries.load(lookup, absPath.get().toString()) :
179 Libraries.loadLibrary(lookup, libName);
180 }).toArray(Library[]::new);
181 }
182 }
183
184 private void initSymChecker() {
185 if (!libraryNames.isEmpty() && !linkCheckPaths.isEmpty()) {
186 try {
187 Library[] libs = loadLibraries(MethodHandles.lookup(),
188 linkCheckPaths.toArray(new String[0]),
189 libraryNames.toArray(new String[0]));
190 // check if the given symbol is found in any of the libraries or not.
191 // If not found, warn the user for the missing symbol.
192 symChecker = name -> {
193 if (Main.DEBUG) {
194 err.println("Searching symbol: " + name);
195 }
196 return (Arrays.stream(libs).filter(lib -> {
197 try {
198 lib.lookup(name);
199 if (Main.DEBUG) {
200 err.println("Found symbol: " + name);
201 }
202 return true;
203 } catch (NoSuchMethodException nsme) {
204 return false;
205 }
206 }).findFirst().isPresent());
207 };
208 } catch (UnsatisfiedLinkError ex) {
209 err.println(Main.format("warn.lib.not.found"));
210 symChecker = null;
211 }
212 } else {
213 symChecker = null;
214 }
215 }
216
217 private boolean isSymbolFound(String name) {
218 return symChecker == null? true : symChecker.test(name);
219 }
220
221 private void initSymFilter() {
222 if (!excludeSymbols.isEmpty()) {
223 Pattern[] pats = excludeSymbols.toArray(new Pattern[0]);
224 symFilter = name -> {
225 return Arrays.stream(pats).filter(pat -> pat.matcher(name).matches()).
226 findFirst().isPresent();
227 };
228 } else {
229 symFilter = null;
230 }
231 }
232
233 private boolean isSymbolExcluded(String name) {
234 return symFilter == null? false : symFilter.test(name);
235 }
236
237 /**
238 * Setup a package name for a given folder.
239 *
240 * @param folder The path to the folder, use null to set catch-all.
241 * @param pkg The package name
242 * @return True if the folder is setup successfully. False is a package
243 * has been assigned for the folder.
244 */
245 public boolean usePackageForFolder(Path folder, String pkg) {
246 if (folder != null) {
247 folder = folder.toAbsolutePath();
248 if (!Files.isDirectory(folder)) {
249 folder = folder.getParent();
250 }
251 }
252 String existing = pkgMap.putIfAbsent(folder, pkg);
253 final String finalFolder = (null == folder) ? "all folders not configured" : folder.toString();
254 if (null == existing) {
255 logger.config(() -> "Package " + pkg + " is selected for " + finalFolder);
256 return true;
257 } else {
258 logger.warning(() -> "Package " + existing + " had been selected for " + finalFolder + ", request to use " + pkg + " is ignored.");
259 return false;
260 }
261 }
262
263 static class Entity {
264 final String pkg;
265 final String entity;
266
267 Entity(String pkg, String entity) {
268 this.pkg = pkg;
269 this.entity = entity;
270 }
271 }
272
273 /**
274 * Determine package and interface name given a path. If the path is
275 * a folder, then only package name is determined. The package name is
276 * determined with the longest path matching the setup. If the path is not
277 * setup for any package, the default package name is returned.
278 *
279 * @param origin The source path
280 * @return The Entity
281 * @see Context::usePackageForFolder(Path, String)
282 */
283 Entity whatis(Path origin) {
284 // normalize to absolute path
285 origin = origin.toAbsolutePath();
286 String filename = null;
287 if (!Files.isDirectory(origin)) {
288 // ensure it's a folder name
289 filename = origin.getFileName().toString();
290 origin = origin.getParent();
291 }
292 Path path = origin;
293
294 // search the map for a hit with longest path
295 while (path != null && !pkgMap.containsKey(path)) {
296 path = path.getParent();
297 }
298
299 int start;
300 String pkg;
301 if (path != null) {
302 start = path.getNameCount();
303 pkg = pkgMap.get(path);
304 } else {
305 pkg = pkgMap.get(null);
306 if (pkg == null) {
307 start = 0;
308 pkg = defaultPkg;
309 } else {
310 start = origin.getNameCount();
311 }
312 }
313
314 if (filename == null) {
315 // a folder, only pkg name matters
316 return new Entity(pkg, null);
317 }
318
319 StringBuilder sb = new StringBuilder();
320 while (start < origin.getNameCount()) {
321 sb.append(Utils.toJavaIdentifier(origin.getName(start++).toString()));
322 sb.append("_");
323 }
324
325 int ext = filename.lastIndexOf('.');
326 if (ext != -1) {
327 sb.append(filename.substring(0, ext));
328 } else {
329 sb.append(filename);
330 }
331 return new Entity(pkg, Utils.toClassName(sb.toString()));
332 }
333
334 HeaderFile getHeaderFile(Path header, HeaderFile main) {
335 if (!Files.isRegularFile(header)) {
336 logger.warning(() -> "Not a regular file: " + header.toString());
337 throw new IllegalArgumentException(header.toString());
338 }
339
340 final Context.Entity e = whatis(header);
341 HeaderFile headerFile = new HeaderFile(this, header, e.pkg, e.entity, main);
342 headerFile.useLibraries(libraryNames, libraryPaths);
343 return headerFile;
344 }
345
346 void processTree(Tree tree, HeaderFile main, Function<HeaderFile, AsmCodeFactory> fn) {
347 SourceLocation loc = tree.location();
348
349 HeaderFile header;
350 boolean isBuiltIn = false;
351
352 if (tree.isFromMain()) {
353 header = main;
354 } else {
355 SourceLocation.Location src = loc.getFileLocation();
356 if (src == null) {
357 logger.info(() -> "Tree " + tree.name() + "@" + tree.USR() + " has no FileLocation");
358 return;
359 }
360
361 Path p = src.path();
362 if (p == null) {
363 logger.fine(() -> "Found built-in type: " + tree.name());
364 header = main;
365 isBuiltIn = true;
366 } else {
367 p = p.normalize().toAbsolutePath();
368 header = headerMap.get(p);
369 if (header == null) {
370 final HeaderFile hf = header = getHeaderFile(p, main);
371 logger.config(() -> "First encounter of header file " + hf.path + ", assigned to package " + hf.pkgName);
372 // Only generate code for header files specified or in the same package
373 if (sources.contains(p) ||
374 (header.pkgName.equals(main.pkgName))) {
375 logger.config("Code gen for header " + p + " enabled in package " + header.pkgName);
376 header.useCodeFactory(fn.apply(header));
377 }
378 headerMap.put(p, header);
379 }
380 }
381 }
382
383 header.processTree(tree, main, isBuiltIn);
384 }
385
386 public void parse() {
387 parse(header -> genStaticForwarder?
388 new AsmCodeFactoryExt(this, header) : new AsmCodeFactory(this, header));
389 }
390
391 private boolean symbolFilter(Tree tree) {
392 String name = tree.name();
393 if (isSymbolExcluded(name)) {
394 return false;
395 }
396
397 // check for function symbols in libraries & warn missing symbols
398 if (tree instanceof FunctionTree && !isSymbolFound(name)) {
399 err.println(Main.format("warn.symbol.not.found", name));
400 //auto-exclude symbols not found
401 return false;
402 }
403
404 return true;
405 }
406
407 public void parse(Function<HeaderFile, AsmCodeFactory> fn) {
408 initSymChecker();
409 initSymFilter();
410
411 List<HeaderTree> headers = parser.parse(sources, clangArgs);
412 processHeaders(headers, fn);
413 }
414
415 private void processHeaders(List<HeaderTree> headers, Function<HeaderFile, AsmCodeFactory> fn) {
416 headers.stream().
417 map(new TreeFilter(this::symbolFilter)).
418 map(new TypedefHandler()).
419 map(new EmptyNameHandler()).
420 forEach(header -> {
421 HeaderFile hf = headerMap.computeIfAbsent(header.path(), p -> getHeaderFile(p, null));
422 hf.useCodeFactory(fn.apply(hf));
423 logger.info(() -> "Processing header file " + header.path());
424
425 header.declarations().stream()
426 .peek(decl -> logger.finest(
427 () -> "Cursor: " + decl.name() + "@" + decl.USR() + "?" + decl.isDeclaration()))
428 .forEach(decl -> processTree(decl, hf, fn));
429 });
430 }
431
432 private Map<String, List<AsmCodeFactory>> getPkgCfMap() {
433 final Map<String, List<AsmCodeFactory>> mapPkgCf = new HashMap<>();
434 // Build the pkg to CodeFactory map
435 headerMap.values().forEach(header -> {
436 AsmCodeFactory cf = header.getCodeFactory();
437 String pkg = header.pkgName;
438 logger.config(() -> "File " + header + " is in package: " + pkg);
439 if (cf == null) {
440 logger.config(() -> "File " + header + " code generation is not activated!");
441 return;
442 }
443 List<AsmCodeFactory> l = mapPkgCf.computeIfAbsent(pkg, k -> new ArrayList<>());
444 l.add(cf);
445 logger.config(() -> "Add cf " + cf + " to pkg " + pkg + ", size is now " + l.size());
446 });
447 return Collections.unmodifiableMap(mapPkgCf);
448 }
449
450 public Map<String, byte[]> collectClasses(String... pkgs) {
451 final Map<String, byte[]> rv = new HashMap<>();
452 final Map<String, List<AsmCodeFactory>> mapPkgCf = getPkgCfMap();
453 for (String pkg_name : pkgs) {
454 mapPkgCf.getOrDefault(pkg_name, Collections.emptyList())
455 .forEach(cf -> rv.putAll(cf.collect()));
456 }
457 return Collections.unmodifiableMap(rv);
458 }
459
460 private static final String JEXTRACT_MANIFEST = "META-INFO" + File.separatorChar + "jextract.properties";
461
462 @SuppressWarnings("deprecation")
463 private byte[] getJextractProperties(String[] args) {
464 Properties props = new Properties();
465 props.setProperty("os.name", System.getProperty("os.name"));
466 props.setProperty("os.version", System.getProperty("os.version"));
467 props.setProperty("os.arch", System.getProperty("os.arch"));
468 props.setProperty("jextract.args", Arrays.toString(args));
469 ByteArrayOutputStream baos = new ByteArrayOutputStream();
470 props.save(baos, "jextract meta data");
471 return baos.toByteArray();
472 }
473
474 void collectClassFiles(Path destDir, String[] args, String... pkgs) throws IOException {
475 try {
476 collectClasses(pkgs).entrySet().stream().forEach(e -> {
477 try {
478 String path = e.getKey().replace('.', File.separatorChar) + ".class";
479 logger.fine(() -> "Writing " + path);
480 Path fullPath = destDir.resolve(path).normalize();
481 Files.createDirectories(fullPath.getParent());
482 try (OutputStream fos = Files.newOutputStream(fullPath)) {
483 fos.write(e.getValue());
484 fos.flush();
485 }
486 } catch (IOException ioe) {
487 throw new UncheckedIOException(ioe);
488 }
489 });
490
491 Path propsPath = destDir.resolve(JEXTRACT_MANIFEST).normalize();
492 Files.createDirectories(propsPath.getParent());
493 try (OutputStream fos = Files.newOutputStream(propsPath)) {
494 fos.write(getJextractProperties(args));
495 fos.flush();
496 }
497 } catch (UncheckedIOException uioe) {
498 throw uioe.getCause();
499 }
500 }
501
502 private void writeJar(AsmCodeFactory cf, JarOutputStream jar) {
503 cf.collect().entrySet().stream().forEach(e -> {
504 try {
505 String path = e.getKey().replace('.', File.separatorChar) + ".class";
506 logger.fine(() -> "Add " + path);
507 jar.putNextEntry(new ZipEntry(path));
508 jar.write(e.getValue());
509 jar.closeEntry();
510 } catch (IOException ioe) {
511 throw new UncheckedIOException(ioe);
512 }
513 });
514 }
515
516 public void collectJarFile(final JarOutputStream jos, String[] args, String... pkgs) {
517 final Map<String, List<AsmCodeFactory>> mapPkgCf = getPkgCfMap();
518
519 for (String pkg_name : pkgs) {
520 // convert '.' to '/' to use as a path
521 String entryName = Utils.toInternalName(pkg_name, "");
522 // package folder
523 if (!entryName.isEmpty()) {
524 try {
525 jos.putNextEntry(new ZipEntry(entryName));
526 } catch (IOException ex) {
527 throw new UncheckedIOException(ex);
528 }
529 }
530 logger.fine(() -> "Produce for package " + pkg_name);
531 mapPkgCf.getOrDefault(pkg_name, Collections.emptyList())
532 .forEach(cf -> writeJar(cf, jos));
533 }
534
535 try {
536 jos.putNextEntry(new ZipEntry(JEXTRACT_MANIFEST));
537 jos.write(getJextractProperties(args));
538 jos.closeEntry();
539 } catch (IOException ioe) {
540 throw new UncheckedIOException(ioe);
541 }
542 }
543
544 void collectJarFile(final Path jar, String[] args, String... pkgs) throws IOException {
545 logger.info(() -> "Collecting jar file " + jar);
546 try (OutputStream os = Files.newOutputStream(jar, CREATE, TRUNCATE_EXISTING, WRITE);
547 JarOutputStream jo = new JarOutputStream(os)) {
548 collectJarFile(jo, args, pkgs);
549 } catch (UncheckedIOException uioe) {
550 throw uioe.getCause();
551 }
552 }
553
554 /**
555 * Perform a local lookup, any undefined type will cause a JType
556 * be defined within origin scope.
557 *
558 * @param type The libclang type
559 * @param origin The path of the file where type is encountered
560 * @return The JType
561 */
562 JType getJType(final Type type, Path origin) {
563 Path p = origin.normalize().toAbsolutePath();
564
565 HeaderFile hf = headerMap.get(p);
566 // We should not encounter a type if the header file reference to it is not yet processed
567 assert(null != hf);
568 if (hf == null) {
569 throw new IllegalArgumentException("Failed to lookup header for " + p + " (origin: " + origin + ")");
570 }
571
572 return hf.localLookup(type);
573 }
574
575 /**
576 * Perform a global lookup
577 *
578 * @param c The cursor define or declare the type.
579 * @return
580 */
581 JType getJType(final Cursor c) {
582 if (c.isInvalid()) {
583 throw new IllegalArgumentException();
584 }
585 SourceLocation loc = c.getSourceLocation();
586 if (null == loc) {
587 return null;
588 }
589 Path p = loc.getFileLocation().path();
590 if (null == p) {
591 return null;
592 }
593 return getJType(c.type(), p);
594 }
595 }