1 /*
2 * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 */
23 package com.sun.tools.jextract;
24
25 import jdk.internal.clang.*;
26 import jdk.internal.foreign.LibrariesHelper;
27
28 import java.io.ByteArrayOutputStream;
29 import java.io.File;
30 import java.io.IOException;
31 import java.io.OutputStream;
32 import java.io.PrintWriter;
33 import java.io.UncheckedIOException;
34 import java.lang.invoke.MethodHandles;
35 import java.foreign.Library;
36 import java.nio.file.Files;
37 import java.nio.file.Path;
38 import java.util.ArrayList;
39 import java.util.Arrays;
40 import java.util.Collections;
41 import java.util.HashMap;
42 import java.util.List;
43 import java.util.Map;
44 import java.util.Properties;
45 import java.util.Set;
46 import java.util.TreeSet;
47 import java.util.function.Function;
48 import java.util.function.Predicate;
49 import java.util.jar.JarOutputStream;
50 import java.util.logging.Logger;
51 import java.util.regex.Pattern;
52 import java.util.stream.Collectors;
53 import java.util.zip.ZipEntry;
54 import com.sun.tools.jextract.parser.Parser;
55 import com.sun.tools.jextract.tree.FunctionTree;
56 import com.sun.tools.jextract.tree.HeaderTree;
57 import com.sun.tools.jextract.tree.Tree;
58
59 import static java.nio.file.StandardOpenOption.CREATE;
60 import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
61 import static java.nio.file.StandardOpenOption.WRITE;
62
63 /**
64 * The setup for the tool execution
65 */
66 public final class Context {
67 // package name to TypeDictionary
68 private final Map<String, TypeDictionary> tdMap;
69 // The folder path mapping to package name
70 private final Map<Path, String> pkgMap;
71 // The header file parsed
72 private final Map<Path, HeaderFile> headerMap;
73 // The args for parsing C
74 private final List<String> clangArgs;
75 // The set of source header files
76 private final Set<Path> sources;
77 // The list of library names
78 private final List<String> libraryNames;
79 // The list of library paths
80 private final List<String> libraryPaths;
81 // The list of library paths for link checks
82 private final List<String> linkCheckPaths;
83 // Symbol patterns to be excluded
84 private final List<Pattern> excludeSymbols;
85
86 final PrintWriter out;
87 final PrintWriter err;
88
89 private Predicate<String> symChecker;
90 private Predicate<String> symFilter;
91
92 private final Parser parser;
93
94 private final static String defaultPkg = "jextract.dump";
95 final Logger logger = Logger.getLogger(getClass().getPackage().getName());
96
97 public Context(PrintWriter out, PrintWriter err) {
98 this.tdMap = new HashMap<>();
99 this.pkgMap = new HashMap<>();
100 this.headerMap = new HashMap<>();
101 this.clangArgs = new ArrayList<>();
102 this.sources = new TreeSet<>();
103 this.libraryNames = new ArrayList<>();
104 this.libraryPaths = new ArrayList<>();
105 this.linkCheckPaths = new ArrayList<>();
106 this.excludeSymbols = new ArrayList<>();
107 this.parser = new Parser(out, err, Main.INCLUDE_MACROS);
108 this.out = out;
109 this.err = err;
110 }
111
112 public Context() {
113 this(new PrintWriter(System.out, true), new PrintWriter(System.err, true));
114 }
115
116 TypeDictionary typeDictionaryFor(String pkg) {
117 return tdMap.computeIfAbsent(pkg, p->new TypeDictionary(this, p));
118 }
119
120 void addClangArg(String arg) {
121 clangArgs.add(arg);
122 }
123
124 public void addSource(Path path) {
125 sources.add(path);
126 }
127
128 void addLibraryName(String name) {
129 libraryNames.add(name);
130 }
131
132 void addLibraryPath(String path) {
133 libraryPaths.add(path);
134 }
135
136 void addLinkCheckPath(String path) {
137 linkCheckPaths.add(path);
138 }
139
140 void addExcludeSymbols(String pattern) {
141 excludeSymbols.add(Pattern.compile(pattern));
142 }
143
144 private void initSymChecker() {
145 if (!libraryNames.isEmpty() && !linkCheckPaths.isEmpty()) {
146 Library[] libs = LibrariesHelper.loadLibraries(MethodHandles.lookup(),
147 linkCheckPaths.toArray(new String[0]),
148 libraryNames.toArray(new String[0]));
149
150 // check if the given symbol is found in any of the libraries or not.
151 // If not found, warn the user for the missing symbol.
152 symChecker = name -> {
153 if (Main.DEBUG) {
154 err.println("Searching symbol: " + name);
155 }
156 return (Arrays.stream(libs).filter(lib -> {
157 try {
158 lib.lookup(name);
159 if (Main.DEBUG) {
160 err.println("Found symbol: " + name);
161 }
162 return true;
163 } catch (NoSuchMethodException nsme) {
164 return false;
165 }
166 }).findFirst().isPresent());
167 };
168 } else {
169 symChecker = null;
170 }
171 }
172
173 private boolean isSymbolFound(String name) {
174 return symChecker == null? true : symChecker.test(name);
175 }
176
177 private void initSymFilter() {
178 if (!excludeSymbols.isEmpty()) {
179 Pattern[] pats = excludeSymbols.toArray(new Pattern[0]);
180 symFilter = name -> {
181 return Arrays.stream(pats).filter(pat -> pat.matcher(name).matches()).
182 findFirst().isPresent();
183 };
184 } else {
185 symFilter = null;
186 }
187 }
188
189 private boolean isSymbolExcluded(String name) {
190 return symFilter == null? false : symFilter.test(name);
191 }
192
193 /**
194 * Setup a package name for a given folder.
195 *
196 * @param folder The path to the folder, use null to set catch-all.
197 * @param pkg The package name
198 * @return True if the folder is setup successfully. False is a package
199 * has been assigned for the folder.
200 */
201 public boolean usePackageForFolder(Path folder, String pkg) {
202 if (folder != null) {
203 folder = folder.toAbsolutePath();
204 if (!Files.isDirectory(folder)) {
205 folder = folder.getParent();
206 }
207 }
208 String existing = pkgMap.putIfAbsent(folder, pkg);
209 final String finalFolder = (null == folder) ? "all folders not configured" : folder.toString();
210 if (null == existing) {
211 logger.config(() -> "Package " + pkg + " is selected for " + finalFolder);
212 return true;
213 } else {
214 logger.warning(() -> "Package " + existing + " had been selected for " + finalFolder + ", request to use " + pkg + " is ignored.");
215 return false;
216 }
217 }
218
219 static class Entity {
220 final String pkg;
221 final String entity;
222
223 Entity(String pkg, String entity) {
224 this.pkg = pkg;
225 this.entity = entity;
226 }
227 }
228
229 /**
230 * Determine package and interface name given a path. If the path is
231 * a folder, then only package name is determined. The package name is
232 * determined with the longest path matching the setup. If the path is not
233 * setup for any package, the default package name is returned.
234 *
235 * @param origin The source path
236 * @return The Entity
237 * @see Context::usePackageForFolder(Path, String)
238 */
239 Entity whatis(Path origin) {
240 // normalize to absolute path
241 origin = origin.toAbsolutePath();
242 String filename = null;
243 if (!Files.isDirectory(origin)) {
244 // ensure it's a folder name
245 filename = origin.getFileName().toString();
246 origin = origin.getParent();
247 }
248 Path path = origin;
249
250 // search the map for a hit with longest path
251 while (path != null && !pkgMap.containsKey(path)) {
252 path = path.getParent();
253 }
254
255 int start;
256 String pkg;
257 if (path != null) {
258 start = path.getNameCount();
259 pkg = pkgMap.get(path);
260 } else {
261 pkg = pkgMap.get(null);
262 if (pkg == null) {
263 start = 0;
264 pkg = defaultPkg;
265 } else {
266 start = origin.getNameCount();
267 }
268 }
269
270 if (filename == null) {
271 // a folder, only pkg name matters
272 return new Entity(pkg, null);
273 }
274
275 StringBuilder sb = new StringBuilder();
276 while (start < origin.getNameCount()) {
277 sb.append(Utils.toJavaIdentifier(origin.getName(start++).toString()));
278 sb.append("_");
279 }
280
281 int ext = filename.lastIndexOf('.');
282 if (ext != -1) {
283 sb.append(filename.substring(0, ext));
284 } else {
285 sb.append(filename);
286 }
287 return new Entity(pkg, Utils.toClassName(sb.toString()));
288 }
289
290 HeaderFile getHeaderFile(Path header, HeaderFile main) {
291 if (!Files.isRegularFile(header)) {
292 logger.warning(() -> "Not a regular file: " + header.toString());
293 throw new IllegalArgumentException(header.toString());
294 }
295
296 final Context.Entity e = whatis(header);
297 return new HeaderFile(this, header, e.pkg, e.entity, main);
298 }
299
300 void processTree(Tree tree, HeaderFile main, Function<HeaderFile, AsmCodeFactory> fn) {
301 SourceLocation loc = tree.location();
302
303 HeaderFile header;
304 boolean isBuiltIn = false;
305
306 if (tree.isFromMain()) {
307 header = main;
308 } else {
309 SourceLocation.Location src = loc.getFileLocation();
310 if (src == null) {
311 logger.info(() -> "Tree " + tree.name() + "@" + tree.USR() + " has no FileLocation");
312 return;
313 }
314
315 Path p = src.path();
316 if (p == null) {
317 logger.fine(() -> "Found built-in type: " + tree.name());
318 header = main;
319 isBuiltIn = true;
320 } else {
321 p = p.normalize().toAbsolutePath();
322 header = headerMap.get(p);
323 if (header == null) {
324 final HeaderFile hf = header = getHeaderFile(p, main);
325 logger.config(() -> "First encounter of header file " + hf.path + ", assigned to package " + hf.pkgName);
326 // Only generate code for header files sepcified or in the same package
327 // System headers are excluded, they need to be explicitly specified in jextract cmdline
328 if (sources.contains(p) ||
329 (!loc.isInSystemHeader()) && (header.pkgName.equals(main.pkgName))) {
330 logger.config("Code gen for header " + p + " enabled in package " + header.pkgName);
331 header.useCodeFactory(fn.apply(header));
332 }
333 headerMap.put(p, header);
334 }
335 }
336 }
337
338 header.processTree(tree, main, isBuiltIn);
339 }
340
341 public void parse() {
342 parse(header -> new AsmCodeFactory(this, header));
343 }
344
345 private boolean symbolFilter(Tree tree) {
346 String name = tree.name();
347 if (isSymbolExcluded(name)) {
348 return false;
349 }
350
351 // check for function symbols in libraries & warn missing symbols
352 if (tree instanceof FunctionTree && !isSymbolFound(name)) {
353 err.println(Main.format("warn.symbol.not.found", name));
354 }
355
356 return true;
357 }
358
359 public void parse(Function<HeaderFile, AsmCodeFactory> fn) {
360 initSymChecker();
361 initSymFilter();
362
363 List<HeaderTree> headers = parser.parse(sources, clangArgs);
364 processHeaders(headers, fn);
365 }
366
367 private void processHeaders(List<HeaderTree> headers, Function<HeaderFile, AsmCodeFactory> fn) {
368 headers.stream().
369 map(new TreeFilter(this::symbolFilter)).
370 map(new TypedefHandler()).
371 map(new EmptyNameHandler()).
372 forEach(header -> {
373 HeaderFile hf = headerMap.computeIfAbsent(header.path(), p -> getHeaderFile(p, null));
374 hf.useLibraries(libraryNames, libraryPaths);
375 hf.useCodeFactory(fn.apply(hf));
376 logger.info(() -> "Processing header file " + header.path());
377
378 header.declarations().stream()
379 .peek(decl -> logger.finest(
380 () -> "Cursor: " + decl.name() + "@" + decl.USR() + "?" + decl.isDeclaration()))
381 .forEach(decl -> processTree(decl, hf, fn));
382 });
383 }
384
385 private Map<String, List<AsmCodeFactory>> getPkgCfMap() {
386 final Map<String, List<AsmCodeFactory>> mapPkgCf = new HashMap<>();
387 // Build the pkg to CodeFactory map
388 headerMap.values().forEach(header -> {
389 AsmCodeFactory cf = header.getCodeFactory();
390 String pkg = header.pkgName;
391 logger.config(() -> "File " + header + " is in package: " + pkg);
392 if (cf == null) {
393 logger.config(() -> "File " + header + " code generation is not activated!");
394 return;
395 }
396 List<AsmCodeFactory> l = mapPkgCf.computeIfAbsent(pkg, k -> new ArrayList<>());
397 l.add(cf);
398 logger.config(() -> "Add cf " + cf + " to pkg " + pkg + ", size is now " + l.size());
399 });
400 return Collections.unmodifiableMap(mapPkgCf);
401 }
402
403 public Map<String, byte[]> collectClasses(String... pkgs) {
404 final Map<String, byte[]> rv = new HashMap<>();
405 final Map<String, List<AsmCodeFactory>> mapPkgCf = getPkgCfMap();
406 for (String pkg_name : pkgs) {
407 mapPkgCf.getOrDefault(pkg_name, Collections.emptyList())
408 .forEach(cf -> rv.putAll(cf.collect()));
409 }
410 return Collections.unmodifiableMap(rv);
411 }
412
413 private static final String JEXTRACT_MANIFEST = "META-INFO" + File.separatorChar + "jextract.properties";
414
415 @SuppressWarnings("deprecation")
416 private byte[] getJextractProperties(String[] args) {
417 Properties props = new Properties();
418 props.setProperty("os.name", System.getProperty("os.name"));
419 props.setProperty("os.version", System.getProperty("os.version"));
420 props.setProperty("os.arch", System.getProperty("os.arch"));
421 props.setProperty("jextract.args", Arrays.toString(args));
422 ByteArrayOutputStream baos = new ByteArrayOutputStream();
423 props.save(baos, "jextract meta data");
424 return baos.toByteArray();
425 }
426
427 void collectClassFiles(Path destDir, String[] args, String... pkgs) throws IOException {
428 try {
429 collectClasses(pkgs).entrySet().stream().forEach(e -> {
430 try {
431 String path = e.getKey().replace('.', File.separatorChar) + ".class";
432 logger.fine(() -> "Writing " + path);
433 Path fullPath = destDir.resolve(path).normalize();
434 Files.createDirectories(fullPath.getParent());
435 try (OutputStream fos = Files.newOutputStream(fullPath)) {
436 fos.write(e.getValue());
437 fos.flush();
438 }
439 } catch (IOException ioe) {
440 throw new UncheckedIOException(ioe);
441 }
442 });
443
444 Path propsPath = destDir.resolve(JEXTRACT_MANIFEST).normalize();
445 Files.createDirectories(propsPath.getParent());
446 try (OutputStream fos = Files.newOutputStream(propsPath)) {
447 fos.write(getJextractProperties(args));
448 fos.flush();
449 }
450 } catch (UncheckedIOException uioe) {
451 throw uioe.getCause();
452 }
453 }
454
455 private void writeJar(AsmCodeFactory cf, JarOutputStream jar) {
456 cf.collect().entrySet().stream().forEach(e -> {
457 try {
458 String path = e.getKey().replace('.', File.separatorChar) + ".class";
459 logger.fine(() -> "Add " + path);
460 jar.putNextEntry(new ZipEntry(path));
461 jar.write(e.getValue());
462 jar.closeEntry();
463 } catch (IOException ioe) {
464 throw new UncheckedIOException(ioe);
465 }
466 });
467 }
468
469 public void collectJarFile(final JarOutputStream jos, String[] args, String... pkgs) {
470 final Map<String, List<AsmCodeFactory>> mapPkgCf = getPkgCfMap();
471
472 for (String pkg_name : pkgs) {
473 // convert '.' to '/' to use as a path
474 String entryName = Utils.toInternalName(pkg_name, "");
475 // package folder
476 if (!entryName.isEmpty()) {
477 try {
478 jos.putNextEntry(new ZipEntry(entryName));
479 } catch (IOException ex) {
480 throw new UncheckedIOException(ex);
481 }
482 }
483 logger.fine(() -> "Produce for package " + pkg_name);
484 mapPkgCf.getOrDefault(pkg_name, Collections.emptyList())
485 .forEach(cf -> writeJar(cf, jos));
486 }
487
488 try {
489 jos.putNextEntry(new ZipEntry(JEXTRACT_MANIFEST));
490 jos.write(getJextractProperties(args));
491 jos.closeEntry();
492 } catch (IOException ioe) {
493 throw new UncheckedIOException(ioe);
494 }
495 }
496
497 void collectJarFile(final Path jar, String[] args, String... pkgs) throws IOException {
498 logger.info(() -> "Collecting jar file " + jar);
499 try (OutputStream os = Files.newOutputStream(jar, CREATE, TRUNCATE_EXISTING, WRITE);
500 JarOutputStream jo = new JarOutputStream(os)) {
501 collectJarFile(jo, args, pkgs);
502 } catch (UncheckedIOException uioe) {
503 throw uioe.getCause();
504 }
505 }
506
507 /**
508 * Perform a local lookup, any undefined type will cause a JType
509 * be defined within origin scope.
510 *
511 * @param type The libclang type
512 * @param origin The path of the file where type is encountered
513 * @return The JType
514 */
515 JType getJType(final Type type, Path origin) {
516 Path p = origin.normalize().toAbsolutePath();
517
518 HeaderFile hf = headerMap.get(p);
519 // We should not encounter a type if the header file reference to it is not yet processed
520 assert(null != hf);
521 if (hf == null) {
522 throw new IllegalArgumentException("Failed to lookup header for " + p + " (origin: " + origin + ")");
523 }
524
525 return hf.localLookup(type);
526 }
527
528 /**
529 * Perform a global lookup
530 *
531 * @param c The cursor define or declare the type.
532 * @return
533 */
534 JType getJType(final Cursor c) {
535 if (c.isInvalid()) {
536 throw new IllegalArgumentException();
537 }
538 SourceLocation loc = c.getSourceLocation();
539 if (null == loc) {
540 return null;
541 }
542 Path p = loc.getFileLocation().path();
543 if (null == p) {
544 return null;
545 }
546 return getJType(c.type(), p);
547 }
548 }