1 /*
2 * Copyright (c) 2012, 2016, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25
26 package com.sun.tools.sjavac;
27
28 import java.io.BufferedReader;
29 import java.io.File;
30 import java.io.FileNotFoundException;
31 import java.io.FileReader;
32 import java.io.FileWriter;
33 import java.io.IOException;
34 import java.io.Writer;
35 import java.net.URI;
36 import java.nio.file.NoSuchFileException;
37 import java.text.SimpleDateFormat;
38 import java.util.Collection;
39 import java.util.Collections;
40 import java.util.Date;
41 import java.util.HashMap;
42 import java.util.HashSet;
43 import java.util.List;
44 import java.util.Map;
45 import java.util.Set;
46 import java.util.stream.Collectors;
47
48 import com.sun.tools.sjavac.comp.CompilationService;
49 import com.sun.tools.sjavac.options.Options;
50 import com.sun.tools.sjavac.pubapi.PubApi;
51
52 /**
53 * The javac state class maintains the previous (prev) and the current (now)
54 * build states and everything else that goes into the javac_state file.
55 *
56 * <p><b>This is NOT part of any supported API.
57 * If you write code that depends on this, you do so at your own risk.
58 * This code and its internal interfaces are subject to change or
59 * deletion without notice.</b>
60 */
61 public class JavacState {
62 // The arguments to the compile. If not identical, then it cannot
63 // be an incremental build!
64 String theArgs;
65 // The number of cores limits how many threads are used for heavy concurrent work.
66 int numCores;
67
68 // The bin_dir/javac_state
69 private File javacState;
70
71 // The previous build state is loaded from javac_state
72 private BuildState prev;
73 // The current build state is constructed during the build,
74 // then saved as the new javac_state.
75 private BuildState now;
76
77 // Something has changed in the javac_state. It needs to be saved!
78 private boolean needsSaving;
79 // If this is a new javac_state file, then do not print unnecessary messages.
80 private boolean newJavacState;
81
82 // These are packages where something has changed and the package
83 // needs to be recompiled. Actions that trigger recompilation:
84 // * source belonging to the package has changed
85 // * artifact belonging to the package is lost, or its timestamp has been changed.
86 // * an unknown artifact has appeared, we simply delete it, but we also trigger a recompilation.
87 // * a package that is tainted, taints all packages that depend on it.
88 private Set<String> taintedPackages;
89 // After a compile, the pubapis are compared with the pubapis stored in the javac state file.
90 // Any packages where the pubapi differ are added to this set.
91 // Later we use this set and the dependency information to taint dependent packages.
92 private Set<String> packagesWithChangedPublicApis;
93 // When a module-info.java file is changed, taint the module,
94 // then taint all modules that depend on that that module.
95 // A module dependency can occur directly through a require, or
96 // indirectly through a module that does a public export for the first tainted module.
97 // When all modules are tainted, then taint all packages belonging to these modules.
98 // Then rebuild. It is perhaps possible (and valuable?) to do a more fine-grained examination of the
99 // change in module-info.java, but that will have to wait.
100 private Set<String> taintedModules;
101 // The set of all packages that has been recompiled.
102 // Copy over the javac_state for the packages that did not need recompilation,
103 // verbatim from the previous (prev) to the new (now) build state.
104 private Set<String> recompiledPackages;
105
106 // The output directories filled with tasty artifacts.
107 private File binDir, gensrcDir, headerDir, stateDir;
108
109 // The current status of the file system.
110 private Set<File> binArtifacts;
111 private Set<File> gensrcArtifacts;
112 private Set<File> headerArtifacts;
113
114 // The status of the sources.
115 Set<Source> removedSources = null;
116 Set<Source> addedSources = null;
117 Set<Source> modifiedSources = null;
118
119 // Visible sources for linking. These are the only
120 // ones that -sourcepath is allowed to see.
121 Set<URI> visibleSrcs;
122
123 // Setup transform that always exist.
124 private CompileJavaPackages compileJavaPackages = new CompileJavaPackages();
125
126 // Command line options.
127 private Options options;
128
129 JavacState(Options op, boolean removeJavacState) {
130 options = op;
131 numCores = options.getNumCores();
132 theArgs = options.getStateArgsString();
133 binDir = Util.pathToFile(options.getDestDir());
134 gensrcDir = Util.pathToFile(options.getGenSrcDir());
135 headerDir = Util.pathToFile(options.getHeaderDir());
136 stateDir = Util.pathToFile(options.getStateDir());
137 javacState = new File(stateDir, "javac_state");
138 if (removeJavacState && javacState.exists()) {
139 javacState.delete();
140 }
141 newJavacState = false;
142 if (!javacState.exists()) {
143 newJavacState = true;
144 // If there is no javac_state then delete the contents of all the artifact dirs!
145 // We do not want to risk building a broken incremental build.
146 // BUT since the makefiles still copy things straight into the bin_dir et al,
147 // we avoid deleting files here, if the option --permit-unidentified-classes was supplied.
148 if (!options.areUnidentifiedArtifactsPermitted()) {
149 deleteContents(binDir);
150 deleteContents(gensrcDir);
151 deleteContents(headerDir);
152 }
153 needsSaving = true;
154 }
155 prev = new BuildState();
156 now = new BuildState();
157 taintedPackages = new HashSet<>();
158 recompiledPackages = new HashSet<>();
159 packagesWithChangedPublicApis = new HashSet<>();
160 }
161
162 public BuildState prev() { return prev; }
163 public BuildState now() { return now; }
164
165 /**
166 * Remove args not affecting the state.
167 */
168 static String[] removeArgsNotAffectingState(String[] args) {
169 String[] out = new String[args.length];
170 int j = 0;
171 for (int i = 0; i<args.length; ++i) {
172 if (args[i].equals("-j")) {
173 // Just skip it and skip following value
174 i++;
175 } else if (args[i].startsWith("--server:")) {
176 // Just skip it.
177 } else if (args[i].startsWith("--log=")) {
178 // Just skip it.
179 } else if (args[i].equals("--compare-found-sources")) {
180 // Just skip it and skip verify file name
181 i++;
182 } else {
183 // Copy argument.
184 out[j] = args[i];
185 j++;
186 }
187 }
188 String[] ret = new String[j];
189 System.arraycopy(out, 0, ret, 0, j);
190 return ret;
191 }
192
193 /**
194 * Specify which sources are visible to the compiler through -sourcepath.
195 */
196 public void setVisibleSources(Map<String,Source> vs) {
197 visibleSrcs = new HashSet<>();
198 for (String s : vs.keySet()) {
199 Source src = vs.get(s);
200 visibleSrcs.add(src.file().toURI());
201 }
202 }
203
204 /**
205 * Returns true if this is an incremental build.
206 */
207 public boolean isIncremental() {
208 return !prev.sources().isEmpty();
209 }
210
211 /**
212 * Find all artifacts that exists on disk.
213 */
214 public void findAllArtifacts() {
215 binArtifacts = findAllFiles(binDir);
216 gensrcArtifacts = findAllFiles(gensrcDir);
217 headerArtifacts = findAllFiles(headerDir);
218 }
219
220 /**
221 * Lookup the artifacts generated for this package in the previous build.
222 */
223 private Map<String,File> fetchPrevArtifacts(String pkg) {
224 Package p = prev.packages().get(pkg);
225 if (p != null) {
226 return p.artifacts();
227 }
228 return new HashMap<>();
229 }
230
231 /**
232 * Delete all prev artifacts in the currently tainted packages.
233 */
234 public void deleteClassArtifactsInTaintedPackages() {
235 for (String pkg : taintedPackages) {
236 Map<String,File> arts = fetchPrevArtifacts(pkg);
237 for (File f : arts.values()) {
238 if (f.exists() && f.getName().endsWith(".class")) {
239 f.delete();
240 }
241 }
242 }
243 }
244
245 /**
246 * Mark the javac_state file to be in need of saving and as a side effect,
247 * it gets a new timestamp.
248 */
249 private void needsSaving() {
250 needsSaving = true;
251 }
252
253 /**
254 * Save the javac_state file.
255 */
256 public void save() throws IOException {
257 if (!needsSaving)
258 return;
259 try (FileWriter out = new FileWriter(javacState)) {
260 StringBuilder b = new StringBuilder();
261 long millisNow = System.currentTimeMillis();
262 Date d = new Date(millisNow);
263 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS");
264 b.append("# javac_state ver 0.4 generated "+millisNow+" "+df.format(d)+"\n");
265 b.append("# This format might change at any time. Please do not depend on it.\n");
266 b.append("# R arguments\n");
267 b.append("# M module\n");
268 b.append("# P package\n");
269 b.append("# S C source_tobe_compiled timestamp\n");
270 b.append("# S L link_only_source timestamp\n");
271 b.append("# G C generated_source timestamp\n");
272 b.append("# A artifact timestamp\n");
273 b.append("# D S dependant -> source dependency\n");
274 b.append("# D C dependant -> classpath dependency\n");
275 b.append("# I pubapi\n");
276 b.append("R ").append(theArgs).append("\n");
277
278 // Copy over the javac_state for the packages that did not need recompilation.
279 now.copyPackagesExcept(prev, recompiledPackages, new HashSet<String>());
280 // Save the packages, ie package names, dependencies, pubapis and artifacts!
281 // I.e. the lot.
282 Module.saveModules(now.modules(), b);
283
284 String s = b.toString();
285 out.write(s, 0, s.length());
286 }
287 }
288
289 /**
290 * Load a javac_state file.
291 */
292 public static JavacState load(Options options) {
293 JavacState db = new JavacState(options, false);
294 Module lastModule = null;
295 Package lastPackage = null;
296 Source lastSource = null;
297 boolean noFileFound = false;
298 boolean foundCorrectVerNr = false;
299 boolean newCommandLine = false;
300 boolean syntaxError = false;
301
302 Log.debug("Loading javac state file: " + db.javacState);
303
304 try (BufferedReader in = new BufferedReader(new FileReader(db.javacState))) {
305 for (;;) {
306 String l = in.readLine();
307 if (l==null) break;
308 if (l.length()>=3 && l.charAt(1) == ' ') {
309 char c = l.charAt(0);
310 if (c == 'M') {
311 lastModule = db.prev.loadModule(l);
312 } else
313 if (c == 'P') {
314 if (lastModule == null) { syntaxError = true; break; }
315 lastPackage = db.prev.loadPackage(lastModule, l);
316 } else
317 if (c == 'D') {
318 if (lastModule == null || lastPackage == null) { syntaxError = true; break; }
319 char depType = l.charAt(2);
320 if (depType != 'S' && depType != 'C')
321 throw new RuntimeException("Bad dependency string: " + l);
322 lastPackage.parseAndAddDependency(l.substring(4), depType == 'C');
323 } else
324 if (c == 'I') {
325 if (lastModule == null || lastPackage == null) { syntaxError = true; break; }
326 lastPackage.getPubApi().appendItem(l.substring(2)); // Strip "I "
327 } else
328 if (c == 'A') {
329 if (lastModule == null || lastPackage == null) { syntaxError = true; break; }
330 lastPackage.loadArtifact(l);
331 } else
332 if (c == 'S') {
333 if (lastModule == null || lastPackage == null) { syntaxError = true; break; }
334 lastSource = db.prev.loadSource(lastPackage, l, false);
335 } else
336 if (c == 'G') {
337 if (lastModule == null || lastPackage == null) { syntaxError = true; break; }
338 lastSource = db.prev.loadSource(lastPackage, l, true);
339 } else
340 if (c == 'R') {
341 String ncmdl = "R "+db.theArgs;
342 if (!l.equals(ncmdl)) {
343 newCommandLine = true;
344 }
345 } else
346 if (c == '#') {
347 if (l.startsWith("# javac_state ver ")) {
348 int sp = l.indexOf(" ", 18);
349 if (sp != -1) {
350 String ver = l.substring(18,sp);
351 if (!ver.equals("0.4")) {
352 break;
353 }
354 foundCorrectVerNr = true;
355 }
356 }
357 }
358 }
359 }
360 } catch (FileNotFoundException | NoSuchFileException e) {
361 // Silently create a new javac_state file.
362 noFileFound = true;
363 } catch (IOException e) {
364 Log.warn("Dropping old javac_state because of errors when reading it.");
365 db = new JavacState(options, true);
366 foundCorrectVerNr = true;
367 newCommandLine = false;
368 syntaxError = false;
369 }
370 if (foundCorrectVerNr == false && !noFileFound) {
371 Log.debug("Dropping old javac_state since it is of an old version.");
372 db = new JavacState(options, true);
373 } else
374 if (newCommandLine == true && !noFileFound) {
375 Log.debug("Dropping old javac_state since a new command line is used!");
376 db = new JavacState(options, true);
377 } else
378 if (syntaxError == true) {
379 Log.warn("Dropping old javac_state since it contains syntax errors.");
380 db = new JavacState(options, true);
381 }
382 db.prev.calculateDependents();
383 return db;
384 }
385
386 /**
387 * Mark a java package as tainted, ie it needs recompilation.
388 */
389 public void taintPackage(String name, String because) {
390 if (!taintedPackages.contains(name)) {
391 if (because != null) Log.debug("Tainting "+Util.justPackageName(name)+" because "+because);
392 // It has not been tainted before.
393 taintedPackages.add(name);
394 needsSaving();
395 Package nowp = now.packages().get(name);
396 if (nowp != null) {
397 for (String d : nowp.dependents()) {
398 taintPackage(d, because);
399 }
400 }
401 }
402 }
403
404 /**
405 * This packages need recompilation.
406 */
407 public Set<String> taintedPackages() {
408 return taintedPackages;
409 }
410
411 /**
412 * Clean out the tainted package set, used after the first round of compiles,
413 * prior to propagating dependencies.
414 */
415 public void clearTaintedPackages() {
416 taintedPackages = new HashSet<>();
417 }
418
419 /**
420 * Go through all sources and check which have been removed, added or modified
421 * and taint the corresponding packages.
422 */
423 public void checkSourceStatus(boolean check_gensrc) {
424 removedSources = calculateRemovedSources();
425 for (Source s : removedSources) {
426 if (!s.isGenerated() || check_gensrc) {
427 taintPackage(s.pkg().name(), "source "+s.name()+" was removed");
428 }
429 }
430
431 addedSources = calculateAddedSources();
432 for (Source s : addedSources) {
433 String msg = null;
434 if (isIncremental()) {
435 // When building from scratch, there is no point
436 // printing "was added" for every file since all files are added.
437 // However for an incremental build it makes sense.
438 msg = "source "+s.name()+" was added";
439 }
440 if (!s.isGenerated() || check_gensrc) {
441 taintPackage(s.pkg().name(), msg);
442 }
443 }
444
445 modifiedSources = calculateModifiedSources();
446 for (Source s : modifiedSources) {
447 if (!s.isGenerated() || check_gensrc) {
448 taintPackage(s.pkg().name(), "source "+s.name()+" was modified");
449 }
450 }
451 }
452
453 /**
454 * Acquire the compile_java_packages suffix rule for .java files.
455 */
456 public Map<String,Transformer> getJavaSuffixRule() {
457 Map<String,Transformer> sr = new HashMap<>();
458 sr.put(".java", compileJavaPackages);
459 return sr;
460 }
461
462
463 /**
464 * If artifacts have gone missing, force a recompile of the packages
465 * they belong to.
466 */
467 public void taintPackagesThatMissArtifacts() {
468 for (Package pkg : prev.packages().values()) {
469 for (File f : pkg.artifacts().values()) {
470 if (!f.exists()) {
471 // Hmm, the artifact on disk does not exist! Someone has removed it....
472 // Lets rebuild the package.
473 taintPackage(pkg.name(), ""+f+" is missing.");
474 }
475 }
476 }
477 }
478
479 /**
480 * Propagate recompilation through the dependency chains.
481 * Avoid re-tainting packages that have already been compiled.
482 */
483 public void taintPackagesDependingOnChangedPackages(Set<String> pkgsWithChangedPubApi, Set<String> recentlyCompiled) {
484 // For each to-be-recompiled-candidates...
485 for (Package pkg : new HashSet<>(prev.packages().values())) {
486 // Find out what it depends upon...
487 Set<String> deps = pkg.typeDependencies()
488 .values()
489 .stream()
490 .flatMap(Collection::stream)
491 .collect(Collectors.toSet());
492 for (String dep : deps) {
493 String depPkg = ":" + dep.substring(0, dep.lastIndexOf('.'));
494 if (depPkg.equals(pkg.name()))
495 continue;
496 // Checking if that dependency has changed
497 if (pkgsWithChangedPubApi.contains(depPkg) && !recentlyCompiled.contains(pkg.name())) {
498 taintPackage(pkg.name(), "its depending on " + depPkg);
499 }
500 }
501 }
502 }
503
504 /**
505 * Compare the javac_state recorded public apis of packages on the classpath
506 * with the actual public apis on the classpath.
507 */
508 public void taintPackagesDependingOnChangedClasspathPackages() throws IOException {
509
510 // 1. Collect fully qualified names of all interesting classpath dependencies
511 Set<String> fqDependencies = new HashSet<>();
512 for (Package pkg : prev.packages().values()) {
513 // Check if this package was compiled. If it's presence is recorded
514 // because it was on the class path and we needed to save it's
515 // public api, it's not a candidate for tainting.
516 if (pkg.sources().isEmpty())
517 continue;
518
519 pkg.typeClasspathDependencies().values().forEach(fqDependencies::addAll);
520 }
521
522 // 2. Extract the public APIs from the on disk .class files
523 // (Reason for doing step 1 in a separate phase is to avoid extracting
524 // public APIs of the same class twice.)
525 PubApiExtractor pubApiExtractor = new PubApiExtractor(options);
526 Map<String, PubApi> onDiskPubApi = new HashMap<>();
527 for (String cpDep : fqDependencies) {
528 onDiskPubApi.put(cpDep, pubApiExtractor.getPubApi(cpDep));
529 }
530 pubApiExtractor.close();
531
532 // 3. Compare them with the public APIs as of last compilation (loaded from javac_state)
533 nextPkg:
534 for (Package pkg : prev.packages().values()) {
535 // Check if this package was compiled. If it's presence is recorded
536 // because it was on the class path and we needed to save it's
537 // public api, it's not a candidate for tainting.
538 if (pkg.sources().isEmpty())
539 continue;
540
541 Set<String> cpDepsOfThisPkg = new HashSet<>();
542 for (Set<String> cpDeps : pkg.typeClasspathDependencies().values())
543 cpDepsOfThisPkg.addAll(cpDeps);
544
545 for (String fqDep : cpDepsOfThisPkg) {
546
547 String depPkg = ":" + fqDep.substring(0, fqDep.lastIndexOf('.'));
548 PubApi prevPkgApi = prev.packages().get(depPkg).getPubApi();
549
550 // This PubApi directly lists the members of the class,
551 // i.e. [ MEMBER1, MEMBER2, ... ]
552 PubApi prevDepApi = prevPkgApi.types.get(fqDep).pubApi;
553
554 // In order to dive *into* the class, we need to add
555 // .types.get(fqDep).pubApi below.
556 PubApi currentDepApi = onDiskPubApi.get(fqDep).types.get(fqDep).pubApi;
557
558 if (!currentDepApi.isBackwardCompatibleWith(prevDepApi)) {
559 List<String> apiDiff = currentDepApi.diff(prevDepApi);
560 taintPackage(pkg.name(), "depends on classpath "
561 + "package which has an updated package api: "
562 + String.join("\n", apiDiff));
563 //Log.debug("========================================");
564 //Log.debug("------ PREV API ------------------------");
565 //prevDepApi.asListOfStrings().forEach(Log::debug);
566 //Log.debug("------ CURRENT API ---------------------");
567 //currentDepApi.asListOfStrings().forEach(Log::debug);
568 //Log.debug("========================================");
569 continue nextPkg;
570 }
571 }
572 }
573 }
574
575 /**
576 * Scan all output dirs for artifacts and remove those files (artifacts?)
577 * that are not recognized as such, in the javac_state file.
578 */
579 public void removeUnidentifiedArtifacts() {
580 Set<File> allKnownArtifacts = new HashSet<>();
581 for (Package pkg : prev.packages().values()) {
582 for (File f : pkg.artifacts().values()) {
583 allKnownArtifacts.add(f);
584 }
585 }
586 // Do not forget about javac_state....
587 allKnownArtifacts.add(javacState);
588
589 for (File f : binArtifacts) {
590 if (!allKnownArtifacts.contains(f) &&
591 !options.isUnidentifiedArtifactPermitted(f.getAbsolutePath())) {
592 Log.debug("Removing "+f.getPath()+" since it is unknown to the javac_state.");
593 f.delete();
594 }
595 }
596 for (File f : headerArtifacts) {
597 if (!allKnownArtifacts.contains(f)) {
598 Log.debug("Removing "+f.getPath()+" since it is unknown to the javac_state.");
599 f.delete();
600 }
601 }
602 for (File f : gensrcArtifacts) {
603 if (!allKnownArtifacts.contains(f)) {
604 Log.debug("Removing "+f.getPath()+" since it is unknown to the javac_state.");
605 f.delete();
606 }
607 }
608 }
609
610 /**
611 * Remove artifacts that are no longer produced when compiling!
612 */
613 public void removeSuperfluousArtifacts(Set<String> recentlyCompiled) {
614 // Nothing to do, if nothing was recompiled.
615 if (recentlyCompiled.size() == 0) return;
616
617 for (String pkg : now.packages().keySet()) {
618 // If this package has not been recompiled, skip the check.
619 if (!recentlyCompiled.contains(pkg)) continue;
620 Collection<File> arts = now.artifacts().values();
621 for (File f : fetchPrevArtifacts(pkg).values()) {
622 if (!arts.contains(f)) {
623 Log.debug("Removing "+f.getPath()+" since it is now superfluous!");
624 if (f.exists()) f.delete();
625 }
626 }
627 }
628 }
629
630 /**
631 * Return those files belonging to prev, but not now.
632 */
633 private Set<Source> calculateRemovedSources() {
634 Set<Source> removed = new HashSet<>();
635 for (String src : prev.sources().keySet()) {
636 if (now.sources().get(src) == null) {
637 removed.add(prev.sources().get(src));
638 }
639 }
640 return removed;
641 }
642
643 /**
644 * Return those files belonging to now, but not prev.
645 */
646 private Set<Source> calculateAddedSources() {
647 Set<Source> added = new HashSet<>();
648 for (String src : now.sources().keySet()) {
649 if (prev.sources().get(src) == null) {
650 added.add(now.sources().get(src));
651 }
652 }
653 return added;
654 }
655
656 /**
657 * Return those files where the timestamp is newer.
658 * If a source file timestamp suddenly is older than what is known
659 * about it in javac_state, then consider it modified, but print
660 * a warning!
661 */
662 private Set<Source> calculateModifiedSources() {
663 Set<Source> modified = new HashSet<>();
664 for (String src : now.sources().keySet()) {
665 Source n = now.sources().get(src);
666 Source t = prev.sources().get(src);
667 if (prev.sources().get(src) != null) {
668 if (t != null) {
669 if (n.lastModified() > t.lastModified()) {
670 modified.add(n);
671 } else if (n.lastModified() < t.lastModified()) {
672 modified.add(n);
673 Log.warn("The source file "+n.name()+" timestamp has moved backwards in time.");
674 }
675 }
676 }
677 }
678 return modified;
679 }
680
681 /**
682 * Recursively delete a directory and all its contents.
683 */
684 private void deleteContents(File dir) {
685 if (dir != null && dir.exists()) {
686 for (File f : dir.listFiles()) {
687 if (f.isDirectory()) {
688 deleteContents(f);
689 }
690 if (!options.isUnidentifiedArtifactPermitted(f.getAbsolutePath())) {
691 Log.debug("Removing "+f.getAbsolutePath());
692 f.delete();
693 }
694 }
695 }
696 }
697
698 /**
699 * Run the copy translator only.
700 */
701 public void performCopying(File binDir, Map<String,Transformer> suffixRules) {
702 Map<String,Transformer> sr = new HashMap<>();
703 for (Map.Entry<String,Transformer> e : suffixRules.entrySet()) {
704 if (e.getValue().getClass().equals(CopyFile.class)) {
705 sr.put(e.getKey(), e.getValue());
706 }
707 }
708 perform(null, binDir, sr);
709 }
710
711 /**
712 * Run all the translators that translate into java source code.
713 * I.e. all translators that are not copy nor compile_java_source.
714 */
715 public void performTranslation(File gensrcDir, Map<String,Transformer> suffixRules) {
716 Map<String,Transformer> sr = new HashMap<>();
717 for (Map.Entry<String,Transformer> e : suffixRules.entrySet()) {
718 Class<?> trClass = e.getValue().getClass();
719 if (trClass == CompileJavaPackages.class || trClass == CopyFile.class)
720 continue;
721
722 sr.put(e.getKey(), e.getValue());
723 }
724 perform(null, gensrcDir, sr);
725 }
726
727 /**
728 * Compile all the java sources. Return true, if it needs to be called again!
729 */
730 public boolean performJavaCompilations(CompilationService sjavac,
731 Options args,
732 Set<String> recentlyCompiled,
733 boolean[] rcValue) {
734 Map<String,Transformer> suffixRules = new HashMap<>();
735 suffixRules.put(".java", compileJavaPackages);
736 compileJavaPackages.setExtra(args);
737 rcValue[0] = perform(sjavac, binDir, suffixRules);
738 recentlyCompiled.addAll(taintedPackages());
739 clearTaintedPackages();
740 boolean again = !packagesWithChangedPublicApis.isEmpty();
741 taintPackagesDependingOnChangedPackages(packagesWithChangedPublicApis, recentlyCompiled);
742 packagesWithChangedPublicApis = new HashSet<>();
743 return again && rcValue[0];
744
745 // TODO: Figure out why 'again' checks packagesWithChangedPublicAPis.
746 // (It shouldn't matter if packages had changed pub apis as long as no
747 // one depends on them. Wouldn't it make more sense to let 'again'
748 // depend on taintedPackages?)
749 }
750
751 /**
752 * Store the source into the set of sources belonging to the given transform.
753 */
754 private void addFileToTransform(Map<Transformer,Map<String,Set<URI>>> gs, Transformer t, Source s) {
755 Map<String,Set<URI>> fs = gs.get(t);
756 if (fs == null) {
757 fs = new HashMap<>();
758 gs.put(t, fs);
759 }
760 Set<URI> ss = fs.get(s.pkg().name());
761 if (ss == null) {
762 ss = new HashSet<>();
763 fs.put(s.pkg().name(), ss);
764 }
765 ss.add(s.file().toURI());
766 }
767
768 /**
769 * For all packages, find all sources belonging to the package, group the sources
770 * based on their transformers and apply the transformers on each source code group.
771 */
772 private boolean perform(CompilationService sjavac,
773 File outputDir,
774 Map<String,Transformer> suffixRules) {
775 boolean rc = true;
776 // Group sources based on transforms. A source file can only belong to a single transform.
777 Map<Transformer,Map<String,Set<URI>>> groupedSources = new HashMap<>();
778 for (Source src : now.sources().values()) {
779 Transformer t = suffixRules.get(src.suffix());
780 if (t != null) {
781 if (taintedPackages.contains(src.pkg().name()) && !src.isLinkedOnly()) {
782 addFileToTransform(groupedSources, t, src);
783 }
784 }
785 }
786 // Go through the transforms and transform them.
787 for (Map.Entry<Transformer, Map<String, Set<URI>>> e : groupedSources.entrySet()) {
788 Transformer t = e.getKey();
789 Map<String, Set<URI>> srcs = e.getValue();
790 // These maps need to be synchronized since multiple threads will be
791 // writing results into them.
792 Map<String, Set<URI>> packageArtifacts = Collections.synchronizedMap(new HashMap<>());
793 Map<String, Map<String, Set<String>>> packageDependencies = Collections.synchronizedMap(new HashMap<>());
794 Map<String, Map<String, Set<String>>> packageCpDependencies = Collections.synchronizedMap(new HashMap<>());
795 Map<String, PubApi> packagePublicApis = Collections.synchronizedMap(new HashMap<>());
796 Map<String, PubApi> dependencyPublicApis = Collections.synchronizedMap(new HashMap<>());
797
798 boolean r = t.transform(sjavac,
799 srcs,
800 visibleSrcs,
801 prev.dependents(),
802 outputDir.toURI(),
803 packageArtifacts,
804 packageDependencies,
805 packageCpDependencies,
806 packagePublicApis,
807 dependencyPublicApis,
808 0,
809 isIncremental(),
810 numCores);
811 if (!r)
812 rc = false;
813
814 for (String p : srcs.keySet()) {
815 recompiledPackages.add(p);
816 }
817 // The transform is done! Extract all the artifacts and store the info into the Package objects.
818 for (Map.Entry<String, Set<URI>> a : packageArtifacts.entrySet()) {
819 Module mnow = now.findModuleFromPackageName(a.getKey());
820 mnow.addArtifacts(a.getKey(), a.getValue());
821 }
822 // Extract all the dependencies and store the info into the Package objects.
823 for (Map.Entry<String, Map<String, Set<String>>> a : packageDependencies.entrySet()) {
824 Map<String, Set<String>> deps = a.getValue();
825 Module mnow = now.findModuleFromPackageName(a.getKey());
826 mnow.setDependencies(a.getKey(), deps, false);
827 }
828 for (Map.Entry<String, Map<String, Set<String>>> a : packageCpDependencies.entrySet()) {
829 Map<String, Set<String>> deps = a.getValue();
830 Module mnow = now.findModuleFromPackageName(a.getKey());
831 mnow.setDependencies(a.getKey(), deps, true);
832 }
833
834 // This map contains the public api of the types that this
835 // compilation depended upon. This means that it may not contain
836 // full packages. In other words, we shouldn't remove knowledge of
837 // public apis but merge these with what we already have.
838 for (Map.Entry<String, PubApi> a : dependencyPublicApis.entrySet()) {
839 String pkg = a.getKey();
840 PubApi packagePartialPubApi = a.getValue();
841 Package pkgNow = now.findModuleFromPackageName(pkg).lookupPackage(pkg);
842 PubApi currentPubApi = pkgNow.getPubApi();
843 PubApi newPubApi = PubApi.mergeTypes(currentPubApi, packagePartialPubApi);
844 pkgNow.setPubapi(newPubApi);
845
846 // See JDK-8071904
847 if (now.packages().containsKey(pkg))
848 now.packages().get(pkg).setPubapi(newPubApi);
849 else
850 now.packages().put(pkg, pkgNow);
851 }
852
853 // The packagePublicApis cover entire packages (since sjavac compiles
854 // stuff on package level). This means that if a type is missing
855 // in the public api of a given package, it means that it has been
856 // removed. In other words, we should *set* the pubapi to whatever
857 // this map contains, and not merge it with what we already have.
858 for (Map.Entry<String, PubApi> a : packagePublicApis.entrySet()) {
859 String pkg = a.getKey();
860 PubApi newPubApi = a.getValue();
861 Module mprev = prev.findModuleFromPackageName(pkg);
862 Module mnow = now.findModuleFromPackageName(pkg);
863 mnow.setPubapi(pkg, newPubApi);
864 if (mprev.hasPubapiChanged(pkg, newPubApi)) {
865 // Aha! The pubapi of this package has changed!
866 // It can also be a new compile from scratch.
867 if (mprev.lookupPackage(pkg).existsInJavacState()) {
868 // This is an incremental compile! The pubapi
869 // did change. Trigger recompilation of dependents.
870 packagesWithChangedPublicApis.add(pkg);
871 Log.debug("The API of " + Util.justPackageName(pkg) + " has changed!");
872 }
873 }
874 }
875 }
876 return rc;
877 }
878
879 /**
880 * Utility method to recursively find all files below a directory.
881 */
882 private static Set<File> findAllFiles(File dir) {
883 Set<File> foundFiles = new HashSet<>();
884 if (dir == null) {
885 return foundFiles;
886 }
887 recurse(dir, foundFiles);
888 return foundFiles;
889 }
890
891 private static void recurse(File dir, Set<File> foundFiles) {
892 for (File f : dir.listFiles()) {
893 if (f.isFile()) {
894 foundFiles.add(f);
895 } else if (f.isDirectory()) {
896 recurse(f, foundFiles);
897 }
898 }
899 }
900
901 /**
902 * Compare the calculate source list, with an explicit list, usually
903 * supplied from the makefile. Used to detect bugs where the makefile and
904 * sjavac have different opinions on which files should be compiled.
905 */
906 public void compareWithMakefileList(File makefileSourceList)
907 throws ProblemException {
908 // If we are building on win32 using for example cygwin the paths in the
909 // makefile source list
910 // might be /cygdrive/c/.... which does not match c:\....
911 // We need to adjust our calculated sources to be identical, if
912 // necessary.
913 boolean mightNeedRewriting = File.pathSeparatorChar == ';';
914
915 if (makefileSourceList == null)
916 return;
917
918 Set<String> calculatedSources = new HashSet<>();
919 Set<String> listedSources = new HashSet<>();
920
921 // Create a set of filenames with full paths.
922 for (Source s : now.sources().values()) {
923 // Don't include link only sources when comparing sources to compile
924 if (!s.isLinkedOnly()) {
925 String path = s.file().getPath();
926 if (mightNeedRewriting)
927 path = Util.normalizeDriveLetter(path);
928 calculatedSources.add(path);
929 }
930 }
931 // Read in the file and create another set of filenames with full paths.
932 try(BufferedReader in = new BufferedReader(new FileReader(makefileSourceList))) {
933 for (;;) {
934 String l = in.readLine();
935 if (l==null) break;
936 l = l.trim();
937 if (mightNeedRewriting) {
938 if (l.indexOf(":") == 1 && l.indexOf("\\") == 2) {
939 // Everything a-ok, the format is already C:\foo\bar
940 } else if (l.indexOf(":") == 1 && l.indexOf("/") == 2) {
941 // The format is C:/foo/bar, rewrite into the above format.
942 l = l.replaceAll("/","\\\\");
943 } else if (l.charAt(0) == '/' && l.indexOf("/",1) != -1) {
944 // The format might be: /cygdrive/c/foo/bar, rewrite into the above format.
945 // Do not hardcode the name cygdrive here.
946 int slash = l.indexOf("/",1);
947 l = l.replaceAll("/","\\\\");
948 l = ""+l.charAt(slash+1)+":"+l.substring(slash+2);
949 }
950 if (Character.isLowerCase(l.charAt(0))) {
951 l = Character.toUpperCase(l.charAt(0))+l.substring(1);
952 }
953 }
954 listedSources.add(l);
955 }
956 } catch (FileNotFoundException | NoSuchFileException e) {
957 throw new ProblemException("Could not open "+makefileSourceList.getPath()+" since it does not exist!");
958 } catch (IOException e) {
959 throw new ProblemException("Could not read "+makefileSourceList.getPath());
960 }
961
962 for (String s : listedSources) {
963 if (!calculatedSources.contains(s)) {
964 throw new ProblemException("The makefile listed source "+s+" was not calculated by the smart javac wrapper!");
965 }
966 }
967
968 for (String s : calculatedSources) {
969 if (!listedSources.contains(s)) {
970 throw new ProblemException("The smart javac wrapper calculated source "+s+" was not listed by the makefiles!");
971 }
972 }
973 }
974 }