79 // * artifact belonging to the package is lost, or its timestamp has been changed.
80 // * an unknown artifact has appeared, we simply delete it, but we also trigger a recompilation.
81 // * a package that is tainted, taints all packages that depend on it.
82 private Set<String> taintedPackages;
83 // After a compile, the pubapis are compared with the pubapis stored in the javac state file.
84 // Any packages where the pubapi differ are added to this set.
85 // Later we use this set and the dependency information to taint dependent packages.
86 private Set<String> packagesWithChangedPublicApis;
87 // When a module-info.java file is changed, taint the module,
88 // then taint all modules that depend on that that module.
89 // A module dependency can occur directly through a require, or
90 // indirectly through a module that does a public export for the first tainted module.
91 // When all modules are tainted, then taint all packages belonging to these modules.
92 // Then rebuild. It is perhaps possible (and valuable?) to do a more finegrained examination of the
93 // change in module-info.java, but that will have to wait.
94 private Set<String> taintedModules;
95 // The set of all packages that has been recompiled.
96 // Copy over the javac_state for the packages that did not need recompilation,
97 // verbatim from the previous (prev) to the new (now) build state.
98 private Set<String> recompiledPackages;
99
100 // The output directories filled with tasty artifacts.
101 private File binDir, gensrcDir, headerDir, stateDir;
102
103 // The current status of the file system.
104 private Set<File> binArtifacts;
105 private Set<File> gensrcArtifacts;
106 private Set<File> headerArtifacts;
107
108 // The status of the sources.
109 Set<Source> removedSources = null;
110 Set<Source> addedSources = null;
111 Set<Source> modifiedSources = null;
112
113 // Visible sources for linking. These are the only
114 // ones that -sourcepath is allowed to see.
115 Set<URI> visibleSrcs;
116
117 // Visible classes for linking. These are the only
118 // ones that -classpath is allowed to see.
119 // It maps from a classpath root to the set of visible classes for that root.
120 // If the set is empty, then all classes are visible for that root.
121 // It can also map from a jar file to the set of visible classes for that jar file.
122 Map<URI,Set<String>> visibleClasses;
123
124 // Setup transform that always exist.
125 private CompileJavaPackages compileJavaPackages = new CompileJavaPackages();
126
127 // Where to send stdout and stderr.
128 private PrintStream out, err;
129
130 JavacState(Options options, boolean removeJavacState, PrintStream o, PrintStream e) {
131 out = o;
132 err = e;
133 numCores = options.getNumCores();
134 theArgs = options.getStateArgsString();
135 binDir = Util.pathToFile(options.getDestDir());
136 gensrcDir = Util.pathToFile(options.getGenSrcDir());
137 headerDir = Util.pathToFile(options.getHeaderDir());
138 stateDir = Util.pathToFile(options.getStateDir());
139 javacState = new File(stateDir, "javac_state");
140 if (removeJavacState && javacState.exists()) {
141 javacState.delete();
142 }
143 newJavacState = false;
144 if (!javacState.exists()) {
145 newJavacState = true;
146 // If there is no javac_state then delete the contents of all the artifact dirs!
147 // We do not want to risk building a broken incremental build.
148 // BUT since the makefiles still copy things straight into the bin_dir et al,
149 // we avoid deleting files here, if the option --permit-unidentified-classes was supplied.
150 if (!options.isUnidentifiedArtifactPermitted()) {
151 deleteContents(binDir);
152 deleteContents(gensrcDir);
153 deleteContents(headerDir);
154 }
155 needsSaving = true;
156 }
157 prev = new BuildState();
158 now = new BuildState();
159 taintedPackages = new HashSet<>();
160 recompiledPackages = new HashSet<>();
161 packagesWithChangedPublicApis = new HashSet<>();
162 }
163
164 public BuildState prev() { return prev; }
165 public BuildState now() { return now; }
166
167 /**
168 * Remove args not affecting the state.
169 */
170 static String[] removeArgsNotAffectingState(String[] args) {
171 String[] out = new String[args.length];
172 int j = 0;
173 for (int i = 0; i<args.length; ++i) {
174 if (args[i].equals("-j")) {
175 // Just skip it and skip following value
176 i++;
177 } else if (args[i].startsWith("--server:")) {
178 // Just skip it.
179 } else if (args[i].startsWith("--log=")) {
180 // Just skip it.
250 if (f.exists() && f.getName().endsWith(".class")) {
251 f.delete();
252 }
253 }
254 }
255 }
256
257 /**
258 * Mark the javac_state file to be in need of saving and as a side effect,
259 * it gets a new timestamp.
260 */
261 private void needsSaving() {
262 needsSaving = true;
263 }
264
265 /**
266 * Save the javac_state file.
267 */
268 public void save() throws IOException {
269 if (!needsSaving) return;
270 try (FileWriter out = new FileWriter(javacState)) {
271 StringBuilder b = new StringBuilder();
272 long millisNow = System.currentTimeMillis();
273 Date d = new Date(millisNow);
274 SimpleDateFormat df =
275 new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS");
276 b.append("# javac_state ver 0.3 generated "+millisNow+" "+df.format(d)+"\n");
277 b.append("# This format might change at any time. Please do not depend on it.\n");
278 b.append("# M module\n");
279 b.append("# P package\n");
280 b.append("# S C source_tobe_compiled timestamp\n");
281 b.append("# S L link_only_source timestamp\n");
282 b.append("# G C generated_source timestamp\n");
283 b.append("# A artifact timestamp\n");
284 b.append("# D dependency\n");
285 b.append("# I pubapi\n");
286 b.append("# R arguments\n");
287 b.append("R ").append(theArgs).append("\n");
288
289 // Copy over the javac_state for the packages that did not need recompilation.
290 now.copyPackagesExcept(prev, recompiledPackages, new HashSet<String>());
291 // Save the packages, ie package names, dependencies, pubapis and artifacts!
292 // I.e. the lot.
293 Module.saveModules(now.modules(), b);
294
295 String s = b.toString();
296 out.write(s, 0, s.length());
297 }
298 }
299
300 /**
301 * Load a javac_state file.
302 */
303 public static JavacState load(Options options, PrintStream out, PrintStream err) {
304 JavacState db = new JavacState(options, false, out, err);
305 Module lastModule = null;
306 Package lastPackage = null;
307 Source lastSource = null;
308 boolean noFileFound = false;
309 boolean foundCorrectVerNr = false;
310 boolean newCommandLine = false;
311 boolean syntaxError = false;
312
313 try (BufferedReader in = new BufferedReader(new FileReader(db.javacState))) {
314 for (;;) {
315 String l = in.readLine();
316 if (l==null) break;
317 if (l.length()>=3 && l.charAt(1) == ' ') {
318 char c = l.charAt(0);
319 if (c == 'M') {
320 lastModule = db.prev.loadModule(l);
321 } else
322 if (c == 'P') {
323 if (lastModule == null) { syntaxError = true; break; }
324 lastPackage = db.prev.loadPackage(lastModule, l);
325 } else
326 if (c == 'D') {
327 if (lastModule == null || lastPackage == null) { syntaxError = true; break; }
328 lastPackage.loadDependency(l);
329 } else
330 if (c == 'I') {
331 if (lastModule == null || lastPackage == null) { syntaxError = true; break; }
332 lastPackage.loadPubapi(l);
333 } else
334 if (c == 'A') {
335 if (lastModule == null || lastPackage == null) { syntaxError = true; break; }
336 lastPackage.loadArtifact(l);
337 } else
338 if (c == 'S') {
339 if (lastModule == null || lastPackage == null) { syntaxError = true; break; }
340 lastSource = db.prev.loadSource(lastPackage, l, false);
341 } else
342 if (c == 'G') {
343 if (lastModule == null || lastPackage == null) { syntaxError = true; break; }
344 lastSource = db.prev.loadSource(lastPackage, l, true);
345 } else
346 if (c == 'R') {
347 String ncmdl = "R "+db.theArgs;
348 if (!l.equals(ncmdl)) {
349 newCommandLine = true;
350 }
351 } else
352 if (c == '#') {
353 if (l.startsWith("# javac_state ver ")) {
354 int sp = l.indexOf(" ", 18);
355 if (sp != -1) {
356 String ver = l.substring(18,sp);
357 if (!ver.equals("0.3")) {
358 break;
359 }
360 foundCorrectVerNr = true;
361 }
362 }
363 }
364 }
365 }
366 } catch (FileNotFoundException e) {
367 // Silently create a new javac_state file.
368 noFileFound = true;
369 } catch (IOException e) {
370 Log.info("Dropping old javac_state because of errors when reading it.");
371 db = new JavacState(options, true, out, err);
372 foundCorrectVerNr = true;
373 newCommandLine = false;
374 syntaxError = false;
375 }
376 if (foundCorrectVerNr == false && !noFileFound) {
377 Log.info("Dropping old javac_state since it is of an old version.");
391
392 /**
393 * Mark a java package as tainted, ie it needs recompilation.
394 */
395 public void taintPackage(String name, String because) {
396 if (!taintedPackages.contains(name)) {
397 if (because != null) Log.debug("Tainting "+Util.justPackageName(name)+" because "+because);
398 // It has not been tainted before.
399 taintedPackages.add(name);
400 needsSaving();
401 Package nowp = now.packages().get(name);
402 if (nowp != null) {
403 for (String d : nowp.dependents()) {
404 taintPackage(d, because);
405 }
406 }
407 }
408 }
409
410 /**
411 * This packages need recompilation.
412 */
413 public Set<String> taintedPackages() {
414 return taintedPackages;
415 }
416
417 /**
418 * Clean out the tainted package set, used after the first round of compiles,
419 * prior to propagating dependencies.
420 */
421 public void clearTaintedPackages() {
422 taintedPackages = new HashSet<>();
423 }
424
425 /**
426 * Go through all sources and check which have been removed, added or modified
427 * and taint the corresponding packages.
428 */
429 public void checkSourceStatus(boolean check_gensrc) {
430 removedSources = calculateRemovedSources();
431 for (Source s : removedSources) {
480 }
481 }
482 }
483 }
484
485 /**
486 * Propagate recompilation through the dependency chains.
487 * Avoid re-tainting packages that have already been compiled.
488 */
489 public void taintPackagesDependingOnChangedPackages(Set<String> pkgs, Set<String> recentlyCompiled) {
490 for (Package pkg : prev.packages().values()) {
491 for (String dep : pkg.dependencies()) {
492 if (pkgs.contains(dep) && !recentlyCompiled.contains(pkg.name())) {
493 taintPackage(pkg.name(), " its depending on "+dep);
494 }
495 }
496 }
497 }
498
499 /**
500 * Scan all output dirs for artifacts and remove those files (artifacts?)
501 * that are not recognized as such, in the javac_state file.
502 */
503 public void removeUnidentifiedArtifacts() {
504 Set<File> allKnownArtifacts = new HashSet<>();
505 for (Package pkg : prev.packages().values()) {
506 for (File f : pkg.artifacts().values()) {
507 allKnownArtifacts.add(f);
508 }
509 }
510 // Do not forget about javac_state....
511 allKnownArtifacts.add(javacState);
512
513 for (File f : binArtifacts) {
514 if (!allKnownArtifacts.contains(f)) {
515 Log.debug("Removing "+f.getPath()+" since it is unknown to the javac_state.");
516 f.delete();
517 }
518 }
519 for (File f : headerArtifacts) {
692 boolean rc = true;
693 // Group sources based on transforms. A source file can only belong to a single transform.
694 Map<Transformer,Map<String,Set<URI>>> groupedSources = new HashMap<>();
695 for (Source src : now.sources().values()) {
696 Transformer t = suffixRules.get(src.suffix());
697 if (t != null) {
698 if (taintedPackages.contains(src.pkg().name()) && !src.isLinkedOnly()) {
699 addFileToTransform(groupedSources, t, src);
700 }
701 }
702 }
703 // Go through the transforms and transform them.
704 for (Map.Entry<Transformer,Map<String,Set<URI>>> e : groupedSources.entrySet()) {
705 Transformer t = e.getKey();
706 Map<String,Set<URI>> srcs = e.getValue();
707 // These maps need to be synchronized since multiple threads will be writing results into them.
708 Map<String,Set<URI>> packageArtifacts =
709 Collections.synchronizedMap(new HashMap<String,Set<URI>>());
710 Map<String,Set<String>> packageDependencies =
711 Collections.synchronizedMap(new HashMap<String,Set<String>>());
712 Map<String,String> packagePublicApis =
713 Collections.synchronizedMap(new HashMap<String, String>());
714
715 boolean r = t.transform(javacService,
716 srcs,
717 visibleSrcs,
718 visibleClasses,
719 prev.dependents(),
720 outputDir.toURI(),
721 packageArtifacts,
722 packageDependencies,
723 packagePublicApis,
724 0,
725 isIncremental(),
726 numCores,
727 out,
728 err);
729 if (!r) rc = false;
730
731 for (String p : srcs.keySet()) {
732 recompiledPackages.add(p);
733 }
734 // The transform is done! Extract all the artifacts and store the info into the Package objects.
735 for (Map.Entry<String,Set<URI>> a : packageArtifacts.entrySet()) {
736 Module mnow = now.findModuleFromPackageName(a.getKey());
737 mnow.addArtifacts(a.getKey(), a.getValue());
738 }
739 // Extract all the dependencies and store the info into the Package objects.
740 for (Map.Entry<String,Set<String>> a : packageDependencies.entrySet()) {
741 Set<String> deps = a.getValue();
742 Module mnow = now.findModuleFromPackageName(a.getKey());
743 mnow.setDependencies(a.getKey(), deps);
744 }
745 // Extract all the pubapis and store the info into the Package objects.
746 for (Map.Entry<String,String> a : packagePublicApis.entrySet()) {
747 Module mprev = prev.findModuleFromPackageName(a.getKey());
748 List<String> pubapi = Package.pubapiToList(a.getValue());
749 Module mnow = now.findModuleFromPackageName(a.getKey());
750 mnow.setPubapi(a.getKey(), pubapi);
751 if (mprev.hasPubapiChanged(a.getKey(), pubapi)) {
752 // Aha! The pubapi of this package has changed!
753 // It can also be a new compile from scratch.
754 if (mprev.lookupPackage(a.getKey()).existsInJavacState()) {
755 // This is an incremental compile! The pubapi
756 // did change. Trigger recompilation of dependents.
757 packagesWithChangedPublicApis.add(a.getKey());
758 Log.info("The pubapi of "+Util.justPackageName(a.getKey())+" has changed!");
759 }
760 }
761 }
762 }
763 return rc;
764 }
765
766 /**
767 * Utility method to recursively find all files below a directory.
768 */
769 private static Set<File> findAllFiles(File dir) {
770 Set<File> foundFiles = new HashSet<>();
771 if (dir == null) {
784 }
785 }
786 }
787
788 /**
789 * Compare the calculate source list, with an explicit list, usually supplied from the makefile.
790 * Used to detect bugs where the makefile and sjavac have different opinions on which files
791 * should be compiled.
792 */
793 public void compareWithMakefileList(File makefileSourceList)
794 throws ProblemException
795 {
796 // If we are building on win32 using for example cygwin the paths in the makefile source list
797 // might be /cygdrive/c/.... which does not match c:\....
798 // We need to adjust our calculated sources to be identical, if necessary.
799 boolean mightNeedRewriting = File.pathSeparatorChar == ';';
800
801 if (makefileSourceList == null) return;
802
803 Set<String> calculatedSources = new HashSet<>();
804 Set<String> listedSources = new HashSet<>();
805
806 // Create a set of filenames with full paths.
807 for (Source s : now.sources().values()) {
808 // Don't include link only sources when comparing sources to compile
809 if (!s.isLinkedOnly()) {
810 String path = s.file().getPath();
811 if (mightNeedRewriting)
812 path = Util.normalizeDriveLetter(path);
813 calculatedSources.add(path);
814 }
815 }
816 // Read in the file and create another set of filenames with full paths.
817 try {
818 BufferedReader in = new BufferedReader(new FileReader(makefileSourceList));
819 for (;;) {
820 String l = in.readLine();
821 if (l==null) break;
822 l = l.trim();
823 if (mightNeedRewriting) {
824 if (l.indexOf(":") == 1 && l.indexOf("\\") == 2) {
825 // Everything a-ok, the format is already C:\foo\bar
826 } else if (l.indexOf(":") == 1 && l.indexOf("/") == 2) {
827 // The format is C:/foo/bar, rewrite into the above format.
828 l = l.replaceAll("/","\\\\");
829 } else if (l.charAt(0) == '/' && l.indexOf("/",1) != -1) {
830 // The format might be: /cygdrive/c/foo/bar, rewrite into the above format.
831 // Do not hardcode the name cygdrive here.
832 int slash = l.indexOf("/",1);
833 l = l.replaceAll("/","\\\\");
834 l = ""+l.charAt(slash+1)+":"+l.substring(slash+2);
835 }
836 if (Character.isLowerCase(l.charAt(0))) {
837 l = Character.toUpperCase(l.charAt(0))+l.substring(1);
838 }
839 }
840 listedSources.add(l);
841 }
842 } catch (FileNotFoundException e) {
843 throw new ProblemException("Could not open "+makefileSourceList.getPath()+" since it does not exist!");
844 } catch (IOException e) {
845 throw new ProblemException("Could not read "+makefileSourceList.getPath());
846 }
847
848 for (String s : listedSources) {
849 if (!calculatedSources.contains(s)) {
850 throw new ProblemException("The makefile listed source "+s+" was not calculated by the smart javac wrapper!");
851 }
852 }
853
854 for (String s : calculatedSources) {
855 if (!listedSources.contains(s)) {
856 throw new ProblemException("The smart javac wrapper calculated source "+s+" was not listed by the makefiles!");
857 }
858 }
859 }
860 }
|
79 // * artifact belonging to the package is lost, or its timestamp has been changed.
80 // * an unknown artifact has appeared, we simply delete it, but we also trigger a recompilation.
81 // * a package that is tainted, taints all packages that depend on it.
82 private Set<String> taintedPackages;
83 // After a compile, the pubapis are compared with the pubapis stored in the javac state file.
84 // Any packages where the pubapi differ are added to this set.
85 // Later we use this set and the dependency information to taint dependent packages.
86 private Set<String> packagesWithChangedPublicApis;
87 // When a module-info.java file is changed, taint the module,
88 // then taint all modules that depend on that that module.
89 // A module dependency can occur directly through a require, or
90 // indirectly through a module that does a public export for the first tainted module.
91 // When all modules are tainted, then taint all packages belonging to these modules.
92 // Then rebuild. It is perhaps possible (and valuable?) to do a more finegrained examination of the
93 // change in module-info.java, but that will have to wait.
94 private Set<String> taintedModules;
95 // The set of all packages that has been recompiled.
96 // Copy over the javac_state for the packages that did not need recompilation,
97 // verbatim from the previous (prev) to the new (now) build state.
98 private Set<String> recompiledPackages;
99 // The set of all classpath packages and their classes,
100 // for which either the timestamps or the pubapi have changed.
101 private Map<String,Set<String>> changedClasspathPackages;
102
103 // The output directories filled with tasty artifacts.
104 private File binDir, gensrcDir, headerDir, stateDir;
105
106 // The current status of the file system.
107 private Set<File> binArtifacts;
108 private Set<File> gensrcArtifacts;
109 private Set<File> headerArtifacts;
110
111 // The status of the sources.
112 Set<Source> removedSources = null;
113 Set<Source> addedSources = null;
114 Set<Source> modifiedSources = null;
115
116 // Visible sources for linking. These are the only
117 // ones that -sourcepath is allowed to see.
118 Set<URI> visibleSrcs;
119
120 // Visible classes for linking. These are the only
121 // ones that -classpath is allowed to see.
122 // It maps from a classpath root to the set of visible classes for that root.
123 // If the set is empty, then all classes are visible for that root.
124 // It can also map from a jar file to the set of visible classes for that jar file.
125 Map<URI,Set<String>> visibleClasses;
126
127 // Setup transform that always exist.
128 private CompileJavaPackages compileJavaPackages = new CompileJavaPackages();
129
130 // Where to send stdout and stderr.
131 private PrintStream out, err;
132
133 private Options options;
134
135 JavacState(Options ops, boolean removeJavacState, PrintStream o, PrintStream e) {
136 options = ops;
137 out = o;
138 err = e;
139 numCores = options.getNumCores();
140 theArgs = options.getStateArgsString();
141 binDir = Util.pathToFile(options.getDestDir());
142 gensrcDir = Util.pathToFile(options.getGenSrcDir());
143 headerDir = Util.pathToFile(options.getHeaderDir());
144 stateDir = Util.pathToFile(options.getStateDir());
145 javacState = new File(stateDir, "javac_state");
146 if (removeJavacState && javacState.exists()) {
147 javacState.delete();
148 }
149 newJavacState = false;
150 if (!javacState.exists()) {
151 newJavacState = true;
152 // If there is no javac_state then delete the contents of all the artifact dirs!
153 // We do not want to risk building a broken incremental build.
154 // BUT since the makefiles still copy things straight into the bin_dir et al,
155 // we avoid deleting files here, if the option --permit-unidentified-classes was supplied.
156 if (!options.isUnidentifiedArtifactPermitted()) {
157 deleteContents(binDir);
158 deleteContents(gensrcDir);
159 deleteContents(headerDir);
160 }
161 needsSaving = true;
162 }
163 prev = new BuildState();
164 now = new BuildState();
165 taintedPackages = new HashSet<>();
166 recompiledPackages = new HashSet<>();
167 changedClasspathPackages = new HashMap<>();
168 packagesWithChangedPublicApis = new HashSet<>();
169 }
170
171 public BuildState prev() { return prev; }
172 public BuildState now() { return now; }
173
174 /**
175 * Remove args not affecting the state.
176 */
177 static String[] removeArgsNotAffectingState(String[] args) {
178 String[] out = new String[args.length];
179 int j = 0;
180 for (int i = 0; i<args.length; ++i) {
181 if (args[i].equals("-j")) {
182 // Just skip it and skip following value
183 i++;
184 } else if (args[i].startsWith("--server:")) {
185 // Just skip it.
186 } else if (args[i].startsWith("--log=")) {
187 // Just skip it.
257 if (f.exists() && f.getName().endsWith(".class")) {
258 f.delete();
259 }
260 }
261 }
262 }
263
264 /**
265 * Mark the javac_state file to be in need of saving and as a side effect,
266 * it gets a new timestamp.
267 */
268 private void needsSaving() {
269 needsSaving = true;
270 }
271
272 /**
273 * Save the javac_state file.
274 */
275 public void save() throws IOException {
276 if (!needsSaving) return;
277 Log.debug("Saving the javac_state file.");
278 try (FileWriter out = new FileWriter(javacState)) {
279 StringBuilder b = new StringBuilder();
280 long millisNow = System.currentTimeMillis();
281 Date d = new Date(millisNow);
282 SimpleDateFormat df =
283 new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS");
284 b.append("# javac_state ver 0.4 generated "+millisNow+" "+df.format(d)+"\n");
285 b.append("# This format might change at any time. Please do not depend on it.\n");
286 b.append("# M module\n");
287 b.append("# P package\n");
288 b.append("# S C source_tobe_compiled timestamp\n");
289 b.append("# S L link_only_source timestamp\n");
290 b.append("# G C generated_source timestamp\n");
291 b.append("# A artifact timestamp\n");
292 b.append("# D dependency\n");
293 b.append("# I C pubapi when compiled from source\n");
294 // The pubapi of compiled source is extracted almost for free when
295 // the compilation is done.
296 //
297 // Should we have pubapi when linked from source?
298 // No, because a linked source might not be entirely compiled because of
299 // performance reasons, thus the full pubapi might not be available for free.
300 // Instead, monitor the timestamp of linked sources, when the timestamp change
301 // always force a recompile of dependents even though it might not be necessary.
302 b.append("# I Z pubapi when linked as classes\n");
303 // The pubapi of linked classes can easily be constructed from the referenced classes.
304 // However this pubapi contains only a subset of the classes actually public in the package.
305 // Because: 1) we cannot easily find all classes 2) we do not want to, we are satisfied in
306 // only tracking the actually referred classes.
307 b.append("# Z archive timestamp\n");
308 // When referred classes are stored in a jar/zip, use this timestamp to shortcut
309 // and avoid testing all internal classes in the jar, if the timestamp of the jar itself
310 // is unchanged.
311 b.append("# R arguments\n");
312 b.append("R ").append(theArgs).append("\n");
313
314 // Copy over the javac_state for the packages that did not need recompilation.
315 now.copyPackagesExcept(prev, recompiledPackages, new HashSet<String>());
316 // Recreate pubapi:s and timestamps for classpath packages that have changed.
317 addToClasspathPubapis(changedClasspathPackages);
318 // Save the packages, ie package names, dependencies, pubapis and artifacts! I.e. the lot.
319 Module.saveModules(now.modules(), b);
320 // Save the archive timestamps.
321 now.saveArchiveTimestamps(b);
322
323 String s = b.toString();
324 out.write(s, 0, s.length());
325 }
326 }
327
328 /**
329 * Load a javac_state file.
330 */
331 public static JavacState load(Options options, PrintStream out, PrintStream err) {
332 JavacState db = new JavacState(options, false, out, err);
333 Module lastModule = null;
334 Package lastPackage = null;
335 Source lastSource = null;
336 boolean noFileFound = false;
337 boolean foundCorrectVerNr = false;
338 boolean newCommandLine = false;
339 boolean syntaxError = false;
340
341 try (BufferedReader in = new BufferedReader(new FileReader(db.javacState))) {
342 for (;;) {
343 String l = in.readLine();
344 if (l==null) break;
345 if (l.length()>=3 && l.charAt(1) == ' ') {
346 char c = l.charAt(0);
347 if (c == 'M') {
348 lastModule = db.prev.loadModule(l);
349 } else
350 if (c == 'P') {
351 if (lastModule == null) { syntaxError = true; break; }
352 lastPackage = db.prev.loadPackage(lastModule, l);
353 } else
354 if (c == 'D') {
355 if (lastModule == null || lastPackage == null) { syntaxError = true; break; }
356 lastPackage.loadDependency(l);
357 } else
358 if (c == 'I') {
359 if (lastModule == null || lastPackage == null) { syntaxError = true; break; }
360 lastPackage.loadPubapi(l);
361 } else
362 if (c == 'Z') {
363 db.prev.loadArchiveTimestamp(l);
364 } else
365 if (c == 'A') {
366 if (lastModule == null || lastPackage == null) { syntaxError = true; break; }
367 lastPackage.loadArtifact(l);
368 } else
369 if (c == 'S') {
370 if (lastModule == null || lastPackage == null) { syntaxError = true; break; }
371 lastSource = db.prev.loadSource(lastPackage, l, false);
372 } else
373 if (c == 'G') {
374 if (lastModule == null || lastPackage == null) { syntaxError = true; break; }
375 lastSource = db.prev.loadSource(lastPackage, l, true);
376 } else
377 if (c == 'R') {
378 String ncmdl = "R "+db.theArgs;
379 if (!l.equals(ncmdl)) {
380 newCommandLine = true;
381 }
382 } else
383 if (c == '#') {
384 if (l.startsWith("# javac_state ver ")) {
385 int sp = l.indexOf(" ", 18);
386 if (sp != -1) {
387 String ver = l.substring(18,sp);
388 if (!ver.equals("0.4")) {
389 break;
390 }
391 foundCorrectVerNr = true;
392 }
393 }
394 }
395 }
396 }
397 } catch (FileNotFoundException e) {
398 // Silently create a new javac_state file.
399 noFileFound = true;
400 } catch (IOException e) {
401 Log.info("Dropping old javac_state because of errors when reading it.");
402 db = new JavacState(options, true, out, err);
403 foundCorrectVerNr = true;
404 newCommandLine = false;
405 syntaxError = false;
406 }
407 if (foundCorrectVerNr == false && !noFileFound) {
408 Log.info("Dropping old javac_state since it is of an old version.");
422
423 /**
424 * Mark a java package as tainted, ie it needs recompilation.
425 */
426 public void taintPackage(String name, String because) {
427 if (!taintedPackages.contains(name)) {
428 if (because != null) Log.debug("Tainting "+Util.justPackageName(name)+" because "+because);
429 // It has not been tainted before.
430 taintedPackages.add(name);
431 needsSaving();
432 Package nowp = now.packages().get(name);
433 if (nowp != null) {
434 for (String d : nowp.dependents()) {
435 taintPackage(d, because);
436 }
437 }
438 }
439 }
440
441 /**
442 * These packages need recompilation.
443 */
444 public Set<String> taintedPackages() {
445 return taintedPackages;
446 }
447
448 /**
449 * Clean out the tainted package set, used after the first round of compiles,
450 * prior to propagating dependencies.
451 */
452 public void clearTaintedPackages() {
453 taintedPackages = new HashSet<>();
454 }
455
456 /**
457 * Go through all sources and check which have been removed, added or modified
458 * and taint the corresponding packages.
459 */
460 public void checkSourceStatus(boolean check_gensrc) {
461 removedSources = calculateRemovedSources();
462 for (Source s : removedSources) {
511 }
512 }
513 }
514 }
515
516 /**
517 * Propagate recompilation through the dependency chains.
518 * Avoid re-tainting packages that have already been compiled.
519 */
520 public void taintPackagesDependingOnChangedPackages(Set<String> pkgs, Set<String> recentlyCompiled) {
521 for (Package pkg : prev.packages().values()) {
522 for (String dep : pkg.dependencies()) {
523 if (pkgs.contains(dep) && !recentlyCompiled.contains(pkg.name())) {
524 taintPackage(pkg.name(), " its depending on "+dep);
525 }
526 }
527 }
528 }
529
530 /**
531 * Compare the javac_state recorded public apis of packages on the classpath
532 * with the actual public apis on the classpath.
533 */
534 public void taintPackagesDependingOnChangedClasspathPackages() {
535 Compile comp = new Compile(options);
536 Set<String> tainteds = new HashSet<String>();
537
538 for (Package pkg : prev.packages().values()) {
539 List<String> current = new ArrayList<String>();
540 Iterator<String> i = current.iterator();
541 boolean tainted = false;
542 boolean skip = false;
543 for (String s : pkg.pubapiForLinkedClasses()) {
544 if (skip && !s.startsWith("PUBAPI ")) {
545 // Skipping, because timestamp or hash checked out ok. Assume no change here!
546 continue;
547 }
548 skip = false;
549 // Check if we have found a new class.
550 if (s.startsWith("PUBAPI ")) {
551 if (i.hasNext()) {
552 // Previous api had not ended! Change is detected!
553 tainted = true;
554 break;
555 }
556 // Extract the class name, hash, file and timestamp from the PUBAPI line
557 int p = s.indexOf(' ', 7);
558 int pp = s.indexOf(' ', p+1);
559 String cln = s.substring(7, p);
560 String hash = s.substring(p+1, pp);
561 String loc = s.substring(pp+1); // loc == file and timestamp
562 String archive = Util.extractArchive(loc);
563 if (archive != null && prev.archives().contains(archive)) {
564 // If it existed, then the timestamp for the archive
565 // is unchanged. Lets skip testing this class inside the archive!
566 Log.debug("Assume "+cln+" unchanged since "+archive+" is unchanged");
567 skip = true;
568 current = new ArrayList<String>();
569 i = current.iterator();
570 continue;
571 }
572 // The archive timestamp has changed, or is new.
573 // Compare the prev classLocInfo with the current classLocInfo
574 String cmp = comp.getClassLocInfo(cln);
575 if (cmp.equals(loc)) {
576 // Equal means that the come from the same class/zip file
577 // and the timestamp is the same. Assume equal!
578 Log.debug("Assume "+cln+" unchanged since "+loc+" is unchanged");
579 skip = true;
580 current = new ArrayList<String>();
581 i = current.iterator();
582 continue;
583 }
584 // The timestamps differ, lets check the pubapi.
585 Log.debug("Timestamp changed for "+cln+" now checking if pubapi is the same.");
586 // Add the package to changedClasspathPackages because this
587 // will trigger a regeneration the package information to javac_state
588 // thus updating the timestamps.
589 Util.addToMapSet(pkg.name(), cln, changedClasspathPackages);
590 needsSaving = true;
591 current = comp.getPubapi(cln, now.archives());
592 i = current.iterator();
593 }
594 if (i.hasNext()) {
595 String ss = i.next();
596 if (s.startsWith("PUBAPI ") && ss.startsWith("PUBAPI ")) {
597 int p = s.indexOf(' ', 7);
598 int pp = s.indexOf(' ', p+1);
599 s = s.substring(0, pp);
600 ss = ss.substring(0, pp);
601 if (s.equals(ss)) {
602 // The pubapi of a class has identical hash!
603 // We assume it is equals!
604 Log.debug("Assume "+s.substring(0, pp)+" unchanged since its hash is unchanged");
605 skip = true;
606 current = new ArrayList<String>();
607 i = current.iterator();
608 } else {
609 // The pubapi hash is not identical! Change is detected!
610 tainted = true;
611 }
612 }
613 }
614 }
615 if (tainted) {
616 Log.info("The pubapi of "+Util.justPackageName(pkg.name())+" has changed!");
617 tainteds.add(pkg.name());
618 } else if (pkg.pubapiForLinkedClasses().size() > 0) {
619 Log.debug("The pubapi of "+Util.justPackageName(pkg.name())+" was unchanged!");
620 }
621 }
622 taintPackagesDependingOnChangedPackages(tainteds, new HashSet<String>());
623 }
624
625 /**
626 * Scan all output dirs for artifacts and remove those files (artifacts?)
627 * that are not recognized as such, in the javac_state file.
628 */
629 public void removeUnidentifiedArtifacts() {
630 Set<File> allKnownArtifacts = new HashSet<>();
631 for (Package pkg : prev.packages().values()) {
632 for (File f : pkg.artifacts().values()) {
633 allKnownArtifacts.add(f);
634 }
635 }
636 // Do not forget about javac_state....
637 allKnownArtifacts.add(javacState);
638
639 for (File f : binArtifacts) {
640 if (!allKnownArtifacts.contains(f)) {
641 Log.debug("Removing "+f.getPath()+" since it is unknown to the javac_state.");
642 f.delete();
643 }
644 }
645 for (File f : headerArtifacts) {
818 boolean rc = true;
819 // Group sources based on transforms. A source file can only belong to a single transform.
820 Map<Transformer,Map<String,Set<URI>>> groupedSources = new HashMap<>();
821 for (Source src : now.sources().values()) {
822 Transformer t = suffixRules.get(src.suffix());
823 if (t != null) {
824 if (taintedPackages.contains(src.pkg().name()) && !src.isLinkedOnly()) {
825 addFileToTransform(groupedSources, t, src);
826 }
827 }
828 }
829 // Go through the transforms and transform them.
830 for (Map.Entry<Transformer,Map<String,Set<URI>>> e : groupedSources.entrySet()) {
831 Transformer t = e.getKey();
832 Map<String,Set<URI>> srcs = e.getValue();
833 // These maps need to be synchronized since multiple threads will be writing results into them.
834 Map<String,Set<URI>> packageArtifacts =
835 Collections.synchronizedMap(new HashMap<String,Set<URI>>());
836 Map<String,Set<String>> packageDependencies =
837 Collections.synchronizedMap(new HashMap<String,Set<String>>());
838 Map<String,List<String>> packagePublicApis =
839 Collections.synchronizedMap(new HashMap<String, List<String>>());
840 // Map from package name to set of classes. The classes are a subset of all classes
841 // within the package. The subset are those that our code has directly referenced.
842 Map<String,Set<String>> classpathPackageDependencies =
843 Collections.synchronizedMap(new HashMap<String, Set<String>>());
844
845 boolean r = t.transform(javacService,
846 srcs,
847 visibleSrcs,
848 visibleClasses,
849 prev.dependents(),
850 outputDir.toURI(),
851 packageArtifacts,
852 packageDependencies,
853 packagePublicApis,
854 classpathPackageDependencies,
855 0,
856 isIncremental(),
857 numCores,
858 out,
859 err);
860 if (!r) rc = false;
861
862 for (String p : srcs.keySet()) {
863 recompiledPackages.add(p);
864 }
865 // The transform is done! Extract all the artifacts and store the info into the Package objects.
866 for (Map.Entry<String,Set<URI>> a : packageArtifacts.entrySet()) {
867 Module mnow = now.findModuleFromPackageName(a.getKey());
868 mnow.addArtifacts(a.getKey(), a.getValue());
869 }
870 // Extract all the dependencies and store the info into the Package objects.
871 for (Map.Entry<String,Set<String>> a : packageDependencies.entrySet()) {
872 Set<String> deps = a.getValue();
873 Module mnow = now.findModuleFromPackageName(a.getKey());
874 mnow.setDependencies(a.getKey(), deps);
875 }
876 // With two threads compiling our sources, sources compiled by a second thread, might look like
877 // classpath dependencies to the first thread or vice versa. We cannot remove such fake classpath dependencies
878 // until the end of the compilation since the knowledge of what is compiled does not exist until now.
879 for (String pkg : packagePublicApis.keySet()) {
880 classpathPackageDependencies.remove(pkg);
881 }
882 // Also, if we doing an incremental compile, then references outside of the small recompiled set,
883 // will also look like classpath deps, lets remove them as well.
884 for (String pkg : prev.packages().keySet()) {
885 Package p = prev.packages().get(pkg);
886 if (p.pubapiForLinkedClasses().size() == 0) {
887 classpathPackageDependencies.remove(pkg);
888 }
889 }
890 // Extract all classpath package classes and store the public ap
891 // into the Package object.
892 addToClasspathPubapis(classpathPackageDependencies);
893
894 // Extract all the pubapis and store the info into the Package objects.
895 for (Map.Entry<String,List<String>> a : packagePublicApis.entrySet()) {
896 Module mprev = prev.findModuleFromPackageName(a.getKey());
897 List<String> pubapi = a.getValue();
898 Module mnow = now.findModuleFromPackageName(a.getKey());
899 mnow.setPubapiForCompiledSources(a.getKey(), pubapi);
900 if (mprev.hasPubapiForCompiledSourcesChanged(a.getKey(), pubapi)) {
901 // Aha! The pubapi of this package has changed!
902 // It can also be a new compile from scratch.
903 if (mprev.lookupPackage(a.getKey()).existsInJavacState()) {
904 // This is an incremental compile! The pubapi
905 // did change. Trigger recompilation of dependents.
906 packagesWithChangedPublicApis.add(a.getKey());
907 Log.info("The pubapi of "+Util.justPackageName(a.getKey())+" has changed!");
908 }
909 }
910 }
911 }
912 return rc;
913 }
914
915 /**
916 * Utility method to recursively find all files below a directory.
917 */
918 private static Set<File> findAllFiles(File dir) {
919 Set<File> foundFiles = new HashSet<>();
920 if (dir == null) {
933 }
934 }
935 }
936
937 /**
938 * Compare the calculate source list, with an explicit list, usually supplied from the makefile.
939 * Used to detect bugs where the makefile and sjavac have different opinions on which files
940 * should be compiled.
941 */
942 public void compareWithMakefileList(File makefileSourceList)
943 throws ProblemException
944 {
945 // If we are building on win32 using for example cygwin the paths in the makefile source list
946 // might be /cygdrive/c/.... which does not match c:\....
947 // We need to adjust our calculated sources to be identical, if necessary.
948 boolean mightNeedRewriting = File.pathSeparatorChar == ';';
949
950 if (makefileSourceList == null) return;
951
952 Set<String> calculatedSources = new HashSet<>();
953 Set<String> listedSources = SourceLocation.loadList(makefileSourceList);
954
955 // Create a set of filenames with full paths.
956 for (Source s : now.sources().values()) {
957 // Don't include link only sources when comparing sources to compile
958 if (!s.isLinkedOnly()) {
959 String path = s.file().getPath();
960 if (mightNeedRewriting)
961 path = Util.normalizeDriveLetter(path);
962 calculatedSources.add(path);
963 }
964 }
965
966
967 for (String s : listedSources) {
968 if (!calculatedSources.contains(s)) {
969 throw new ProblemException("The makefile listed source "+s+" was not calculated by the smart javac wrapper!");
970 }
971 }
972
973 for (String s : calculatedSources) {
974 if (!listedSources.contains(s)) {
975 throw new ProblemException("The smart javac wrapper calculated source "+s+" was not listed by the makefiles!");
976 }
977 }
978 }
979
980 /**
981 * Add the classes in deps, to the pubapis of the Packages.
982 * The pubapis are stored within the corresponding Package in now.
983 */
984 public void addToClasspathPubapis(Map<String, Set<String>> deps) {
985 Compile comp = new Compile(options);
986 // Extract all the pubapis of the classes inside deps and
987 // store the info into the corresponding Package objects.
988 for (Map.Entry<String,Set<String>> a : deps.entrySet()) {
989 String pkg = a.getKey();
990 Module mnow = now.findModuleFromPackageName(pkg);
991 Set<String> classes = new HashSet<>();
992 classes.addAll(a.getValue());
993 classes.addAll(mnow.lookupPackage(pkg).getClassesFromClasspathPubapi());
994 List<String> sorted_classes = new ArrayList<>();
995 for (String key : classes) {
996 sorted_classes.add(key);
997 }
998 Collections.sort(sorted_classes);
999
1000 List<String> pubapis = new ArrayList<>();
1001 for (String s : sorted_classes) {
1002 pubapis.addAll(comp.getPubapi(s, now.archives()));
1003 }
1004 mnow.setPubapiForLinkedClasses(a.getKey(), pubapis);
1005 }
1006 }
1007 }
|