--- old/src/share/classes/com/sun/tools/sjavac/JavacState.java 2014-08-09 00:28:45.307927999 +0200 +++ new/src/share/classes/com/sun/tools/sjavac/JavacState.java 2014-08-09 00:28:45.175931831 +0200 @@ -96,6 +96,9 @@ // Copy over the javac_state for the packages that did not need recompilation, // verbatim from the previous (prev) to the new (now) build state. private Set recompiledPackages; + // The set of all classpath packages and their classes, + // for which either the timestamps or the pubapi have changed. + private Map> changedClasspathPackages; // The output directories filled with tasty artifacts. private File binDir, gensrcDir, headerDir, stateDir; @@ -127,7 +130,10 @@ // Where to send stdout and stderr. private PrintStream out, err; - JavacState(Options options, boolean removeJavacState, PrintStream o, PrintStream e) { + private Options options; + + JavacState(Options ops, boolean removeJavacState, PrintStream o, PrintStream e) { + options = ops; out = o; err = e; numCores = options.getNumCores(); @@ -158,6 +164,7 @@ now = new BuildState(); taintedPackages = new HashSet<>(); recompiledPackages = new HashSet<>(); + changedClasspathPackages = new HashMap<>(); packagesWithChangedPublicApis = new HashSet<>(); } @@ -267,13 +274,14 @@ */ public void save() throws IOException { if (!needsSaving) return; + Log.debug("Saving the javac_state file."); try (FileWriter out = new FileWriter(javacState)) { StringBuilder b = new StringBuilder(); long millisNow = System.currentTimeMillis(); Date d = new Date(millisNow); SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss SSS"); - b.append("# javac_state ver 0.3 generated "+millisNow+" "+df.format(d)+"\n"); + b.append("# javac_state ver 0.4 generated "+millisNow+" "+df.format(d)+"\n"); b.append("# This format might change at any time. Please do not depend on it.\n"); b.append("# M module\n"); b.append("# P package\n"); @@ -282,15 +290,35 @@ b.append("# G C generated_source timestamp\n"); b.append("# A artifact timestamp\n"); b.append("# D dependency\n"); - b.append("# I pubapi\n"); + b.append("# I C pubapi when compiled from source\n"); + // The pubapi of compiled source is extracted almost for free when + // the compilation is done. + // + // Should we have pubapi when linked from source? + // No, because a linked source might not be entirely compiled because of + // performance reasons, thus the full pubapi might not be available for free. + // Instead, monitor the timestamp of linked sources, when the timestamp change + // always force a recompile of dependents even though it might not be necessary. + b.append("# I Z pubapi when linked as classes\n"); + // The pubapi of linked classes can easily be constructed from the referenced classes. + // However this pubapi contains only a subset of the classes actually public in the package. + // Because: 1) we cannot easily find all classes 2) we do not want to, we are satisfied in + // only tracking the actually referred classes. + b.append("# Z archive timestamp\n"); + // When referred classes are stored in a jar/zip, use this timestamp to shortcut + // and avoid testing all internal classes in the jar, if the timestamp of the jar itself + // is unchanged. b.append("# R arguments\n"); b.append("R ").append(theArgs).append("\n"); // Copy over the javac_state for the packages that did not need recompilation. now.copyPackagesExcept(prev, recompiledPackages, new HashSet()); - // Save the packages, ie package names, dependencies, pubapis and artifacts! - // I.e. the lot. + // Recreate pubapi:s and timestamps for classpath packages that have changed. + addToClasspathPubapis(changedClasspathPackages); + // Save the packages, ie package names, dependencies, pubapis and artifacts! I.e. the lot. Module.saveModules(now.modules(), b); + // Save the archive timestamps. + now.saveArchiveTimestamps(b); String s = b.toString(); out.write(s, 0, s.length()); @@ -331,6 +359,9 @@ if (lastModule == null || lastPackage == null) { syntaxError = true; break; } lastPackage.loadPubapi(l); } else + if (c == 'Z') { + db.prev.loadArchiveTimestamp(l); + } else if (c == 'A') { if (lastModule == null || lastPackage == null) { syntaxError = true; break; } lastPackage.loadArtifact(l); @@ -354,7 +385,7 @@ int sp = l.indexOf(" ", 18); if (sp != -1) { String ver = l.substring(18,sp); - if (!ver.equals("0.3")) { + if (!ver.equals("0.4")) { break; } foundCorrectVerNr = true; @@ -408,7 +439,7 @@ } /** - * This packages need recompilation. + * These packages need recompilation. */ public Set taintedPackages() { return taintedPackages; @@ -497,6 +528,101 @@ } /** + * Compare the javac_state recorded public apis of packages on the classpath + * with the actual public apis on the classpath. + */ + public void taintPackagesDependingOnChangedClasspathPackages() { + Compile comp = new Compile(options); + Set tainteds = new HashSet(); + + for (Package pkg : prev.packages().values()) { + List current = new ArrayList(); + Iterator i = current.iterator(); + boolean tainted = false; + boolean skip = false; + for (String s : pkg.pubapiForLinkedClasses()) { + if (skip && !s.startsWith("PUBAPI ")) { + // Skipping, because timestamp or hash checked out ok. Assume no change here! + continue; + } + skip = false; + // Check if we have found a new class. + if (s.startsWith("PUBAPI ")) { + if (i.hasNext()) { + // Previous api had not ended! Change is detected! + tainted = true; + break; + } + // Extract the class name, hash, file and timestamp from the PUBAPI line + int p = s.indexOf(' ', 7); + int pp = s.indexOf(' ', p+1); + String cln = s.substring(7, p); + String hash = s.substring(p+1, pp); + String loc = s.substring(pp+1); // loc == file and timestamp + String archive = Util.extractArchive(loc); + if (archive != null && prev.archives().contains(archive)) { + // If it existed, then the timestamp for the archive + // is unchanged. Lets skip testing this class inside the archive! + Log.debug("Assume "+cln+" unchanged since "+archive+" is unchanged"); + skip = true; + current = new ArrayList(); + i = current.iterator(); + continue; + } + // The archive timestamp has changed, or is new. + // Compare the prev classLocInfo with the current classLocInfo + String cmp = comp.getClassLocInfo(cln); + if (cmp.equals(loc)) { + // Equal means that the come from the same class/zip file + // and the timestamp is the same. Assume equal! + Log.debug("Assume "+cln+" unchanged since "+loc+" is unchanged"); + skip = true; + current = new ArrayList(); + i = current.iterator(); + continue; + } + // The timestamps differ, lets check the pubapi. + Log.debug("Timestamp changed for "+cln+" now checking if pubapi is the same."); + // Add the package to changedClasspathPackages because this + // will trigger a regeneration the package information to javac_state + // thus updating the timestamps. + Util.addToMapSet(pkg.name(), cln, changedClasspathPackages); + needsSaving = true; + current = comp.getPubapi(cln, now.archives()); + i = current.iterator(); + } + if (i.hasNext()) { + String ss = i.next(); + if (s.startsWith("PUBAPI ") && ss.startsWith("PUBAPI ")) { + int p = s.indexOf(' ', 7); + int pp = s.indexOf(' ', p+1); + s = s.substring(0, pp); + ss = ss.substring(0, pp); + if (s.equals(ss)) { + // The pubapi of a class has identical hash! + // We assume it is equals! + Log.debug("Assume "+s.substring(0, pp)+" unchanged since its hash is unchanged"); + skip = true; + current = new ArrayList(); + i = current.iterator(); + } else { + // The pubapi hash is not identical! Change is detected! + tainted = true; + } + } + } + } + if (tainted) { + Log.info("The pubapi of "+Util.justPackageName(pkg.name())+" has changed!"); + tainteds.add(pkg.name()); + } else if (pkg.pubapiForLinkedClasses().size() > 0) { + Log.debug("The pubapi of "+Util.justPackageName(pkg.name())+" was unchanged!"); + } + } + taintPackagesDependingOnChangedPackages(tainteds, new HashSet()); + } + + /** * Scan all output dirs for artifacts and remove those files (artifacts?) * that are not recognized as such, in the javac_state file. */ @@ -709,8 +835,12 @@ Collections.synchronizedMap(new HashMap>()); Map> packageDependencies = Collections.synchronizedMap(new HashMap>()); - Map packagePublicApis = - Collections.synchronizedMap(new HashMap()); + Map> packagePublicApis = + Collections.synchronizedMap(new HashMap>()); + // Map from package name to set of classes. The classes are a subset of all classes + // within the package. The subset are those that our code has directly referenced. + Map> classpathPackageDependencies = + Collections.synchronizedMap(new HashMap>()); boolean r = t.transform(javacService, srcs, @@ -721,6 +851,7 @@ packageArtifacts, packageDependencies, packagePublicApis, + classpathPackageDependencies, 0, isIncremental(), numCores, @@ -742,13 +873,31 @@ Module mnow = now.findModuleFromPackageName(a.getKey()); mnow.setDependencies(a.getKey(), deps); } + // With two threads compiling our sources, sources compiled by a second thread, might look like + // classpath dependencies to the first thread or vice versa. We cannot remove such fake classpath dependencies + // until the end of the compilation since the knowledge of what is compiled does not exist until now. + for (String pkg : packagePublicApis.keySet()) { + classpathPackageDependencies.remove(pkg); + } + // Also, if we doing an incremental compile, then references outside of the small recompiled set, + // will also look like classpath deps, lets remove them as well. + for (String pkg : prev.packages().keySet()) { + Package p = prev.packages().get(pkg); + if (p.pubapiForLinkedClasses().size() == 0) { + classpathPackageDependencies.remove(pkg); + } + } + // Extract all classpath package classes and store the public ap + // into the Package object. + addToClasspathPubapis(classpathPackageDependencies); + // Extract all the pubapis and store the info into the Package objects. - for (Map.Entry a : packagePublicApis.entrySet()) { + for (Map.Entry> a : packagePublicApis.entrySet()) { Module mprev = prev.findModuleFromPackageName(a.getKey()); - List pubapi = Package.pubapiToList(a.getValue()); + List pubapi = a.getValue(); Module mnow = now.findModuleFromPackageName(a.getKey()); - mnow.setPubapi(a.getKey(), pubapi); - if (mprev.hasPubapiChanged(a.getKey(), pubapi)) { + mnow.setPubapiForCompiledSources(a.getKey(), pubapi); + if (mprev.hasPubapiForCompiledSourcesChanged(a.getKey(), pubapi)) { // Aha! The pubapi of this package has changed! // It can also be a new compile from scratch. if (mprev.lookupPackage(a.getKey()).existsInJavacState()) { @@ -801,7 +950,7 @@ if (makefileSourceList == null) return; Set calculatedSources = new HashSet<>(); - Set listedSources = new HashSet<>(); + Set listedSources = SourceLocation.loadList(makefileSourceList); // Create a set of filenames with full paths. for (Source s : now.sources().values()) { @@ -813,38 +962,8 @@ calculatedSources.add(path); } } - // Read in the file and create another set of filenames with full paths. - try { - BufferedReader in = new BufferedReader(new FileReader(makefileSourceList)); - for (;;) { - String l = in.readLine(); - if (l==null) break; - l = l.trim(); - if (mightNeedRewriting) { - if (l.indexOf(":") == 1 && l.indexOf("\\") == 2) { - // Everything a-ok, the format is already C:\foo\bar - } else if (l.indexOf(":") == 1 && l.indexOf("/") == 2) { - // The format is C:/foo/bar, rewrite into the above format. - l = l.replaceAll("/","\\\\"); - } else if (l.charAt(0) == '/' && l.indexOf("/",1) != -1) { - // The format might be: /cygdrive/c/foo/bar, rewrite into the above format. - // Do not hardcode the name cygdrive here. - int slash = l.indexOf("/",1); - l = l.replaceAll("/","\\\\"); - l = ""+l.charAt(slash+1)+":"+l.substring(slash+2); - } - if (Character.isLowerCase(l.charAt(0))) { - l = Character.toUpperCase(l.charAt(0))+l.substring(1); - } - } - listedSources.add(l); - } - } catch (FileNotFoundException e) { - throw new ProblemException("Could not open "+makefileSourceList.getPath()+" since it does not exist!"); - } catch (IOException e) { - throw new ProblemException("Could not read "+makefileSourceList.getPath()); - } + for (String s : listedSources) { if (!calculatedSources.contains(s)) { throw new ProblemException("The makefile listed source "+s+" was not calculated by the smart javac wrapper!"); @@ -857,4 +976,32 @@ } } } + + /** + * Add the classes in deps, to the pubapis of the Packages. + * The pubapis are stored within the corresponding Package in now. + */ + public void addToClasspathPubapis(Map> deps) { + Compile comp = new Compile(options); + // Extract all the pubapis of the classes inside deps and + // store the info into the corresponding Package objects. + for (Map.Entry> a : deps.entrySet()) { + String pkg = a.getKey(); + Module mnow = now.findModuleFromPackageName(pkg); + Set classes = new HashSet<>(); + classes.addAll(a.getValue()); + classes.addAll(mnow.lookupPackage(pkg).getClassesFromClasspathPubapi()); + List sorted_classes = new ArrayList<>(); + for (String key : classes) { + sorted_classes.add(key); + } + Collections.sort(sorted_classes); + + List pubapis = new ArrayList<>(); + for (String s : sorted_classes) { + pubapis.addAll(comp.getPubapi(s, now.archives())); + } + mnow.setPubapiForLinkedClasses(a.getKey(), pubapis); + } + } }