--- /dev/null Fri Jun 15 22:43:39 2012 +++ new/src/share/classes/org/openjdk/jigsaw/ModuleFileParser.java Fri Jun 15 22:43:39 2012 @@ -0,0 +1,286 @@ +/* + * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package org.openjdk.jigsaw; + +import java.io.InputStream; +import java.util.Iterator; +import java.util.Map.Entry; +import org.openjdk.jigsaw.ModuleFile.ModuleFileHeader; +import org.openjdk.jigsaw.ModuleFile.SectionHeader; +import org.openjdk.jigsaw.ModuleFile.SubSectionFileHeader; + +/** + * The ModuleFileParser interface supports parsing of a + * + * module file. + * + *

The ModuleFileParser is designed to iterate over the module file using + * {@code hasNext()} and {@code next()}. The section type and data can be + * accessed using methods such as {@code getSectionHeader()}, + * {@code getContentStream()}, and {@code getClasses()}.

+ * + *

The {@code next()} method causes the parser to read the next parse event, + * and returns an {@code Event} which identifies the type of event just read. + * The current event type can be determined invoking the {@code event()} + * method.

+ * + *

Convenience methods, such as {@code skipToNextStartSection} and + * {@code skipToNextStartSubSection}, are defined to support easy access to + * (sub)section content without having to handle specific event types.

+ * + *

A minimal example for using this API may look as follows: + *

+ *       try (FileInputStream fis = new FileInputStream(jmodFile)) {
+ *           ModuleFileParser parser = ModuleFile.newParser(fis);
+ *           while (parser.skipToNextStartSection()) {
+ *               SectionHeader header = parser.getSectionHeader();
+ *               SectionType type = header.getType();
+ *               switch (type) {
+ *                   case MODULE_INFO:
+ *                       out.format("%s section, %s%n", type, header); break;
+ *                   case SIGNATURE:
+ *                       out.format("%s section, %s%n", type, header); break;
+ *                   case CLASSES:
+ *                       out.format("%s section, %s%n", type, header); break;
+ *                   case RESOURCES:
+ *                       out.format("%s section, %s%n", type, header); break;
+ *                   case NATIVE_LIBS:
+ *                       out.format("%s section, %s%n", type, header); break;
+ *                   case NATIVE_CMDS:
+ *                       out.format("%s section, %s%n", type, header); break;
+ *                   case CONFIG:
+ *                       out.format("%s section, %s%n", type, header); break;
+ *                   default:
+ *                       throw new IOException("Unknown section type");
+ *               }
+ *           }
+ *       }
+ * 

+ */ + +public interface ModuleFileParser { + + /** + * ModuleFileParser parsing events. + * + *

Events must follow the following grammar. + *

+     *  START_FILE
+     *    START_SECTION          (MODULE_INFO must be present)
+     *    END_SECTION
+     *    [START_SECTION         (optional sections)
+     *       [START_SUBSECTION   (sections with subsections)
+     *        END_SUBSECTION]+
+     *     END_SECTION]+
+     *  END_FILE
+     * 

+ */ + public static enum Event { + /** + * The first event returned. The module header has been consumed. + */ + START_FILE, + /** + * The start of a section. The section header has been consumed. + */ + START_SECTION, + /** + * The start of a subsection. The subsection header has been consumed. + */ + START_SUBSECTION, + /** + * The end of a subsection. The subsection content has been consumed. + */ + END_SUBSECTION, + /** + * The end of a section. The section content has been consumed. + */ + END_SECTION, + /** + * The end of the module file. The file content has been consumed. + */ + END_FILE; + } + + /** + * Returns the module file header. + * + * @return The module file header + */ + public ModuleFileHeader fileHeader(); + + /** + * Returns current event of the parser. + */ + public Event event(); + + /** + * Check if there are more events. + * + * @return true, if and only if, there are more events + */ + public boolean hasNext(); + + /** + * Returns the next parsing event. + * + *

Skips over any unread data from a previous (sub)section.

+ * + * @return the next parse event + * + * @throws NoSuchElementException + * If invoked when {@code hasNext} returns false + * + * @throws ModuleFileParserException + * If there is an error processing the underlying module file + */ + public Event next(); + + /** + * Returns the header of the current section. + * + * @return the section header + * + * @throws ModuleFileParserException + * If the current event is one of START_FILE or END_FILE + */ + public SectionHeader getSectionHeader(); + + /** + * Returns the header of the current file subsection. + * + * @return the subsection header + * + * @throws ModuleFileParserException + * If the current event is not one of START_SUBSECTION or END_SUBSECTION + */ + public SubSectionFileHeader getSubSectionFileHeader(); + + /** + * Returns the hash of the module file header, current section, or file. + * + *

Returns the hash of the module file header if the current event is + * START_FILE, the section hash if the current event is END_SECTION, or the + * complete file contents hash if the current event is END_FILE.

+ * + *

The specific hashes are calculated as per the module-file + * specification. More specifically, the module file header hash and the + * section hash exclude the follow fields from their header, the hash length + * and the hash value. The complete file contents hash excludes the file + * hash value in the module file header and the signature section + * (if present).

+ * + * @return the hash bytes. + * + * @throws ModuleFileParserException + * If the current event is not one of START_FILE, END_SECTION, + * or END_FILE + */ + public byte[] getHash(); + + /** + * Returns an InputStream of the uncompressed content of the current + * section or subsection (if the section defines a compressor), otherwise + * just the raw bytes. + * + *

For the CLASSES section ( {@code getSectionHeader.getType() == + * ModuleFile.SectionType.CLASSES} ) the {@code getClasses} method should be + * invoked. All other sections and subsections can invoke this method to get + * the uncompressed (sub)section content.

+ * + * @return the content stream + * + * @throws ModuleFileParserException + * If the current event is not one of START_SECTION or + * START_SUBSECTION, if {@code getSectionHeader().getType()} returns + * {@code ModuleFile.SectionType.CLASSES}, or if there is an error + * processing the underlying module file + */ + public InputStream getContentStream(); + + /** + * Returns an Iterator over the classes in the CLASSES section. + * + *

If this method is invoked to extract the content of the classes + * section then it must be invoked in a section whose + * {@code getSectionHeader.getType()} returns + * {@code ModuleFile.SectionType.CLASSES}. Any other time will throw a + * {@code ModuleFileParserException}.

+ * + * @return an Iterator over the classes, where the entry key is the class + * file name, e.g. java/lang/Object.class, and the value is an + * input stream containing the class bytes. + * + * @throws ModuleFileParserException + * If the current event is not START_SECTION, if + * {@code getSectionHeader().getType()} does not return + * {@code ModuleFile.SectionType.CLASSES}, or if there is an error + * processing the underlying module file + */ + public Iterator> getClasses(); + + /** + * Returns an InputStream of the raw bytes of the current section or + * subsection. + * + *

If the (sub)section data is compressed then this method simply + * returns the compressed bytes, and the invoker is responsible for + * decompressing.

+ * + * @return The raw (sub)section data + * + * @throws ModuleFileParserException + * If the current event is not one of START_SECTION or START_SUBSECTION + */ + public InputStream getRawStream(); + + /** + * Skips to the start of the next section. + * + *

Skips over any unread data, and consumes events until START_SECTION + * or END_FILE is encountered.

+ * + * @return true, if and only if, the next event is START_SECTION + * + * @throws ModuleFileParserException + * If there is an error processing the underlying module file + */ + public boolean skipToNextStartSection(); + + /** + * Skips to the start of the next subsection. + * + *

Skips over any unread data, and consumes events until END_SECTION + * is encountered.

+ * + * @return true, if and only if, the current event is START_SUBSECTION + * + * @throws ModuleFileParserException + * If not within a section that contains subsections, or if there is + * an error processing the underlying module file + */ + public boolean skipToNextStartSubSection(); +} --- /dev/null Fri Jun 15 22:43:40 2012 +++ new/src/share/classes/org/openjdk/jigsaw/ModuleFileParserException.java Fri Jun 15 22:43:40 2012 @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package org.openjdk.jigsaw; + +/** + * An Exception thrown when parsing a module file. + */ +public class ModuleFileParserException extends RuntimeException { + // ## TODO: fix uid value + private static final long serialVersionUID = 74132770414881L; + + // ## section type/event useful to capture in exception? + //private Event event; + //private SectionType; + + public ModuleFileParserException(String message) { + super(message); + } + + public ModuleFileParserException(Throwable cause) { + super(cause); + } + + public ModuleFileParserException(String message, Throwable cause) { + super(message, cause); + } +} --- /dev/null Fri Jun 15 22:43:41 2012 +++ new/src/share/classes/org/openjdk/jigsaw/ModuleFileParserImpl.java Fri Jun 15 22:43:40 2012 @@ -0,0 +1,606 @@ +/* + * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package org.openjdk.jigsaw; + +import java.io.*; +import java.security.DigestInputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Map.Entry; +import java.util.*; +import java.util.jar.JarOutputStream; +import java.util.jar.Pack200; +import java.util.zip.GZIPInputStream; +import java.util.zip.ZipEntry; +import org.openjdk.jigsaw.FileConstants.ModuleFile.Compressor; +import org.openjdk.jigsaw.FileConstants.ModuleFile.HashType; +import org.openjdk.jigsaw.FileConstants.ModuleFile.SectionType; +import static org.openjdk.jigsaw.FileConstants.ModuleFile.SectionType.*; +import org.openjdk.jigsaw.ModuleFile.ModuleFileHeader; +import org.openjdk.jigsaw.ModuleFile.SectionHeader; +import org.openjdk.jigsaw.ModuleFile.SubSectionFileHeader; +import static org.openjdk.jigsaw.ModuleFileParser.Event.*; + +public class ModuleFileParserImpl + implements ModuleFileParser +{ + private static class CountingInputStream extends FilterInputStream { + private int count; + public CountingInputStream(InputStream stream, int count) { + super(stream); + this.count = count; + } + + public int available() throws IOException { + return count; + } + + public boolean markSupported() { + return false; + } + + public int read() throws IOException { + if (count == 0) + return -1; + int read = super.read(); + if (-1 != read) + count--; + return read; + } + + public int read(byte[] b, int off, int len) throws IOException { + if (count == 0) + return -1; + len = Math.min(len, count); + int read = super.read(b, off, len); + if (-1 != read) + count-=read; + return read; + } + + public void reset() throws IOException { + throw new IOException("Can't reset this stream"); + } + + public long skip(long n) throws IOException { + // ## never skip, always read for digest, skip could just call read + throw new IOException("skip should never be called"); + } + + public void close() throws IOException { + // Do nothing, CountingInputStream is used to wrap (sub)section + // content. We never want to close the underlying stream. + } + } + + private final DataInputStream stream; // dataInput wrapped raw stream + private final HashType hashtype = HashType.SHA256; + private final ModuleFileHeader fileHeader; + private final MessageDigest fileDigest; + private final MessageDigest sectionDigest; + private DataInputStream digestStream; // fileDigest, wrapper input stream + + // parser state + private Event curEvent; + private SectionHeader curSectionHeader; + private SubSectionFileHeader curSubSectionHeader; + private InputStream curSectionIn; + private InputStream curSubSectionIn; + private int subSectionCount; + private byte[] hash; + private ModuleFileParserException parserException; + + /*package*/ ModuleFileParserImpl(InputStream in) { + // Ensure that mark/reset is supported + if (in.markSupported()) + stream = new DataInputStream(in); + else + stream = new DataInputStream(new BufferedInputStream(in)); + + try { + fileDigest = getHashInstance(hashtype); + sectionDigest = getHashInstance(hashtype); + DigestInputStream dis = new DigestInputStream(stream, fileDigest); + fileHeader = ModuleFileHeader.read(dis); + // calculate module header hash + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + fileHeader.write(new DataOutputStream(baos)); + sectionDigest.update(baos.toByteArray()); + digestStream = new DataInputStream(dis); + hash = sectionDigest.digest(); + curEvent = START_FILE; + } catch (IOException | ModuleFileParserException x) { + throw parserException(x); + } + } + + @Override + public ModuleFileHeader fileHeader() { + return fileHeader; + } + + @Override + public Event event() { + return curEvent; + } + + @Override + public boolean hasNext() { + if (parserException != null) + return false; + + return curEvent != END_FILE; + } + + @Override + public Event next() { + if (!hasNext()) { + if (parserException != null) + throw new NoSuchElementException("END_FILE reached"); + else + throw parserException("Error processing input. The input stream is not complete."); + } + + // Reset general state + hash = null; + + try { + switch (curEvent) { + case START_FILE: + // can only transition to START_SECTION, module-info + curSectionHeader = SectionHeader.read(digestStream); + SectionType type = curSectionHeader.getType(); + if (type != MODULE_INFO) + throw parserException(type + ": expected MODULE_INFO"); + sectionDigest.reset(); + curSectionIn = new DigestInputStream(new CountingInputStream(digestStream, + curSectionHeader.getCSize()), sectionDigest); + return curEvent = START_SECTION; + case START_SECTION : + // can only transition to START_SUBSECTION or END_SECTION + if (subSectionCount != 0) + return curEvent = startSubSection(); + // END_SECTION + skipAnyUnread(curSectionIn); + hash = sectionDigest.digest(); + return curEvent = END_SECTION; + case START_SUBSECTION : + // can only transition to END_SUBSECTION + skipAnyUnread(curSubSectionIn); + return curEvent = END_SUBSECTION; + case END_SUBSECTION : + // can only transition to START_SUBSECTION or END_SECTION + if (subSectionCount != 0) + return curEvent = startSubSection(); + checkAllRead(curSectionIn, + "subsections do not consume all section data"); + hash = sectionDigest.digest(); + return curEvent = END_SECTION; + case END_SECTION : + // must transition to START_SECTION or END_FILE + SectionHeader nextHeader = peekNextSection(false); + if (nextHeader == null) { + hash = fileDigest.digest(); + return curEvent = END_FILE; + } + // START_SECTION + return curEvent = startSection(nextHeader); + case END_FILE : + throw parserException( + "should not reach here, next with current event END_FILE"); + default : + throw parserException("Unknown event: " + curEvent); + } + } catch (IOException | ModuleFileParserException x) { + throw parserException(x); + } + } + + private Event startSection(SectionHeader nextHeader) throws IOException { + sectionDigest.reset(); + DataInputStream in = digestStream; + if (nextHeader.getType() == SIGNATURE ) + // special handling for SIGNATURE section, skip file digest + in = stream; + + curSectionHeader = SectionHeader.read(in); + if (curSectionHeader.getType() == MODULE_INFO) + throw parserException("Unexpected MODULE_INFO"); + curSectionIn = new DigestInputStream(new CountingInputStream(in, + curSectionHeader.getCSize()), sectionDigest); + + if (curSectionHeader.getType().hasFiles()) + subSectionCount = curSectionHeader.getSubsections(); + else + subSectionCount = 0; + curSubSectionIn = null; + return START_SECTION; + } + + private Event startSubSection() throws IOException { + curSubSectionHeader = SubSectionFileHeader.read(new DataInputStream(curSectionIn)); + curSubSectionIn = new CountingInputStream(curSectionIn, curSubSectionHeader.getCSize()); + subSectionCount--; + return START_SUBSECTION; + } + + private ModuleFileParserException parserException(String message) { + return parserException = new ModuleFileParserException(message); + } + + private ModuleFileParserException parserException(Exception x) { + if (x instanceof ModuleFileParserException) + return parserException = (ModuleFileParserException) x; + + return parserException = new ModuleFileParserException(x); + } + + private static void skipAnyUnread(InputStream is) + throws IOException + { + byte[] ba = new byte[8192]; + while (is.read(ba) != -1); + } + + private void checkAllRead(InputStream is, String message) + throws IOException + { + if (is.read() != -1) + throw parserException(message); + } + + // required to be able to handle special case the signature + private SectionHeader peekNextSection(boolean throwOnEOF) + throws IOException + { + // Mark the position & read from stream (does not effect digest) + stream.mark(SectionHeader.LENGTH); + + if (stream.read() == -1) + return null; + + stream.reset(); + SectionHeader header = SectionHeader.read(stream); + stream.reset(); + if (header != null) + return header; + + throw parserException("Error parsing section header"); + } + + @Override + public boolean skipToNextStartSection() { + if (curEvent == END_FILE) return false; + + while (hasNext()) { + Event e = next(); + if (e == START_SECTION) + return true; + if (e == END_FILE) + return false; + } + return false; + } + + @Override + public boolean skipToNextStartSubSection() { + if (!(curEvent == START_SECTION || curEvent == START_SUBSECTION || + curEvent == END_SUBSECTION)) + return false; + + if (!getSectionHeader().getType().hasFiles()) + throw parserException(getSectionHeader().getType() + + " section does not contain subsections"); + + while(hasNext()) { + Event e = next(); + if (e == START_SUBSECTION) return true; + if (e == END_SECTION) return false; + } + return false; + } + + @Override + public SectionHeader getSectionHeader() { + if (curEvent == START_FILE || curEvent == END_FILE) + throw parserException("No section header for: " + curEvent); + return curSectionHeader; + } + + @Override + public SubSectionFileHeader getSubSectionFileHeader() { + if (!(curEvent == START_SUBSECTION || curEvent == END_SUBSECTION)) + throw parserException("No subsection header for " + curEvent); + return curSubSectionHeader; + } + + @Override + public byte[] getHash() { + if (!(curEvent == START_FILE || curEvent == END_SECTION || + curEvent == END_FILE)) + throw parserException("Hash not calculatable at " + curEvent); + + return hash; + } + + @Override + public InputStream getContentStream() { + if (!(curEvent == START_SECTION || curEvent == START_SUBSECTION)) + throw parserException("current event " + curEvent + + ", expected one of START_SECTION or START_SUBSECTION"); + + InputStream is = curSubSectionIn != null ? curSubSectionIn : curSectionIn; + + SectionType type = curSectionHeader.getType(); + Compressor compressor = curSectionHeader.getCompressor(); + + if (type == CLASSES) { + throw parserException("should not be called for CLASSES"); + } else { + try { + Decompressor decompressor = Decompressor.newInstance(is, compressor); + return decompressor.extractStream(); + } catch (IOException | ModuleFileParserException x) { + throw parserException(x); + } + } + } + + @Override + public InputStream getRawStream() { + if (!(curEvent == START_SECTION || curEvent == START_SUBSECTION)) + throw parserException("current event " + curEvent + + ", expected one of START_SECTION or START_SUBSECTION"); + + return curSubSectionIn != null ? curSubSectionIn : curSectionIn; + } + + @Override + public Iterator> getClasses() { + if (curEvent != START_SECTION) + throw parserException("current event " + curEvent + + ", expected START_SECTION"); + + SectionType type = curSectionHeader.getType(); + Compressor compressor = curSectionHeader.getCompressor(); + + if (type != CLASSES) + throw parserException(type + ": not classes section"); + if (curSectionIn == null) + throw parserException("not at a valid classes section"); + + try { + ClassesDecompressor decompressor = + ClassesDecompressor.newInstance(curSectionIn, compressor, /*deflate*/false); + ClassesJarOutputStream cjos = new ClassesJarOutputStream(); + decompressor.extractTo(cjos); + + return cjos.classes().iterator(); + } catch (IOException | ModuleFileParserException x) { + throw parserException(x); + } + } + + private static class ClassesEntry + implements Entry, java.io.Serializable + { + //private static final long serialVersionUID = -8499721149061103585L; + + private final String key; + private InputStream value; + + ClassesEntry(String key, InputStream value) { + this.key = key; + this.value = value; + } + + @Override + public String getKey() { + return key; + } + + @Override + public InputStream getValue() { + return value; + } + + @Override + public InputStream setValue(InputStream value) { + InputStream oldValue = this.value; + this.value = value; + return oldValue; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof ClassesEntry)) + return false; + ClassesEntry e = (ClassesEntry)o; + if (key == null ? e.key != null : !key.equals(e.key)) + return false; + if (value == null ? e.value != null : !value.equals(e.value)) + return false; + return true; + } + + @Override + public int hashCode() { + return (key == null ? 0 : key.hashCode()) ^ + (value == null ? 0 : value.hashCode()); + } + + @Override + public String toString() { + return key + ":" + value; + } + } + + private static class ClassesJarOutputStream extends JarOutputStream { + private Set> classes; + private ByteArrayOutputStream classBytes; + private String path; + + ClassesJarOutputStream() + throws IOException + { + super(nullOutputStream); + classes = new HashSet<>(); + classBytes = new ByteArrayOutputStream(); + } + + @Override + public void putNextEntry(ZipEntry ze) throws IOException { + classBytes.reset(); + path = ze.getName(); + } + + @Override + public void closeEntry() throws IOException { + classes.add(new ClassesEntry(path, + new ByteArrayInputStream(classBytes.toByteArray()))); + } + + @Override + public void write(int b) throws IOException { + classBytes.write(b); + } + + @Override + public void write(byte[] ba) throws IOException { + classBytes.write(ba); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + classBytes.write(b, off, len); + } + + Set> classes() { + return classes; + } + } + + private static OutputStream nullOutputStream = new NullOutputStream(); + + static class NullOutputStream extends OutputStream { + @Override + public void write(int b) throws IOException {} + @Override + public void write(byte[] b) throws IOException {} + @Override + public void write(byte[] b, int off, int len) throws IOException {} + } + + static class Decompressor { + protected InputStream in; + protected Decompressor() { } + protected Decompressor(InputStream in) { + // no decompression + this.in = in; + } + + InputStream extractStream() { + return in; + } + + static Decompressor newInstance(InputStream in, + Compressor compressor) + throws IOException + { + switch (compressor) { + case NONE: + return new Decompressor(in); + case GZIP: + return new GZIPDecompressor(in); + default: + throw new ModuleFileParserException( + "Unsupported compressor type: " + compressor); + } + } + } + + static class GZIPDecompressor extends Decompressor { + GZIPDecompressor(InputStream in) throws IOException { + this.in = new GZIPInputStream(in) { + public void close() throws IOException {} + }; + } + } + + static abstract class ClassesDecompressor { + protected InputStream in; + + abstract void extractTo(JarOutputStream out) throws IOException; + + static ClassesDecompressor newInstance(InputStream in, + Compressor compressor, + boolean deflate) + throws IOException + { + switch (compressor) { + case PACK200_GZIP: + return new Pack200GZIPDecompressor(in, deflate); + default: + throw new ModuleFileParserException( + "Unsupported compressor type: " + compressor); + } + } + } + + static class Pack200GZIPDecompressor extends ClassesDecompressor { + private Pack200.Unpacker unpacker; + + Pack200GZIPDecompressor(InputStream in, boolean deflate) + throws IOException + { + this.in = new GZIPInputStream(in) { + public void close() throws IOException {} + }; + unpacker = Pack200.newUnpacker(); + if (deflate) { + Map p = unpacker.properties(); + p.put(Pack200.Unpacker.DEFLATE_HINT, Pack200.Unpacker.TRUE); + } + } + + void extractTo(JarOutputStream out) throws IOException { + unpacker.unpack(in, out); + } + } + + static MessageDigest getHashInstance(HashType hashtype) { + try { + switch(hashtype) { + case SHA256: + return MessageDigest.getInstance("SHA-256"); + default: + throw new ModuleFileParserException("Unknown hash type: " + hashtype); + } + } catch (NoSuchAlgorithmException x) { + throw new ModuleFileParserException(hashtype + " not found", x); + } + } +} --- /dev/null Fri Jun 15 22:43:42 2012 +++ new/test/org/openjdk/jigsaw/ModuleFileParserTest.java Fri Jun 15 22:43:41 2012 @@ -0,0 +1,231 @@ +/* + * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * Basic tests for ModuleFileParser + */ + +import java.io.*; +import static java.lang.System.out; +import java.util.Arrays; +import java.util.Iterator; +import java.util.Map.Entry; +import org.openjdk.jigsaw.FileConstants.ModuleFile.SectionType; +import static org.openjdk.jigsaw.FileConstants.ModuleFile.SectionType.*; +import org.openjdk.jigsaw.ModuleFile; +import org.openjdk.jigsaw.ModuleFile.SectionHeader; +import org.openjdk.jigsaw.ModuleFile.SubSectionFileHeader; +import org.openjdk.jigsaw.ModuleFileParser; +import org.openjdk.jigsaw.ModuleFileParser.Event; +import static org.openjdk.jigsaw.ModuleFileParser.Event.END_FILE; +import static org.openjdk.jigsaw.ModuleFileParser.Event.END_SECTION; +import org.openjdk.jigsaw.ModuleFileParserException; + +public class ModuleFileParserTest { + public static void main(String[] args) throws Exception { + if (args.length != 1) { + usage(); + return; + } + File jmod = new File(args[0]); + + list(jmod); + getClasses(jmod); + getOneClass(jmod); + getlibs(jmod); + verify(jmod); + } + + /* + * List the sections of the given module file. + */ + static void list(File jmod) { + out.format("Listing sections of %s%n", jmod); + + try (FileInputStream fis = new FileInputStream(jmod)) { + ModuleFileParser parser = ModuleFile.newParser(fis); + while (parser.skipToNextStartSection()) { + SectionHeader header = parser.getSectionHeader(); + SectionType type = header.getType(); + switch (type) { + case MODULE_INFO: + out.format("%s section, %s%n", type, header); break; + case SIGNATURE: + out.format("%s section, %s%n", type, header); break; + case CLASSES: + out.format("%s section, %s%n", type, header); break; + case RESOURCES: + out.format("%s section, %s%n", type, header); break; + case NATIVE_LIBS: + out.format("%s section, %s%n", type, header); break; + case NATIVE_CMDS: + out.format("%s section, %s%n", type, header); break; + case CONFIG: + out.format("%s section, %s%n", type, header); break; + default: + throw new IOException("Unknown section type"); + } + } + } catch (IOException | ModuleFileParserException e) { + out.format("%s%n", e); + } + } + + /* + * Extract classes from the given module file. + */ + static void getClasses(File jmod) { + out.format("Reading classes from %s%n", jmod); + + try (FileInputStream fis = new FileInputStream(jmod)) { + ModuleFileParser parser = ModuleFile.newParser(fis); + while (parser.skipToNextStartSection()) { + SectionHeader header = parser.getSectionHeader(); + if (header.getType() == CLASSES) { + out.format("Has %s section%n", header.getType()); + Iterator> classes = parser.getClasses(); + while (classes.hasNext()) { + Entry entry = classes.next(); + try (OutputStream os = createFile(jmod.getName() + + File.separator + "classes" + File.separator + + entry.getKey())) { + copyStream(entry.getValue(), os); + } + } + break; + } + } + } catch (IOException | ModuleFileParserException e) { + out.format("%s%n", e); + } + } + + /* + * Extracts the first class from the CLASSES section of the given module file. + */ + static void getOneClass(File jmod) { + out.format("Reading one classes from %s%n", jmod); + + try (FileInputStream fis = new FileInputStream(jmod)) { + ModuleFileParser parser = ModuleFile.newParser(fis); + while (parser.skipToNextStartSection()) { + SectionHeader header = parser.getSectionHeader(); + if (header.getType() == CLASSES) { + out.format("Has %s section%n", header.getType()); + Iterator> classes = parser.getClasses(); + if (classes.hasNext()) { + Entry entry = classes.next(); + try (OutputStream os = createFile(jmod.getName() + + File.separator + "classes" + File.separator + + entry.getKey())) { + copyStream(entry.getValue(), os); + } + } + break; + } + } + } catch (IOException | ModuleFileParserException e) { + out.format("%s%n", e); + } + } + + /* + * Extract the native libs from the given module file. + */ + static void getlibs(File jmod) { + out.format("Reading libs from %s%n", jmod); + + try (FileInputStream fis = new FileInputStream(jmod)) { + ModuleFileParser parser = ModuleFile.newParser(fis); + while (parser.skipToNextStartSection()) { + SectionHeader header = parser.getSectionHeader(); + if (header.getType() == NATIVE_LIBS) { + out.format("Has %s section%n", header.getType()); + while (parser.skipToNextStartSubSection()) { + SubSectionFileHeader subHeader = parser.getSubSectionFileHeader(); + String path = subHeader.getPath(); + try (OutputStream os = createFile(jmod.getName() + + File.separator + "lib" + File.separator + path)) { + copyStream(parser.getContentStream(), os); + } + } + break; + } + } + } catch (IOException | ModuleFileParserException e) { + out.format("%s%n", e); + } + } + + /* + * Verify the integraty of the given module file. + */ + static void verify(File jmod) { + out.format("Verifying %s%n", jmod); + + try (FileInputStream fis = new FileInputStream(jmod)) { + ModuleFileParser parser = ModuleFile.newParser(fis); + while (parser.hasNext()) { + Event event = parser.next(); + if (event == END_SECTION) { + SectionHeader header = parser.getSectionHeader(); + // compare hash in header to actual hash of content + if (!Arrays.equals(header.getHash(), parser.getHash())) + throw new RuntimeException("Verifying hash failed for " + + header.getType()); + } else if (event == END_FILE) { + // compare hash in header to actual hash of file content + if (!Arrays.equals(parser.fileHeader().getHash(), parser.getHash())) + throw new RuntimeException("Verifying hash failed for File"); + } + } + } catch (IOException | ModuleFileParserException e) { + out.format("%s%n", e); + } + } + + /* + * Helper method to create sub directories (if necessary) + */ + private static OutputStream createFile(String pathName) throws IOException { + File path = new File(pathName); + File parent = new File(path.getParent()); + if (!parent.exists()) + parent.mkdirs(); + FileOutputStream fos = new FileOutputStream(path); + return new BufferedOutputStream(fos); + } + + private static void copyStream(InputStream in, OutputStream out) + throws IOException + { + byte[] buf = new byte[8192]; + int read = 0; + while ((read = in.read(buf)) > 0) + out.write(buf, 0, read); + } + + private static void usage() { + out.format("Usage: java ModuleFileTest %n"); + } +} --- old/src/share/classes/org/openjdk/jigsaw/ModuleFile.java Fri Jun 15 22:43:43 2012 +++ new/src/share/classes/org/openjdk/jigsaw/ModuleFile.java Fri Jun 15 22:43:42 2012 @@ -30,9 +30,10 @@ import java.security.*; import java.util.*; import java.util.jar.*; -import java.util.zip.*; - +import org.openjdk.jigsaw.ModuleFileParser.Event; import static org.openjdk.jigsaw.FileConstants.ModuleFile.*; +import static org.openjdk.jigsaw.FileConstants.ModuleFile.SectionType.*; +import static org.openjdk.jigsaw.ModuleFileParser.Event.*; public final class ModuleFile { /** @@ -57,164 +58,135 @@ } } + /** + * Returns a ModuleFileParser instance. + * + * @param stream + * module file stream + * + * @return a module file parser + * + * @throws ModuleFileParserException + * If there is an error processing the underlying module file + */ + public static ModuleFileParser newParser(InputStream stream) { + return new ModuleFileParserImpl(stream); + } + public final static class Reader implements Closeable { + private final ModuleFileParser parser; + private final ModuleFileHeader fileHeader; + private final byte[] moduleInfoBytes; + private final SignatureType moduleSignatureType; + private final byte[] moduleSignatureBytes ; + private final List calculatedHashes; - private DataInputStream stream; private File destination; private boolean deflate; - private final HashType hashtype = HashType.SHA256; private File natlibs; private File natcmds; private File configs; - private static class CountingInputStream extends FilterInputStream { - int count; - public CountingInputStream(InputStream stream, int count) { - super(stream); - this.count = count; - } + public Reader(InputStream stream) throws IOException { + calculatedHashes = new ArrayList<>(); + parser = ModuleFile.newParser(stream); + fileHeader = parser.fileHeader(); + calculatedHashes.add(parser.getHash()); + // Read the MODULE_INFO and the Signature section (if present), + // but does not write any files. + parser.next(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + copyStream(parser.getRawStream(), baos); + moduleInfoBytes = baos.toByteArray(); + assert moduleInfoBytes != null; - public int available() throws IOException { - return count; - } + if (parser.next() != END_SECTION) + throw new ModuleFileParserException( + "Expected END_SECTION of module-info"); + calculatedHashes.add(parser.getHash()); - public boolean markSupported() { - return false; + if (parser.hasNext()) { + Event event = parser.next(); + if (event != END_FILE) { // more sections + SectionHeader header = parser.getSectionHeader(); + if (header.type == SIGNATURE) { + SignatureSection sh = SignatureSection.read( + new DataInputStream(parser.getRawStream())); + moduleSignatureType = SignatureType.valueOf(sh.getSignatureType()); + moduleSignatureBytes = sh.getSignature(); + if (parser.next() != END_SECTION) + throw new ModuleFileParserException( + "Expected END_SECTION of signature"); + if (parser.hasNext()) + parser.next(); // position parser at next event + return; + } + } } + // no signature section, or possibly other sections at all. + moduleSignatureBytes = null; + moduleSignatureType = null; + } - public int read() throws IOException { - if (count == 0) - return -1; - int read = super.read(); - if (-1 != read) - count--; - return read; - } - - public int read(byte[] b, int off, int len) throws IOException { - if (count == 0) - return -1; - len = Math.min(len, count); - int read = super.read(b, off, len); - if (-1 != read) - count-=read; - return read; - } - - public void reset() throws IOException { - throw new IOException("Can't reset this stream"); - } - - public long skip(long n) throws IOException { - if (count == 0) - return -1; - n = Math.min(n, count); - long skipped = super.skip(n); - if (n > 0) - count-=skipped; - return skipped; - } + public void extractTo(File dst) throws IOException { + extractTo(dst, false); } - public Reader(DataInputStream stream) { - // Ensure that mark/reset is supported - if (stream.markSupported()) { - this.stream = stream; - } else { - this.stream = - new DataInputStream(new BufferedInputStream(stream)); - } + public void extractTo(File dst, boolean deflate) throws IOException { + extractTo(dst, deflate, null, null, null); } - private void checkHashMatch(byte[] expected, byte[] computed) + public void extractTo(File dst, boolean deflate, File natlibs, + File natcmds, File configs) throws IOException { - if (!MessageDigest.isEqual(expected, computed)) - throw new IOException("Expected hash " - + hashHexString(expected) - + " instead of " - + hashHexString(computed)); - } - - private ModuleFileHeader fileHeader = null; - private MessageDigest fileDigest = null; - private MessageDigest sectionDigest = null; - private DataInputStream fileIn = null; - private byte[] moduleInfoBytes = null; - private SignatureType moduleSignatureType = null; - private byte[] moduleSignatureBytes = null; - private final int MAX_SECTION_HEADER_LENGTH = 128; - private List calculatedHashes = new ArrayList<>(); - private boolean extract = true; - - /* - * Reads the MODULE_INFO section and the Signature section, if present, - * but does not write any files. - */ - public byte[] readStart() throws IOException { - - try { - fileDigest = getHashInstance(hashtype); - sectionDigest = getHashInstance(hashtype); - DigestInputStream dis = - new DigestInputStream(stream, fileDigest); - fileHeader = ModuleFileHeader.read(dis); - // calculate module header hash - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - fileHeader.write(new DataOutputStream(baos)); - sectionDigest.update(baos.toByteArray()); - calculatedHashes.add(sectionDigest.digest()); - - fileIn = new DataInputStream(dis); - if (readSection(fileIn) != SectionType.MODULE_INFO) - throw new IOException("First module-file section" - + " is not MODULE_INFO"); - assert moduleInfoBytes != null; - - // Read the Signature Section, if present - readSignatureSection(fileIn, dis); - - return moduleInfoBytes.clone(); - } catch (IOException x) { - close(); - throw x; - } - } - - public void readRest() throws IOException { - extract = false; - readRest(null, false, null, null, null); - } - - public void readRest(File dst, boolean deflate) throws IOException { - readRest(dst, deflate, null, null, null); - } - - public void readRest(File dst, boolean deflate, File natlibs, - File natcmds, File configs) - throws IOException - { this.deflate = deflate; this.destination = dst != null ? dst.getCanonicalFile() : null; this.natlibs = natlibs != null ? natlibs : new File(destination, "lib"); this.natcmds = natcmds != null ? natcmds : new File(destination, "bin"); this.configs = configs != null ? configs : new File(destination, "etc"); + try { - if (extract) - Files.store(moduleInfoBytes, computeRealPath("info")); - // Module-Info and Signature, if present, have been consumed + Files.store(moduleInfoBytes, computeRealPath("info")); - // Read rest of file until all sections have been read - stream.mark(1); - while (-1 != stream.read()) { - stream.reset(); - readSection(fileIn); - stream.mark(1); - } + Event event = parser.event(); + if (event == END_FILE) + return; - close(); + if (event != START_SECTION) + throw new ModuleFileParserException( + "Expected START_SECTION, got : " + event); + // Module-Info and Signature, if present, have been consumed + do { + SectionHeader header = parser.getSectionHeader(); + SectionType type = header.getType(); + if (type.hasFiles()) { + while(parser.skipToNextStartSubSection()) { + readSubSection(type); + } + } else if (type == CLASSES) { + Iterator> classes = + parser.getClasses(); + while (classes.hasNext()) { + Map.Entry entry = classes.next(); + try (OutputStream out = openOutputStream(type, entry.getKey())) { + copyStream(entry.getValue(), out); + } + } + // END_SECTION + parser.next(); + } else { + throw new IllegalArgumentException("Unknown type: " + type); + } + byte[] sectionHash = parser.getHash(); + calculatedHashes.add(sectionHash); + checkHashMatch(sectionHash, header.getHash()); + } while (parser.skipToNextStartSection()); + + if (parser.event() != END_FILE) + throw new IOException("Expected END_FILE"); byte[] fileHeaderHash = fileHeader.getHashNoClone(); - checkHashMatch(fileHeaderHash, fileDigest.digest()); + checkHashMatch(fileHeaderHash, parser.getHash()); calculatedHashes.add(fileHeaderHash); } finally { close(); @@ -221,9 +193,11 @@ } } - public byte[] getHash() throws IOException { - if (null == fileHeader) - readStart(); + public byte[] getModuleInfoBytes() { + return moduleInfoBytes.clone(); + } + + public byte[] getHash() { return fileHeader.getHash(); } @@ -231,46 +205,23 @@ return calculatedHashes; } - public boolean hasSignature() throws IOException { - if (null == fileHeader) - readStart(); + public boolean hasSignature() { return moduleSignatureBytes != null; } - public SignatureType getSignatureType() throws IOException { - if (null == fileHeader) - readStart(); + public SignatureType getSignatureType() { return moduleSignatureType; } - public byte[] getSignature() throws IOException { - if (null == fileHeader) - readStart(); - return moduleSignatureBytes != null - ? moduleSignatureBytes.clone() - : null; + public byte[] getSignature() { + return moduleSignatureBytes == null ? null : + moduleSignatureBytes.clone(); } - byte[] getSignatureNoClone() { - return moduleSignatureBytes; + /*package*/byte[] getSignatureNoClone() { + return moduleSignatureBytes; } - private JarOutputStream contentStream = null; - - private JarOutputStream contentStream() throws IOException { - if (contentStream == null) { - if (extract) { - FileOutputStream fos - = new FileOutputStream(computeRealPath("classes")); - contentStream - = new JarOutputStream(new BufferedOutputStream(fos)); - } else { - contentStream = new JarOutputStream(new NullOutputStream()); - } - } - return contentStream; - } - public void close() throws IOException { try { try { @@ -279,10 +230,10 @@ contentStream = null; } } finally { - if (fileIn != null) { - fileIn.close(); - fileIn = null; - } + /*if (parser != null) { + parser.close(); + parser = null; + }*/ } } finally { if (filesWriter != null) { @@ -292,135 +243,40 @@ } } - public void readModule() throws IOException { - extract = false; - readStart(); - readRest(); - } + // subsections/files (resources, libs, cmds, configs) + public void readSubSection(SectionType type) throws IOException { + assert type == RESOURCES || type == NATIVE_LIBS || + type == NATIVE_CMDS || type == CONFIG; - public void readModule(File dst) throws IOException { - readStart(); - readRest(dst, false); - } - - private void readSignatureSection(DataInputStream stream, - DigestInputStream dis) - throws IOException - { - - // Turn off digest computation before reading Signature Section - dis.on(false); - - // Mark the starting position - stream.mark(MAX_SECTION_HEADER_LENGTH); - if (stream.read() != -1) { - stream.reset(); - SectionHeader header = SectionHeader.read(stream); - if (header != null && - header.getType() == SectionType.SIGNATURE) { - readSectionContent(header, stream); - } else { - // Revert back to the starting position - stream.reset(); - } + SubSectionFileHeader subHeader = parser.getSubSectionFileHeader(); + String path = subHeader.getPath(); + try (OutputStream sink = openOutputStream(type, path)) { + copyStream(parser.getContentStream(), sink); } - // Turn on digest computation again - dis.on(true); + // post processing for executable and files outside the module dir + postExtract(type, currentPath); } - private SectionType readSection(DataInputStream stream) - throws IOException - { - SectionHeader header = SectionHeader.read(stream); - readSectionContent(header, stream); - return header.getType(); - } + private JarOutputStream contentStream = null; - private void readSectionContent(SectionHeader header, - DataInputStream stream) - throws IOException - { - SectionType type = header.getType(); - Compressor compressor = header.getCompressor(); - int csize = header.getCSize(); - short subsections = - type.hasFiles() ? header.getSubsections() : 1; + private JarOutputStream contentStream() throws IOException { + if (contentStream != null) + return contentStream; - CountingInputStream cs = new CountingInputStream(stream, csize); - sectionDigest.reset(); - DigestInputStream dis = new DigestInputStream(cs, sectionDigest); - DataInputStream in = new DataInputStream(dis); - - for (int subsection = 0; subsection < subsections; subsection++) - readFile(in, compressor, type, csize); - - byte[] headerHash = header.getHashNoClone(); - checkHashMatch(headerHash, sectionDigest.digest()); - if (header.getType() != SectionType.SIGNATURE) { - calculatedHashes.add(headerHash); - } + return contentStream = new JarOutputStream( + new BufferedOutputStream( + new FileOutputStream(computeRealPath("classes")))); } - public void readFile(DataInputStream in, - Compressor compressor, - SectionType type, - int csize) - throws IOException - { - switch (compressor) { - case NONE: - if (type == SectionType.MODULE_INFO) { - moduleInfoBytes = readModuleInfo(in, csize); - - } else if (type == SectionType.SIGNATURE) { - // Examine the Signature header - int signatureTypeValue = (int)in.readShort(); - try { - moduleSignatureType = - SignatureType.valueOf(signatureTypeValue); - } catch (IllegalArgumentException x) { - throw new IOException("Invalid signature type: " + - signatureTypeValue); - } - int length = in.readInt(); - moduleSignatureBytes = readModuleSignature(in, csize - 6); - if (length != moduleSignatureBytes.length) { - throw new IOException("Invalid Signature length"); - } - } else { - readUncompressedFile(in, type, csize); - } - break; - case GZIP: - readGZIPCompressedFile(in, type); - break; - case PACK200_GZIP: - readClasses( - new DataInputStream(new CountingInputStream(in, csize))); - break; - default: - throw new IOException("Unsupported Compressor for files: " + - compressor); - } - } - - public void readClasses(DataInputStream in) throws IOException { - unpack200gzip(in); - } - private File currentPath = null; - private OutputStream openOutputStream(SectionType type, - String path) + private OutputStream openOutputStream(SectionType type, String path) throws IOException { - if (!extract) - return new NullOutputStream(); currentPath = null; - assert type != SectionType.CLASSES; - if (type == SectionType.RESOURCES) - return Files.newOutputStream(contentStream(), path); + if (type == CLASSES || type == RESOURCES) + return Files.newOutputStream(contentStream(), deflate, path); currentPath = computeRealPath(type, path); File parent = currentPath.getParentFile(); if (!parent.exists()) @@ -428,81 +284,21 @@ return new BufferedOutputStream(new FileOutputStream(currentPath)); } - private static class NullOutputStream extends OutputStream { - @Override - public void write(int b) throws IOException {} - @Override - public void write(byte[] b) throws IOException {} - @Override - public void write(byte[] b, int off, int len) throws IOException {} - } - - public void readGZIPCompressedFile(DataInputStream in, - SectionType type) + private static void checkHashMatch(byte[] expected, byte[] computed) throws IOException { - SubSectionFileHeader header = SubSectionFileHeader.read(in); - int csize = header.getCSize(); - - // Splice off the compressed file from input stream - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - copyStream(new CountingInputStream(in, csize), baos, csize); - - byte[] compressedfile = baos.toByteArray(); - ByteArrayInputStream bain - = new ByteArrayInputStream(compressedfile); - try (GZIPInputStream gin = new GZIPInputStream(bain); - OutputStream out = openOutputStream(type, header.getPath())) { - copyStream(gin, out); - } - - if (extract) - postExtract(type, currentPath); + if (!MessageDigest.isEqual(expected, computed)) + throw new IOException("Expected hash " + + hashHexString(expected) + + " instead of " + + hashHexString(computed)); } - public void readUncompressedFile(DataInputStream in, - SectionType type, - int csize) - throws IOException - { - assert type != SectionType.MODULE_INFO; - SubSectionFileHeader header = SubSectionFileHeader.read(in); - csize = header.getCSize(); - try (OutputStream out = openOutputStream(type, header.getPath())) { - CountingInputStream cin = new CountingInputStream(in, csize); - byte[] buf = new byte[8192]; - int n; - while ((n = cin.read(buf)) >= 0) - out.write(buf, 0, n); - } - if (extract) { - postExtract(type, currentPath); - } - } - - public byte[] readModuleInfo(DataInputStream in, int csize) - throws IOException - { - CountingInputStream cin = new CountingInputStream(in, csize); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - byte[] buf = new byte[8192]; - int n; - while ((n = cin.read(buf)) >= 0) - out.write(buf, 0, n); - return out.toByteArray(); - } - - public byte[] readModuleSignature(DataInputStream in, int csize) - throws IOException - { - return readModuleInfo(in, csize); // signature has the same format - } - // Track files installed outside the module library. For later removal. // files are relative to the modules directory. private PrintWriter filesWriter; - private void trackFiles(SectionType type, File file) + private void trackFiles(File file) throws IOException { if (file == null || file.toPath().startsWith(destination.toPath())) @@ -549,11 +345,11 @@ // Returns the absolute path of the given section type. private File getDirOfSection(SectionType type) { - if (type == SectionType.NATIVE_LIBS) + if (type == NATIVE_LIBS) return natlibs; - else if (type == SectionType.NATIVE_CMDS) + else if (type == NATIVE_CMDS) return natcmds; - else if (type == SectionType.CONFIG) + else if (type == CONFIG) return configs; // resolve sub dir section paths against the modules directory @@ -584,12 +380,9 @@ private static void markNativeCodeExecutable(SectionType type, File file) { - if (type == SectionType.NATIVE_CMDS - || (type == SectionType.NATIVE_LIBS - && System.getProperty("os.name").startsWith("Windows"))) - { - file.setExecutable(true); - } + if (type == NATIVE_CMDS || (type == NATIVE_LIBS && + System.getProperty("os.name").startsWith("Windows"))) + file.setExecutable(true); } private void postExtract(SectionType type, File path) @@ -596,30 +389,15 @@ throws IOException { markNativeCodeExecutable(type, path); - trackFiles(type, path); + trackFiles(path); } - - private void unpack200gzip(DataInputStream in) throws IOException { - GZIPInputStream gis = new GZIPInputStream(in) { - public void close() throws IOException {} - }; - Pack200.Unpacker unpacker = Pack200.newUnpacker(); - if (deflate) { - Map p = unpacker.properties(); - p.put(Pack200.Unpacker.DEFLATE_HINT, Pack200.Unpacker.TRUE); - } - unpacker.unpack(gis, contentStream()); - } - } private static void checkCompressor(SectionType type, Compressor compressor) { - if ((SectionType.MODULE_INFO == type && - Compressor.NONE != compressor) - || (SectionType.CLASSES == type && - Compressor.PACK200_GZIP != compressor)) + if ((MODULE_INFO == type && Compressor.NONE != compressor) || + (CLASSES == type && Compressor.PACK200_GZIP != compressor)) throw new IllegalArgumentException(type + " may not use compressor " + compressor); @@ -635,45 +413,15 @@ throw new IllegalArgumentException(type + " subsection count is 0"); } - private static void copyStream(InputStream in, DataOutput out) - throws IOException - { - - byte[] buffer = new byte[1024 * 8]; - for (int b_read = in.read(buffer); - -1 != b_read; - b_read = in.read(buffer)) - out.write(buffer, 0, b_read); - } - private static void copyStream(InputStream in, OutputStream out) throws IOException { - copyStream(in, (DataOutput) new DataOutputStream(out)); + byte[] buf = new byte[8192]; + int read; + while ((read = in.read(buf)) > 0) + out.write(buf, 0, read); } - private static void copyStream(InputStream in, DataOutput out, - int count) - throws IOException - { - byte[] buffer = new byte[1024 * 8]; - - while(count > 0) { - int b_read = in.read(buffer, 0, Math.min(count, buffer.length)); - if (-1 == b_read) - return; - out.write(buffer, 0, b_read); - count-=b_read; - } - } - - private static void copyStream(InputStream in, OutputStream out, - int count) - throws IOException - { - copyStream(in, (DataOutput) new DataOutputStream(out), count); - } - private static void ensureNonNegativity(long size, String parameter) { if (size < 0) throw new IllegalArgumentException(parameter + "<0: " + size); @@ -804,7 +552,6 @@ private static byte[] readFileHash(DigestInputStream dis) throws IOException { - DataInputStream in = new DataInputStream(dis); final short hashLength = readHashLength(in); @@ -1056,6 +803,56 @@ return new SubSectionFileHeader(csize, path); } } + + public final static class SignatureSection { + private final int signatureType; // One of FileConstants.ModuleFile.HashType + private final int signatureLength; // Length of signature + private final byte[] signature; // Signature bytes + + public int getSignatureType() { + return signatureType; + } + + public int getSignatureLength() { + return signatureLength; + } + + public byte[] getSignature() { + return signature; + } + + public SignatureSection(int signatureType, int signatureLength, + byte[] signature) { + ensureNonNegativity(signatureLength, "signatureLength"); + + this.signatureType = signatureType; + this.signatureLength = signatureLength; + this.signature = signature.clone(); + } + + public void write(DataOutput out) throws IOException { + out.writeShort(signatureType); + out.writeInt(signatureLength); + out.write(signature); + } + + public static SignatureSection read(DataInputStream in) + throws IOException + { + short signatureType = in.readShort(); + try { + SignatureType.valueOf(signatureType); + } catch (IllegalArgumentException x) { + throw new IOException("Invalid signature type: " + signatureType); + } + final int signatureLength = in.readInt(); + ensureNonNegativity(signatureLength, "signatureLength"); + final byte[] signature = new byte[signatureLength]; + in.readFully(signature); + return new SignatureSection(signatureType, signatureLength, + signature); + } + } private static void writeHash(DataOutput out, byte[] hash) throws IOException --- old/src/share/classes/org/openjdk/jigsaw/cli/Signer.java Fri Jun 15 22:43:44 2012 +++ new/src/share/classes/org/openjdk/jigsaw/cli/Signer.java Fri Jun 15 22:43:43 2012 @@ -32,6 +32,7 @@ import java.nio.file.StandardCopyOption; import java.security.*; import java.security.cert.X509Certificate; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import javax.security.auth.DestroyFailedException; @@ -43,7 +44,9 @@ import static java.security.KeyStore.PrivateKeyEntry; import org.openjdk.jigsaw.*; -import org.openjdk.jigsaw.ModuleFile.Reader; +import org.openjdk.jigsaw.ModuleFileParser; +import org.openjdk.jigsaw.ModuleFileParserException; +import org.openjdk.jigsaw.ModuleFileParser.Event; import org.openjdk.internal.joptsimple.OptionException; import org.openjdk.internal.joptsimple.OptionParser; import org.openjdk.internal.joptsimple.OptionSet; @@ -215,17 +218,27 @@ } // First, read in module file and calculate hashes - List hashes = null; - byte[] moduleInfoBytes = null; - try (FileInputStream mfis = new FileInputStream(moduleFile); - Reader reader = new Reader(new DataInputStream(mfis))) - { - moduleInfoBytes = reader.readStart(); - if (reader.hasSignature()) - throw new Command.Exception("module file is already signed"); - reader.readRest(); - hashes = reader.getCalculatedHashes(); - } catch (IOException x) { + List hashes = new ArrayList<>(); + int moduleInfoLength = 0; + try (FileInputStream mfis = new FileInputStream(moduleFile)) { + ModuleFileParser parser = ModuleFile.newParser(mfis); + hashes.add(parser.getHash()); + while (parser.hasNext()) { + Event event = parser.next(); + if (event == Event.END_SECTION) { + SectionHeader header = parser.getSectionHeader(); + if (header.getType() == SectionType.SIGNATURE) + throw new Command.Exception("module file is already signed"); + if (header.getType() == SectionType.MODULE_INFO) + moduleInfoLength = header.getCSize(); + checkHashMatch(header.getHash(), parser.getHash()); + hashes.add(parser.getHash()); + } else if (event == Event.END_FILE) { + checkHashMatch(parser.fileHeader().getHash(), parser.getHash()); + hashes.add(parser.getHash()); + } + } + } catch (IOException | ModuleFileParserException x) { throw new Command.Exception("unable to read module file", x); } @@ -234,7 +247,7 @@ ? new File(moduleFile + ".sig") : signedModuleFile; try (RandomAccessFile mraf = new RandomAccessFile(moduleFile, "r"); RandomAccessFile raf = new RandomAccessFile(tmpFile, "rw")) - { + { raf.setLength(0); // Transfer header and module-info from module file @@ -241,7 +254,7 @@ // to signed module file. long remainderStart = ModuleFileHeader.LENGTH + SectionHeader.LENGTH - + moduleInfoBytes.length; + + moduleInfoLength; FileChannel source = mraf.getChannel(); FileChannel dest = raf.getChannel(); for (long pos = 0; pos < remainderStart;) { @@ -275,12 +288,31 @@ } } + private void checkHashMatch(byte[] expected, byte[] computed) { + if (!MessageDigest.isEqual(expected, computed)) + throw new ModuleFileParserException("Expected hash " + + hashHexString(expected) + + " instead of " + + hashHexString(computed)); + } + + private String hashHexString(byte[] hash) { + StringBuilder hex = new StringBuilder("0x"); + for (int i = 0; i < hash.length; i++) { + int val = (hash[i] & 0xFF); + if (val <= 16) + hex.append("0"); + hex.append(Integer.toHexString(val)); + } + return hex.toString(); + } + private PrivateKeyEntry getPrivateKeyEntry(String signer) throws GeneralSecurityException, IOException { PasswordProtection storePassword = null; PasswordProtection keyPassword = null; - + try (InputStream inStream = new FileInputStream(keystore)) { // Prompt user for the keystore password (except when @@ -317,7 +349,7 @@ // Otherwise prompt the user for key password err.print("Enter password for '" + signer + "' key: "); err.flush(); - keyPassword = + keyPassword = new PasswordProtection(Password.readPassword(in)); return (PrivateKeyEntry)ks.getEntry(signer, keyPassword); } --- old/src/share/classes/org/openjdk/jigsaw/Library.java Fri Jun 15 22:43:45 2012 +++ new/src/share/classes/org/openjdk/jigsaw/Library.java Fri Jun 15 22:43:44 2012 @@ -235,6 +235,9 @@ * * @throws SignatureException * If an error occurs while validating the signature + * + * @throws ModuleFileParserException + * If there is an error processing one the underlying module files */ public abstract void install(Collection mfs, boolean verifySignature) throws ConfigurationException, IOException, SignatureException; @@ -263,7 +266,7 @@ * * @param res * A {@link Resolution} previously computed by the - * {@link Library#install install()} method + * {@link Library#resolve resolve} method * * @param verifySignature * Perform signature verification, if true @@ -276,6 +279,10 @@ * * @throws SignatureException * If an error occurs while validating the signature + * + * @throws ModuleFileParserException + * If there is an error processing the underlying module file + * required by the given resolution */ public abstract void install(Resolution res, boolean verifySignature) throws ConfigurationException, IOException, SignatureException; @@ -285,7 +292,7 @@ * * @param mids * The module identifiers - * + * * @param dry * Perform a dry run (no changes to the module library), if true. * Otherwise the modules may be removed. --- old/src/share/classes/org/openjdk/jigsaw/SimpleLibrary.java Fri Jun 15 22:43:46 2012 +++ new/src/share/classes/org/openjdk/jigsaw/SimpleLibrary.java Fri Jun 15 22:43:45 2012 @@ -1159,8 +1159,7 @@ DataInputStream in = new DataInputStream(bin); ModuleInfo mi = null; try (ModuleFile.Reader mr = new ModuleFile.Reader(in)) { - byte[] mib = mr.readStart(); - ModuleInfo moduleInfo = jms.parseModuleInfo(mib); + ModuleInfo moduleInfo = jms.parseModuleInfo(mr.getModuleInfoBytes()); File md = moduleDictionary.add(moduleInfo); mi = moduleInfo; if (verifySignature && mr.hasSignature()) { @@ -1183,8 +1182,8 @@ // Verify the module header hash and the module info hash sm.verifyHashesStart(); - // Read the rest of the hashes - mr.readRest(md, isDeflated(), natlibs(), natcmds(), configs()); + // Extract remainder of the module file, and calculate hashes + mr.extractTo(md, isDeflated(), natlibs(), natcmds(), configs()); // Verify the rest of the hashes sm.verifyHashesRest(); @@ -1192,7 +1191,7 @@ // Store signer info new Signers(md, signers).store(); } else { - mr.readRest(md, isDeflated(), natlibs(), natcmds(), configs()); + mr.extractTo(md, isDeflated(), natlibs(), natcmds(), configs()); } if (strip) @@ -1201,7 +1200,8 @@ return mi.id(); - } catch (ConfigurationException | IOException | SignatureException x) { + } catch (ConfigurationException | IOException | SignatureException | + ModuleFileParserException x) { // ## should we catch Throwable if (mi != null) { try { moduleDictionary.remove(mi); @@ -1391,7 +1391,8 @@ mids.add(installWhileLocked(mf, verifySignature, strip)); configureWhileModuleDirectoryLocked(mids); complete = true; - } catch (ConfigurationException | IOException | SignatureException x) { + } catch (ConfigurationException | IOException | SignatureException | + ModuleFileParserException x) { // ## catch throwable?? try { for (ModuleId mid : mids) { ModuleInfo mi = readLocalModuleInfo(mid); @@ -1413,7 +1414,8 @@ @Override public void install(Collection mfs, boolean verifySignature) - throws ConfigurationException, IOException, SignatureException + throws ConfigurationException, IOException, SignatureException, + ModuleFileParserException { install(mfs, verifySignature, false); } @@ -1420,6 +1422,7 @@ // Public entry point, since the Resolver itself is package-private // + @Override public Resolution resolve(Collection midqs) throws ConfigurationException, IOException { @@ -1467,7 +1470,8 @@ // configureWhileModuleDirectoryLocked(res.modulesNeeded()); complete = true; - } catch (ConfigurationException | IOException | SignatureException x) { + } catch (ConfigurationException | IOException | SignatureException | + ModuleFileParserException x) { // ## catch throwable?? try { for (ModuleId mid : res.modulesNeeded()) { ModuleInfo mi = readLocalModuleInfo(mid); @@ -1489,7 +1493,8 @@ @Override public void install(Resolution res, boolean verifySignature) - throws ConfigurationException, IOException, SignatureException + throws ConfigurationException, IOException, SignatureException, + ModuleFileParserException { install(res, verifySignature, false); } @@ -1568,7 +1573,7 @@ throw new ConfigurationException(mid + ": being used by " + rootid); } - } + } } } --- old/src/share/classes/org/openjdk/jigsaw/cli/Librarian.java Fri Jun 15 22:43:47 2012 +++ new/src/share/classes/org/openjdk/jigsaw/cli/Librarian.java Fri Jun 15 22:43:46 2012 @@ -155,24 +155,21 @@ while (hasArg()) { File module = new File(takeArg()); File destination = null; - try (FileInputStream fis = new FileInputStream(module); - DataInputStream dis = new DataInputStream(fis); - ModuleFile.Reader reader = new ModuleFile.Reader(dis)) { + try (FileInputStream fis = new FileInputStream(module)) { + ModuleFile.Reader reader = new ModuleFile.Reader(fis); - ModuleInfo mi = jms.parseModuleInfo(reader.readStart()); + ModuleInfo mi = jms.parseModuleInfo(reader.getModuleInfoBytes()); destination = new File(mi.id().name()); Path path = destination.toPath(); Files.deleteIfExists(path); Files.createDirectory(path); - reader.readRest(destination, false); - } - catch (IOException x) { + reader.extractTo(destination, false); + } catch (IOException | ModuleFileParserException x) { // Try to cleanup if an exception is thrown if (destination != null && destination.exists()) try { FilePaths.deleteTree(destination.toPath()); - } - catch (IOException y) { + } catch (IOException y) { throw (Command.Exception) new Command.Exception(y).initCause(x); } @@ -205,10 +202,8 @@ command); try { lib.installFromManifests(mfs, strip); - } catch (ConfigurationException x) { + } catch (ConfigurationException | IOException x) { throw new Command.Exception(x); - } catch (IOException x) { - throw new Command.Exception(x); } return; } @@ -224,12 +219,9 @@ finishArgs(); try { lib.install(fs, verifySignature, strip); - } catch (ConfigurationException x) { + } catch (ConfigurationException | IOException | + SignatureException | ModuleFileParserException x) { throw new Command.Exception(x); - } catch (IOException x) { - throw new Command.Exception(x); - } catch (SignatureException x) { - throw new Command.Exception(x); } return; } @@ -269,7 +261,8 @@ if (dry) return; lib.install(res, verifySignature, strip); - } catch (ConfigurationException | IOException | SignatureException x) { + } catch (ConfigurationException | IOException | SignatureException | + ModuleFileParserException x) { throw new Command.Exception(x); } @@ -320,7 +313,7 @@ lib.remove(mids, dry); } catch (ConfigurationException x) { throw new Command.Exception(x); - } catch (IOException x) { + } catch (IOException x) { if (!quiet) { for (Throwable t : x.getSuppressed()) err.format("Warning: %s%n", t.getMessage()); --- old/test/org/openjdk/jigsaw/cli/ModuleFormatTestLeftOverBytes.java Fri Jun 15 22:43:48 2012 +++ new/test/org/openjdk/jigsaw/cli/ModuleFormatTestLeftOverBytes.java Fri Jun 15 22:43:47 2012 @@ -59,7 +59,7 @@ System.err.println("Test: Empty module"); count++; reset(); - List files = new ArrayList(); + List files = new ArrayList<>(); addFile(files, createFile("module-info.java", moduleinfo)); compile(files); compress(MNAME); @@ -66,9 +66,12 @@ append(MNAME, MVER, "These bytes are not made for parsing!"); try { extract(MNAME, MVER); - } catch (IllegalArgumentException e) { - /* swallow expected IllegalArgumentException */ - if (! e.getMessage().startsWith("No SectionType")) + } catch (Exception e) { + // ## We don't need to be so strict here about the specific message + /* expected ModuleFileParserException wrapped by a CommandException */ + Throwable cause = e.getCause(); + if (!(cause instanceof ModuleFileParserException) || + ! cause.getCause().getMessage().startsWith("No SectionType")) throw e; } } @@ -91,10 +94,10 @@ */ void append(String name, String version, String content) throws Exception { String fname = moduleDir + File.separator + name + "@" + version + ".jmod"; - RandomAccessFile module = new RandomAccessFile(fname, "rw"); - module.seek(module.length()); - module.writeUTF(content); - module.close(); + try (RandomAccessFile module = new RandomAccessFile(fname, "rw")) { + module.seek(module.length()); + module.writeUTF(content); + } } /** @@ -117,7 +120,7 @@ void compress(String name, boolean haveNatLibs, boolean haveNatCmds, boolean haveConfig) throws Exception { - List args = new ArrayList(); + List args = new ArrayList<>(); args.add("-m"); args.add(classesDir.getAbsolutePath()); args.add("-d"); @@ -143,16 +146,17 @@ * Compile a list of files. */ void compile(List files) { - List options = new ArrayList(); - options.addAll(Arrays.asList("-source", "7", "-d", classesDir.getPath())); + List options = new ArrayList<>(); + options.addAll(Arrays.asList("-source", "8", "-d", classesDir.getPath())); for (File f: files) options.add(f.getPath()); String[] opts = options.toArray(new String[options.size()]); StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - int rc = com.sun.tools.javac.Main.compile(opts, pw); - pw.close(); + int rc; + try (PrintWriter pw = new PrintWriter(sw)) { + rc = com.sun.tools.javac.Main.compile(opts, pw); + } String out = sw.toString(); if (out.trim().length() > 0) @@ -178,9 +182,9 @@ return null; File file = new File(srcDir, path); file.getAbsoluteFile().getParentFile().mkdirs(); - FileWriter out = new FileWriter(file); - out.write(body); - out.close(); + try (FileWriter out = new FileWriter(file)) { + out.write(body); + } return file; } --- old/make/java/java/FILES_java.gmk Fri Jun 15 22:43:49 2012 +++ new/make/java/java/FILES_java.gmk Fri Jun 15 22:43:48 2012 @@ -543,6 +543,9 @@ org/openjdk/jigsaw/LocatableCatalog.java \ org/openjdk/jigsaw/Manifest.java \ org/openjdk/jigsaw/ModuleFile.java \ + org/openjdk/jigsaw/ModuleFileParser.java \ + org/openjdk/jigsaw/ModuleFileParserException.java \ + org/openjdk/jigsaw/ModuleFileParserImpl.java \ org/openjdk/jigsaw/ModuleFileWriter.java \ org/openjdk/jigsaw/PathContext.java \ org/openjdk/jigsaw/PathLinker.java \