# HG changeset patch # User sherman # Date 1537292581 25200 # Tue Sep 18 10:43:01 2018 -0700 # Node ID 4814af49ecbb7abf5cfad9c7aa25e6323c39f286 # Parent 2c35c998a216dc55643687d530e893b9d2f64857 8034802: (zipfs) newFileSystem throws UOE when the zip file is located in a custom file system Reviewed-by: xiaofeya, clanger diff --git a/src/jdk.zipfs/share/classes/jdk/nio/zipfs/ByteArrayChannel.java b/src/jdk.zipfs/share/classes/jdk/nio/zipfs/ByteArrayChannel.java new file mode 100644 --- /dev/null +++ b/src/jdk.zipfs/share/classes/jdk/nio/zipfs/ByteArrayChannel.java @@ -0,0 +1,252 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package jdk.nio.zipfs; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.NonWritableChannelException; +import java.nio.channels.SeekableByteChannel; +import java.util.Arrays; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public class ByteArrayChannel implements SeekableByteChannel { + + private final ReadWriteLock rwlock = new ReentrantReadWriteLock(); + private byte buf[]; + + /* + * The current position of this channel. + */ + private int pos; + + /* + * The index that is one greater than the last valid byte in the channel. + */ + private int last; + + private boolean closed; + private boolean readonly; + + /* + * Creates a {@code ByteArrayChannel} with size {@code sz}. + */ + ByteArrayChannel(int sz, boolean readonly) { + this.buf = new byte[sz]; + this.pos = this.last = 0; + this.readonly = readonly; + } + + /* + * Creates a ByteArrayChannel with its 'pos' at 0 and its 'last' at buf's end. + * Note: no defensive copy of the 'buf', used directly. + */ + ByteArrayChannel(byte[] buf, boolean readonly) { + this.buf = buf; + this.pos = 0; + this.last = buf.length; + this.readonly = readonly; + } + + @Override + public boolean isOpen() { + return !closed; + } + + @Override + public long position() throws IOException { + beginRead(); + try { + ensureOpen(); + return pos; + } finally { + endRead(); + } + } + + @Override + public SeekableByteChannel position(long pos) throws IOException { + beginWrite(); + try { + ensureOpen(); + if (pos < 0 || pos >= Integer.MAX_VALUE) + throw new IllegalArgumentException("Illegal position " + pos); + this.pos = Math.min((int)pos, last); + return this; + } finally { + endWrite(); + } + } + + @Override + public int read(ByteBuffer dst) throws IOException { + beginWrite(); + try { + ensureOpen(); + if (pos == last) + return -1; + int n = Math.min(dst.remaining(), last - pos); + dst.put(buf, pos, n); + pos += n; + return n; + } finally { + endWrite(); + } + } + + @Override + public SeekableByteChannel truncate(long size) throws IOException { + if (readonly) + throw new NonWritableChannelException(); + ensureOpen(); + throw new UnsupportedOperationException(); + } + + @Override + public int write(ByteBuffer src) throws IOException { + if (readonly) + throw new NonWritableChannelException(); + beginWrite(); + try { + ensureOpen(); + int n = src.remaining(); + ensureCapacity(pos + n); + src.get(buf, pos, n); + pos += n; + if (pos > last) { + last = pos; + } + return n; + } finally { + endWrite(); + } + } + + @Override + public long size() throws IOException { + beginRead(); + try { + ensureOpen(); + return last; + } finally { + endRead(); + } + } + + @Override + public void close() throws IOException { + if (closed) + return; + beginWrite(); + try { + closed = true; + buf = null; + pos = 0; + last = 0; + } finally { + endWrite(); + } + } + + /** + * Creates a newly allocated byte array. Its size is the current + * size of this channel and the valid contents of the buffer + * have been copied into it. + * + * @return the current contents of this channel, as a byte array. + */ + public byte[] toByteArray() { + beginRead(); + try { + // avoid copy if last == bytes.length? + return Arrays.copyOf(buf, last); + } finally { + endRead(); + } + } + + private void ensureOpen() throws IOException { + if (closed) + throw new ClosedChannelException(); + } + + private final void beginWrite() { + rwlock.writeLock().lock(); + } + + private final void endWrite() { + rwlock.writeLock().unlock(); + } + + private final void beginRead() { + rwlock.readLock().lock(); + } + + private final void endRead() { + rwlock.readLock().unlock(); + } + + private void ensureCapacity(int minCapacity) { + // overflow-conscious code + if (minCapacity - buf.length > 0) { + grow(minCapacity); + } + } + + /** + * The maximum size of array to allocate. + * Some VMs reserve some header words in an array. + * Attempts to allocate larger arrays may result in + * OutOfMemoryError: Requested array size exceeds VM limit + */ + private static final int MAX_ARRAY_SIZE = Integer.MAX_VALUE - 8; + + /** + * Increases the capacity to ensure that it can hold at least the + * number of elements specified by the minimum capacity argument. + * + * @param minCapacity the desired minimum capacity + */ + private void grow(int minCapacity) { + // overflow-conscious code + int oldCapacity = buf.length; + int newCapacity = oldCapacity << 1; + if (newCapacity - minCapacity < 0) + newCapacity = minCapacity; + if (newCapacity - MAX_ARRAY_SIZE > 0) + newCapacity = hugeCapacity(minCapacity); + buf = Arrays.copyOf(buf, newCapacity); + } + + private static int hugeCapacity(int minCapacity) { + if (minCapacity < 0) // overflow + throw new OutOfMemoryError(); + return (minCapacity > MAX_ARRAY_SIZE) ? + Integer.MAX_VALUE : + MAX_ARRAY_SIZE; + } +} diff --git a/src/jdk.zipfs/share/classes/jdk/nio/zipfs/ZipFileSystem.java b/src/jdk.zipfs/share/classes/jdk/nio/zipfs/ZipFileSystem.java --- a/src/jdk.zipfs/share/classes/jdk/nio/zipfs/ZipFileSystem.java +++ b/src/jdk.zipfs/share/classes/jdk/nio/zipfs/ZipFileSystem.java @@ -30,6 +30,7 @@ import java.io.ByteArrayOutputStream; import java.io.EOFException; import java.io.File; +import java.io.FilterOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -70,33 +71,34 @@ private final ZipFileSystemProvider provider; private final Path zfpath; final ZipCoder zc; + private final ZipPath rootdir; + private boolean readOnly = false; // readonly file system + + // configurable by env map private final boolean noExtt; // see readExtra() - private final ZipPath rootdir; - // configurable by env map private final boolean useTempFile; // use a temp file for newOS, default // is to use BAOS for better performance - private boolean readOnly = false; // readonly file system private static final boolean isWindows = AccessController.doPrivileged( (PrivilegedAction) () -> System.getProperty("os.name") .startsWith("Windows")); private final boolean forceEnd64; + private final int defaultMethod; // METHOD_STORED if "noCompression=true" + // METHOD_DEFLATED otherwise ZipFileSystem(ZipFileSystemProvider provider, Path zfpath, Map env) throws IOException { - // create a new zip if not exists - boolean createNew = "true".equals(env.get("create")); // default encoding for name/comment String nameEncoding = env.containsKey("encoding") ? (String)env.get("encoding") : "UTF-8"; this.noExtt = "false".equals(env.get("zipinfo-time")); - this.useTempFile = TRUE.equals(env.get("useTempFile")); - this.forceEnd64 = "true".equals(env.get("forceZIP64End")); - this.provider = provider; - this.zfpath = zfpath; + this.useTempFile = isTrue(env, "useTempFile"); + this.forceEnd64 = isTrue(env, "forceZIP64End"); + this.defaultMethod = isTrue(env, "noCompression") ? METHOD_STORED: METHOD_DEFLATED; if (Files.notExists(zfpath)) { - if (createNew) { + // create a new zip if not exists + if (isTrue(env, "create")) { try (OutputStream os = Files.newOutputStream(zfpath, CREATE_NEW, WRITE)) { new END().write(os, 0, forceEnd64); } @@ -122,6 +124,13 @@ } throw x; } + this.provider = provider; + this.zfpath = zfpath; + } + + // returns true if there is a name=true/"true" setting in env + private static boolean isTrue(Map env, String name) { + return "true".equals(env.get(name)) || TRUE.equals(env.get(name)); } @Override @@ -254,22 +263,23 @@ try { if (!isOpen) return; - isOpen = false; // set closed + isOpen = false; // set closed } finally { endWrite(); } - if (!streams.isEmpty()) { // unlock and close all remaining streams + if (!streams.isEmpty()) { // unlock and close all remaining streams Set copy = new HashSet<>(streams); for (InputStream is: copy) is.close(); } - beginWrite(); // lock and sync + beginWrite(); // lock and sync try { AccessController.doPrivileged((PrivilegedExceptionAction) () -> { sync(); return null; }); - ch.close(); // close the ch just in case no update - } catch (PrivilegedActionException e) { // and sync dose not close the ch + ch.close(); // close the ch just in case no update + // and sync didn't close the ch + } catch (PrivilegedActionException e) { throw (IOException)e.getException(); } finally { endWrite(); @@ -316,8 +326,8 @@ IndexNode inode = getInode(path); if (inode == null) return null; - e = new Entry(inode.name, inode.isdir); // pseudo directory - e.method = METHOD_STORED; // STORED for dir + // pseudo directory, uses METHOD_STORED + e = new Entry(inode.name, inode.isdir, METHOD_STORED); e.mtime = e.atime = e.ctime = zfsDefaultTimeStamp; } } finally { @@ -425,8 +435,7 @@ if (dir.length == 0 || exists(dir)) // root dir, or exiting dir throw new FileAlreadyExistsException(getString(dir)); checkParents(dir); - Entry e = new Entry(dir, Entry.NEW, true); - e.method = METHOD_STORED; // STORED for dir + Entry e = new Entry(dir, Entry.NEW, true, METHOD_STORED); update(e); } finally { endWrite(); @@ -467,7 +476,7 @@ checkParents(dst); } Entry u = new Entry(eSrc, Entry.COPY); // copy eSrc entry - u.name(dst); // change name + u.name(dst); // change name if (eSrc.type == Entry.NEW || eSrc.type == Entry.FILECH) { u.type = eSrc.type; // make it the same type @@ -527,7 +536,7 @@ if (hasAppend) { InputStream is = getInputStream(e); OutputStream os = getOutputStream(new Entry(e, Entry.NEW)); - copyStream(is, os); + is.transferTo(os); is.close(); return os; } @@ -536,7 +545,7 @@ if (!hasCreate && !hasCreateNew) throw new NoSuchFileException(getString(path)); checkParents(path); - return getOutputStream(new Entry(path, Entry.NEW, false)); + return getOutputStream(new Entry(path, Entry.NEW, false, defaultMethod)); } } finally { endRead(); @@ -572,6 +581,37 @@ throw new IllegalArgumentException("APPEND + TRUNCATE_EXISTING not allowed"); } + + // Returns an output SeekableByteChannel for either + // (1) writing the contents of a new entry, if the entry doesn't exit, or + // (2) updating/replacing the contents of an existing entry. + // Note: The content is not compressed. + private class EntryOutputChannel extends ByteArrayChannel { + Entry e; + + EntryOutputChannel(Entry e) throws IOException { + super(e.size > 0? (int)e.size : 8192, false); + this.e = e; + if (e.mtime == -1) + e.mtime = System.currentTimeMillis(); + if (e.method == -1) + e.method = defaultMethod; + // store size, compressed size, and crc-32 in datadescriptor + e.flag = FLAG_DATADESCR; + if (zc.isUTF8()) + e.flag |= FLAG_USE_UTF8; + } + + @Override + public void close() throws IOException { + e.bytes = toByteArray(); + e.size = e.bytes.length; + e.crc = -1; + super.close(); + update(e); + } + } + // Returns a Writable/ReadByteChannel for now. Might consdier to use // newFileChannel() instead, which dump the entry data into a regular // file on the default file system and create a FileChannel on top of @@ -585,57 +625,36 @@ if (options.contains(StandardOpenOption.WRITE) || options.contains(StandardOpenOption.APPEND)) { checkWritable(); - beginRead(); + beginRead(); // only need a readlock, the "update()" will obtain + // thewritelock when the channel is closed try { - final WritableByteChannel wbc = Channels.newChannel( - newOutputStream(path, options.toArray(new OpenOption[0]))); - long leftover = 0; - if (options.contains(StandardOpenOption.APPEND)) { - Entry e = getEntry(path); - if (e != null && e.size >= 0) - leftover = e.size; - } - final long offset = leftover; - return new SeekableByteChannel() { - long written = offset; - public boolean isOpen() { - return wbc.isOpen(); - } - - public long position() throws IOException { - return written; - } - - public SeekableByteChannel position(long pos) - throws IOException - { - throw new UnsupportedOperationException(); + ensureOpen(); + Entry e = getEntry(path); + if (e != null) { + if (e.isDir() || options.contains(CREATE_NEW)) + throw new FileAlreadyExistsException(getString(path)); + SeekableByteChannel sbc = + new EntryOutputChannel(new Entry(e, Entry.NEW)); + if (options.contains(APPEND)) { + try (InputStream is = getInputStream(e)) { // copyover + byte[] buf = new byte[8192]; + ByteBuffer bb = ByteBuffer.wrap(buf); + int n; + while ((n = is.read(buf)) != -1) { + bb.position(0); + bb.limit(n); + sbc.write(bb); + } + } } - - public int read(ByteBuffer dst) throws IOException { - throw new UnsupportedOperationException(); - } - - public SeekableByteChannel truncate(long size) - throws IOException - { - throw new UnsupportedOperationException(); - } + return sbc; + } + if (!options.contains(CREATE) && !options.contains(CREATE_NEW)) + throw new NoSuchFileException(getString(path)); + checkParents(path); + return new EntryOutputChannel( + new Entry(path, Entry.NEW, false, defaultMethod)); - public int write(ByteBuffer src) throws IOException { - int n = wbc.write(src); - written += n; - return n; - } - - public long size() throws IOException { - return written; - } - - public void close() throws IOException { - wbc.close(); - } - }; } finally { endRead(); } @@ -646,51 +665,10 @@ Entry e = getEntry(path); if (e == null || e.isDir()) throw new NoSuchFileException(getString(path)); - final ReadableByteChannel rbc = - Channels.newChannel(getInputStream(e)); - final long size = e.size; - return new SeekableByteChannel() { - long read = 0; - public boolean isOpen() { - return rbc.isOpen(); - } - - public long position() throws IOException { - return read; - } - - public SeekableByteChannel position(long pos) - throws IOException - { - throw new UnsupportedOperationException(); - } - - public int read(ByteBuffer dst) throws IOException { - int n = rbc.read(dst); - if (n > 0) { - read += n; - } - return n; - } - - public SeekableByteChannel truncate(long size) - throws IOException - { - throw new NonWritableChannelException(); - } - - public int write (ByteBuffer src) throws IOException { - throw new NonWritableChannelException(); - } - - public long size() throws IOException { - return size; - } - - public void close() throws IOException { - rbc.close(); - } - }; + try (InputStream is = getInputStream(e)) { + // TBD: if (e.size < NNNNN); + return new ByteArrayChannel(is.readAllBytes(), true); + } } finally { endRead(); } @@ -846,10 +824,6 @@ private Set streams = Collections.synchronizedSet(new HashSet()); - // the ex-channel and ex-path that need to close when their outstanding - // input streams are all closed by the obtainers. - private Set exChClosers = new HashSet<>(); - private Set tmppaths = Collections.synchronizedSet(new HashSet()); private Path getTempPathForEntry(byte[] path) throws IOException { Path tmpPath = createTempFileInSameDirectoryAs(zfpath); @@ -1087,7 +1061,7 @@ if (pos + CENHDR + nlen > limit) { zerror("invalid CEN header (bad header size)"); } - IndexNode inode = new IndexNode(cen, pos + CENHDR, nlen, pos); + IndexNode inode = new IndexNode(cen, pos + CENHDR, pos, nlen); inodes.put(inode, inode); // skip ext and comment @@ -1200,19 +1174,37 @@ return written; } + private long writeEntry(Entry e, OutputStream os, byte[] buf) + throws IOException { + + if (e.bytes == null && e.file == null) // dir, 0-length data + return 0; + + long written = 0; + try (OutputStream os2 = e.method == METHOD_STORED ? + new EntryOutputStreamCRC32(e, os) : new EntryOutputStreamDef(e, os)) { + if (e.bytes != null) { // in-memory + os2.write(e.bytes, 0, e.bytes.length); + } else if (e.file != null) { // tmp file + if (e.type == Entry.NEW || e.type == Entry.FILECH) { + try (InputStream is = Files.newInputStream(e.file)) { + is.transferTo(os2); + } + } + Files.delete(e.file); + tmppaths.remove(e.file); + } + } + written += e.csize; + if ((e.flag & FLAG_DATADESCR) != 0) { + written += e.writeEXT(os); + } + return written; + } + // sync the zip file system, if there is any udpate private void sync() throws IOException { - // System.out.printf("->sync(%s) starting....!%n", toString()); - // check ex-closer - if (!exChClosers.isEmpty()) { - for (ExChannelCloser ecc : exChClosers) { - if (ecc.streams.isEmpty()) { - ecc.ch.close(); - Files.delete(ecc.path); - exChClosers.remove(ecc); - } - } - } + if (!hasUpdate) return; Path tmpFile = createTempFileInSameDirectoryAs(zfpath); @@ -1238,34 +1230,7 @@ } else { // NEW, FILECH or CEN e.locoff = written; written += e.writeLOC(os); // write loc header - if (e.bytes != null) { // in-memory, deflated - os.write(e.bytes); // already - written += e.bytes.length; - } else if (e.file != null) { // tmp file - try (InputStream is = Files.newInputStream(e.file)) { - int n; - if (e.type == Entry.NEW) { // deflated already - while ((n = is.read(buf)) != -1) { - os.write(buf, 0, n); - written += n; - } - } else if (e.type == Entry.FILECH) { - // the data are not deflated, use ZEOS - try (OutputStream os2 = new EntryOutputStream(e, os)) { - while ((n = is.read(buf)) != -1) { - os2.write(buf, 0, n); - } - } - written += e.csize; - if ((e.flag & FLAG_DATADESCR) != 0) - written += e.writeEXT(os); - } - } - Files.delete(e.file); - tmppaths.remove(e.file); - } else { - // dir, 0-length data - } + written += writeEntry(e, os, buf); } elist.add(e); } catch (IOException x) { @@ -1294,27 +1259,9 @@ end.cenlen = written - end.cenoff; end.write(os, written, forceEnd64); } - if (!streams.isEmpty()) { - // - // TBD: ExChannelCloser should not be necessary if we only - // sync when being closed, all streams should have been - // closed already. Keep the logic here for now. - // - // There are outstanding input streams open on existing "ch", - // so, don't close the "cha" and delete the "file for now, let - // the "ex-channel-closer" to handle them - ExChannelCloser ecc = new ExChannelCloser( - createTempFileInSameDirectoryAs(zfpath), - ch, - streams); - Files.move(zfpath, ecc.path, REPLACE_EXISTING); - exChClosers.add(ecc); - streams = Collections.synchronizedSet(new HashSet()); - } else { - ch.close(); - Files.delete(zfpath); - } + ch.close(); + Files.delete(zfpath); Files.move(tmpFile, zfpath, REPLACE_EXISTING); hasUpdate = false; // clear } @@ -1352,16 +1299,6 @@ } } - private static void copyStream(InputStream is, OutputStream os) - throws IOException - { - byte[] copyBuf = new byte[8192]; - int n; - while ((n = is.read(copyBuf)) != -1) { - os.write(copyBuf, 0, n); - } - } - // Returns an out stream for either // (1) writing the contents of a new entry, if the entry exits, or // (2) updating/replacing the contents of the specified existing entry. @@ -1370,9 +1307,9 @@ if (e.mtime == -1) e.mtime = System.currentTimeMillis(); if (e.method == -1) - e.method = METHOD_DEFLATED; // TBD: use default method - // store size, compressed size, and crc-32 in LOC header - e.flag = 0; + e.method = defaultMethod; + // store size, compressed size, and crc-32 in datadescr + e.flag = FLAG_DATADESCR; if (zc.isUTF8()) e.flag |= FLAG_USE_UTF8; OutputStream os; @@ -1385,16 +1322,130 @@ return new EntryOutputStream(e, os); } + private class EntryOutputStream extends FilterOutputStream { + private Entry e; + private long written; + private boolean isClosed; + + EntryOutputStream(Entry e, OutputStream os) throws IOException { + super(os); + this.e = Objects.requireNonNull(e, "Zip entry is null"); + // this.written = 0; + } + + @Override + public synchronized void write(int b) throws IOException { + out.write(b); + written += 1; + } + + @Override + public synchronized void write(byte b[], int off, int len) + throws IOException { + out.write(b, off, len); + written += len; + } + + @Override + public synchronized void close() throws IOException { + if (isClosed) { + return; + } + isClosed = true; + e.size = written; + if (out instanceof ByteArrayOutputStream) + e.bytes = ((ByteArrayOutputStream)out).toByteArray(); + super.close(); + update(e); + } + } + + // Wrapper output stream class to write out a "stored" entry. + // (1) this class does not close the underlying out stream when + // being closed. + // (2) no need to be "synchronized", only used by sync() + private class EntryOutputStreamCRC32 extends FilterOutputStream { + private Entry e; + private CRC32 crc; + private long written; + private boolean isClosed; + + EntryOutputStreamCRC32(Entry e, OutputStream os) throws IOException { + super(os); + this.e = Objects.requireNonNull(e, "Zip entry is null"); + this.crc = new CRC32(); + } + + @Override + public void write(int b) throws IOException { + out.write(b); + crc.update(b); + written += 1; + } + + @Override + public void write(byte b[], int off, int len) + throws IOException { + out.write(b, off, len); + crc.update(b, off, len); + written += len; + } + + @Override + public void close() throws IOException { + if (isClosed) + return; + isClosed = true; + e.size = e.csize = written; + e.size = crc.getValue(); + } + } + + // Wrapper output stream class to write out a "deflated" entry. + // (1) this class does not close the underlying out stream when + // being closed. + // (2) no need to be "synchronized", only used by sync() + private class EntryOutputStreamDef extends DeflaterOutputStream { + private CRC32 crc; + private Entry e; + private boolean isClosed; + + EntryOutputStreamDef(Entry e, OutputStream os) throws IOException { + super(os, getDeflater()); + this.e = Objects.requireNonNull(e, "Zip entry is null"); + this.crc = new CRC32(); + } + + @Override + public void write(byte b[], int off, int len) + throws IOException { + super.write(b, off, len); + crc.update(b, off, len); + } + + @Override + public void close() throws IOException { + if (isClosed) + return; + isClosed = true; + finish(); + e.size = def.getBytesRead(); + e.csize = def.getBytesWritten(); + e.crc = crc.getValue(); + } + } + private InputStream getInputStream(Entry e) throws IOException { InputStream eis = null; if (e.type == Entry.NEW) { + // now bytes & file is uncompressed. if (e.bytes != null) - eis = new ByteArrayInputStream(e.bytes); + return new ByteArrayInputStream(e.bytes); else if (e.file != null) - eis = Files.newInputStream(e.file); + return Files.newInputStream(e.file); else throw new ZipException("update entry data is missing"); } else if (e.type == Entry.FILECH) { @@ -1560,87 +1611,6 @@ } } - class EntryOutputStream extends DeflaterOutputStream - { - private CRC32 crc; - private Entry e; - private long written; - private boolean isClosed = false; - - EntryOutputStream(Entry e, OutputStream os) - throws IOException - { - super(os, getDeflater()); - if (e == null) - throw new NullPointerException("Zip entry is null"); - this.e = e; - crc = new CRC32(); - } - - @Override - public synchronized void write(byte b[], int off, int len) - throws IOException - { - if (e.type != Entry.FILECH) // only from sync - ensureOpen(); - if (isClosed) { - throw new IOException("Stream closed"); - } - if (off < 0 || len < 0 || off > b.length - len) { - throw new IndexOutOfBoundsException(); - } else if (len == 0) { - return; - } - switch (e.method) { - case METHOD_DEFLATED: - super.write(b, off, len); - break; - case METHOD_STORED: - written += len; - out.write(b, off, len); - break; - default: - throw new ZipException("invalid compression method"); - } - crc.update(b, off, len); - } - - @Override - public synchronized void close() throws IOException { - if (isClosed) { - return; - } - isClosed = true; - // TBD ensureOpen(); - switch (e.method) { - case METHOD_DEFLATED: - finish(); - e.size = def.getBytesRead(); - e.csize = def.getBytesWritten(); - e.crc = crc.getValue(); - break; - case METHOD_STORED: - // we already know that both e.size and e.csize are the same - e.size = e.csize = written; - e.crc = crc.getValue(); - break; - default: - throw new ZipException("invalid compression method"); - } - //crc.reset(); - if (out instanceof ByteArrayOutputStream) - e.bytes = ((ByteArrayOutputStream)out).toByteArray(); - - if (e.type == Entry.FILECH) { - releaseDeflater(def); - return; - } - super.close(); - releaseDeflater(def); - update(e); - } - } - static void zerror(String msg) throws ZipException { throw new ZipException(msg); } @@ -1797,7 +1767,7 @@ } // constructor for cenInit() - IndexNode(byte[] cen, int noff, int nlen, int pos) { + IndexNode(byte[] cen, int noff, int pos, int nlen) { if (cen[noff + nlen - 1] == '/') { isdir = true; nlen--; @@ -1888,18 +1858,18 @@ Entry() {} - Entry(byte[] name, boolean isdir) { + Entry(byte[] name, boolean isdir, int method) { name(name); this.isdir = isdir; this.mtime = this.ctime = this.atime = System.currentTimeMillis(); this.crc = 0; this.size = 0; this.csize = 0; - this.method = METHOD_DEFLATED; + this.method = method; } - Entry(byte[] name, int type, boolean isdir) { - this(name, isdir); + Entry(byte[] name, int type, boolean isdir, int method) { + this(name, isdir, method); this.type = type; } @@ -1927,9 +1897,8 @@ } Entry (byte[] name, Path file, int type) { - this(name, type, false); + this(name, type, false, METHOD_STORED); this.file = file; - this.method = METHOD_STORED; } int version() throws ZipException { @@ -2408,6 +2377,7 @@ public String toString() { StringBuilder sb = new StringBuilder(1024); Formatter fm = new Formatter(sb); + fm.format(" name : %s%n", new String(name)); fm.format(" creationTime : %tc%n", creationTime().toMillis()); fm.format(" lastAccessTime : %tc%n", lastAccessTime().toMillis()); fm.format(" lastModifiedTime: %tc%n", lastModifiedTime().toMillis()); @@ -2425,20 +2395,6 @@ } } - private static class ExChannelCloser { - Path path; - SeekableByteChannel ch; - Set streams; - ExChannelCloser(Path path, - SeekableByteChannel ch, - Set streams) - { - this.path = path; - this.ch = ch; - this.streams = streams; - } - } - // ZIP directory has two issues: // (1) ZIP spec does not require the ZIP file to include // directory entry diff --git a/src/jdk.zipfs/share/classes/jdk/nio/zipfs/ZipFileSystemProvider.java b/src/jdk.zipfs/share/classes/jdk/nio/zipfs/ZipFileSystemProvider.java --- a/src/jdk.zipfs/share/classes/jdk/nio/zipfs/ZipFileSystemProvider.java +++ b/src/jdk.zipfs/share/classes/jdk/nio/zipfs/ZipFileSystemProvider.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009, 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2009, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -124,9 +124,6 @@ public FileSystem newFileSystem(Path path, Map env) throws IOException { - if (path.getFileSystem() != FileSystems.getDefault()) { - throw new UnsupportedOperationException(); - } ensureFile(path); try { ZipFileSystem zipfs; diff --git a/test/jdk/jdk/nio/zipfs/ZipFSTester.java b/test/jdk/jdk/nio/zipfs/ZipFSTester.java --- a/test/jdk/jdk/nio/zipfs/ZipFSTester.java +++ b/test/jdk/jdk/nio/zipfs/ZipFSTester.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2010, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -28,6 +28,7 @@ import java.net.URI; import java.net.URLDecoder; import java.nio.ByteBuffer; +import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.nio.channels.SeekableByteChannel; import java.nio.file.DirectoryStream; @@ -58,8 +59,10 @@ import java.util.Random; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.zip.CRC32; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; +import java.util.zip.ZipOutputStream; import static java.nio.file.StandardOpenOption.*; import static java.nio.file.StandardCopyOption.*; @@ -70,7 +73,7 @@ * @test * @bug 6990846 7009092 7009085 7015391 7014948 7005986 7017840 7007596 * 7157656 8002390 7012868 7012856 8015728 8038500 8040059 8069211 - * 8131067 + * 8131067 8034802 * @summary Test Zip filesystem provider * @modules jdk.zipfs * @run main ZipFSTester @@ -80,23 +83,27 @@ public class ZipFSTester { public static void main(String[] args) throws Exception { - // create JAR file for test, actual contents don't matter Path jarFile = Utils.createJarFile("tester.jar", "META-INF/MANIFEST.MF", "dir1/foo", - "dir2/bar"); + "dir2/bar", + "dir1/dir3/fooo"); try (FileSystem fs = newZipFileSystem(jarFile, Collections.emptyMap())) { test0(fs); test1(fs); test2(fs); // more tests } + + testStreamChannel(); testTime(jarFile); test8069211(); test8131067(); } + private static Random rdm = new Random(); + static void test0(FileSystem fs) throws Exception { @@ -121,13 +128,28 @@ static void test1(FileSystem fs0) throws Exception { - Random rdm = new Random(); - // clone a fs and test on it + // prepare a src for testing + Path src = getTempPath(); + String tmpName = src.toString(); + try (OutputStream os = Files.newOutputStream(src)) { + byte[] bits = new byte[12345]; + rdm.nextBytes(bits); + os.write(bits); + } + + // clone a fs from fs0 and test on it Path tmpfsPath = getTempPath(); Map env = new HashMap(); env.put("create", "true"); try (FileSystem copy = newZipFileSystem(tmpfsPath, env)) { z2zcopy(fs0, copy, "/", 0); + + // copy the test jar itself in + Files.copy(Paths.get(fs0.toString()), copy.getPath("/foo.jar")); + Path zpath = copy.getPath("/foo.jar"); + try (FileSystem zzfs = FileSystems.newFileSystem(zpath, null)) { + Files.copy(src, zzfs.getPath("/srcInjarjar")); + } } try (FileSystem fs = newZipFileSystem(tmpfsPath, new HashMap())) { @@ -142,15 +164,6 @@ throw new RuntimeException("newFileSystem(URI...) does not throw exception"); } catch (FileSystemAlreadyExistsException fsaee) {} - // prepare a src - Path src = getTempPath(); - String tmpName = src.toString(); - OutputStream os = Files.newOutputStream(src); - byte[] bits = new byte[12345]; - rdm.nextBytes(bits); - os.write(bits); - os.close(); - try { provider.newFileSystem(new File(System.getProperty("test.src", ".")).toPath(), new HashMap()); @@ -162,6 +175,8 @@ throw new RuntimeException("newFileSystem() opens a non-zip file as zipfs"); } catch (UnsupportedOperationException uoe) {} + // walk + walk(fs.getPath("/")); // copyin Path dst = getPathWithParents(fs, tmpName); @@ -236,10 +251,29 @@ // test channels channel(fs, dst); Files.delete(dst); - Files.delete(src); + + // test foo.jar in jar/zipfs #8034802 + Path jpath = fs.getPath("/foo.jar"); + System.out.println("walking: " + jpath); + try (FileSystem zzfs = FileSystems.newFileSystem(jpath, null)) { + walk(zzfs.getPath("/")); + // foojar:/srcInjarjar + checkEqual(src, zzfs.getPath("/srcInjarjar")); + + dst = getPathWithParents(zzfs, tmpName); + fchCopy(src, dst); + checkEqual(src, dst); + tmp = Paths.get(tmpName + "_Tmp"); + fchCopy(dst, tmp); // out + checkEqual(src, tmp); + Files.delete(tmp); + + channel(zzfs, dst); + Files.delete(dst); + } } finally { - if (Files.exists(tmpfsPath)) - Files.delete(tmpfsPath); + Files.deleteIfExists(tmpfsPath); + Files.deleteIfExists(src); } } @@ -383,6 +417,158 @@ Files.delete(fs3Path); } + static final int METHOD_STORED = 0; + static final int METHOD_DEFLATED = 8; + + static Object[][] getEntries() { + Object[][] entries = new Object[10 + rdm.nextInt(20)][3]; + for (int i = 0; i < entries.length; i++) { + entries[i][0] = "entries" + i; + entries[i][1] = rdm.nextInt(10) % 2 == 0 ? + METHOD_STORED : METHOD_DEFLATED; + entries[i][2] = new byte[rdm.nextInt(8192)]; + rdm.nextBytes((byte[])entries[i][2]); + } + return entries; + } + + // check the content of read from zipfs is equal to the "bytes" + private static void checkRead(Path path, byte[] expected) throws IOException { + //streams + try (InputStream is = Files.newInputStream(path)) { + if (!Arrays.equals(is.readAllBytes(), expected)) { + System.out.printf(" newInputStream <%s> failed...%n", path.toString()); + throw new RuntimeException("CHECK FAILED!"); + } + } + + // channels -- via sun.nio.ch.ChannelInputStream + try (SeekableByteChannel sbc = Files.newByteChannel(path); + InputStream is = Channels.newInputStream(sbc)) { + + // check all bytes match + if (!Arrays.equals(is.readAllBytes(), expected)) { + System.out.printf(" newByteChannel <%s> failed...%n", path.toString()); + throw new RuntimeException("CHECK FAILED!"); + } + + // Check if read position is at the end + if (sbc.position() != expected.length) { + System.out.printf("pos [%s]: size=%d, position=%d%n", + path.toString(), expected.length, sbc.position()); + throw new RuntimeException("CHECK FAILED!"); + } + + // Check position(x) + read() at the random/specific pos/len + byte[] buf = new byte[1024]; + ByteBuffer bb = ByteBuffer.wrap(buf); + for (int i = 0; i < 10; i++) { + int pos = rdm.nextInt((int)sbc.size()); + int len = rdm.nextInt(Math.min(buf.length, expected.length - pos)); + // System.out.printf(" --> %d, %d%n", pos, len); + bb.position(0).limit(len); // bb.flip().limit(len); + if (sbc.position(pos).position() != pos || + sbc.read(bb) != len || + !Arrays.equals(buf, 0, bb.position(), expected, pos, pos + len)) { + System.out.printf("read()/position() failed%n"); + } + } + } catch (IOException x) { + x.printStackTrace(); + throw new RuntimeException("CHECK FAILED!"); + } + } + + // test entry stream/channel reading + static void testStreamChannel() throws Exception { + Path zpath = getTempPath(); + try { + var crc = new CRC32(); + Object[][] entries = getEntries(); + + // [1] create zip via ZipOutputStream + try (var os = Files.newOutputStream(zpath); + var zos = new ZipOutputStream(os)) { + for (Object[] entry : entries) { + var ze = new ZipEntry((String)entry[0]); + int method = (int)entry[1]; + byte[] bytes = (byte[])entry[2]; + if (method == METHOD_STORED) { + ze.setSize(bytes.length); + crc.reset(); + crc.update(bytes); + ze.setCrc(crc.getValue()); + } + ze.setMethod(method); + zos.putNextEntry(ze); + zos.write(bytes); + zos.closeEntry(); + } + } + try (var zfs = newZipFileSystem(zpath, Collections.emptyMap())) { + for (Object[] e : entries) { + Path path = zfs.getPath((String)e[0]); + int method = (int)e[1]; + byte[] bytes = (byte[])e[2]; + // System.out.printf("checking read [%s, %d, %d]%n", + // path.toString(), bytes.length, method); + checkRead(path, bytes); + } + } + Files.deleteIfExists(zpath); + + // [2] create zip via zfs.newByteChannel + try (var zfs = newZipFileSystem(zpath, Map.of("create", "true"))) { + for (Object[] e : entries) { + // tbd: method is not used + try (var sbc = Files.newByteChannel(zfs.getPath((String)e[0]), + CREATE_NEW, WRITE)) { + sbc.write(ByteBuffer.wrap((byte[])e[2])); + } + } + } + try (var zfs = newZipFileSystem(zpath, Collections.emptyMap())) { + for (Object[] e : entries) { + checkRead(zfs.getPath((String)e[0]), (byte[])e[2]); + } + } + Files.deleteIfExists(zpath); + + // [3] create zip via Files.write()/newoutputStream/ + try (var zfs = newZipFileSystem(zpath, Map.of("create", "true"))) { + for (Object[] e : entries) { + Files.write(zfs.getPath((String)e[0]), (byte[])e[2]); + } + } + try (var zfs = newZipFileSystem(zpath, Collections.emptyMap())) { + for (Object[] e : entries) { + checkRead(zfs.getPath((String)e[0]), (byte[])e[2]); + } + } + Files.deleteIfExists(zpath); + + // [4] create zip via zfs.newByteChannel, with "method_stored" + try (var zfs = newZipFileSystem(zpath, + Map.of("create", true, "noCompression", true))) { + for (Object[] e : entries) { + try (var sbc = Files.newByteChannel(zfs.getPath((String)e[0]), + CREATE_NEW, WRITE)) { + sbc.write(ByteBuffer.wrap((byte[])e[2])); + } + } + } + try (var zfs = newZipFileSystem(zpath, Collections.emptyMap())) { + for (Object[] e : entries) { + checkRead(zfs.getPath((String)e[0]), (byte[])e[2]); + } + } + Files.deleteIfExists(zpath); + + } finally { + Files.deleteIfExists(zpath); + } + } + // test file stamp static void testTime(Path src) throws Exception { BasicFileAttributes attrs = Files @@ -392,34 +578,35 @@ Map env = new HashMap(); env.put("create", "true"); Path fsPath = getTempPath(); - FileSystem fs = newZipFileSystem(fsPath, env); + try (FileSystem fs = newZipFileSystem(fsPath, env)) { + System.out.println("test copy with timestamps..."); + // copyin + Path dst = getPathWithParents(fs, "me"); + Files.copy(src, dst, COPY_ATTRIBUTES); + checkEqual(src, dst); + System.out.println("mtime: " + attrs.lastModifiedTime()); + System.out.println("ctime: " + attrs.creationTime()); + System.out.println("atime: " + attrs.lastAccessTime()); + System.out.println(" ==============>"); + BasicFileAttributes dstAttrs = Files + .getFileAttributeView(dst, BasicFileAttributeView.class) + .readAttributes(); + System.out.println("mtime: " + dstAttrs.lastModifiedTime()); + System.out.println("ctime: " + dstAttrs.creationTime()); + System.out.println("atime: " + dstAttrs.lastAccessTime()); - System.out.println("test copy with timestamps..."); - // copyin - Path dst = getPathWithParents(fs, "me"); - Files.copy(src, dst, COPY_ATTRIBUTES); - checkEqual(src, dst); - System.out.println("mtime: " + attrs.lastModifiedTime()); - System.out.println("ctime: " + attrs.creationTime()); - System.out.println("atime: " + attrs.lastAccessTime()); - System.out.println(" ==============>"); - BasicFileAttributes dstAttrs = Files - .getFileAttributeView(dst, BasicFileAttributeView.class) - .readAttributes(); - System.out.println("mtime: " + dstAttrs.lastModifiedTime()); - System.out.println("ctime: " + dstAttrs.creationTime()); - System.out.println("atime: " + dstAttrs.lastAccessTime()); - - // 1-second granularity - if (attrs.lastModifiedTime().to(TimeUnit.SECONDS) != - dstAttrs.lastModifiedTime().to(TimeUnit.SECONDS) || - attrs.lastAccessTime().to(TimeUnit.SECONDS) != - dstAttrs.lastAccessTime().to(TimeUnit.SECONDS) || - attrs.creationTime().to(TimeUnit.SECONDS) != - dstAttrs.creationTime().to(TimeUnit.SECONDS)) { - throw new RuntimeException("Timestamp Copy Failed!"); + // 1-second granularity + if (attrs.lastModifiedTime().to(TimeUnit.SECONDS) != + dstAttrs.lastModifiedTime().to(TimeUnit.SECONDS) || + attrs.lastAccessTime().to(TimeUnit.SECONDS) != + dstAttrs.lastAccessTime().to(TimeUnit.SECONDS) || + attrs.creationTime().to(TimeUnit.SECONDS) != + dstAttrs.creationTime().to(TimeUnit.SECONDS)) { + throw new RuntimeException("Timestamp Copy Failed!"); + } + } finally { + Files.delete(fsPath); } - Files.delete(fsPath); } static void test8069211() throws Exception { @@ -624,8 +811,8 @@ // check the content of two paths are equal private static void checkEqual(Path src, Path dst) throws IOException { - //System.out.printf("checking <%s> vs <%s>...%n", - // src.toString(), dst.toString()); + System.out.printf("checking <%s> vs <%s>...%n", + src.toString(), dst.toString()); //streams byte[] bufSrc = new byte[8192]; @@ -702,6 +889,21 @@ chDst.toString(), chDst.size(), chDst.position()); throw new RuntimeException("CHECK FAILED!"); } + + // Check position(x) + read() at the specific pos/len + for (int i = 0; i < 10; i++) { + int pos = rdm.nextInt((int)chSrc.size()); + int limit = rdm.nextInt(1024); + if (chSrc.position(pos).position() != chDst.position(pos).position()) { + System.out.printf("dst/src.position(pos failed%n"); + } + bbSrc.clear().limit(limit); + bbDst.clear().limit(limit); + if (chSrc.read(bbSrc) != chDst.read(bbDst) || + !bbSrc.flip().equals(bbDst.flip())) { + System.out.printf("dst/src.read() failed%n"); + } + } } catch (IOException x) { x.printStackTrace(); }