--- /dev/null 2020-02-19 10:36:40.000000000 +0100 +++ new/test/jdk/jdk/nio/zipfs/ZipFSManifestOrderTest.java 2020-02-19 10:36:39.546634100 +0100 @@ -0,0 +1,517 @@ +/* + * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Arrays; +import java.util.Enumeration; +import java.util.Map; +import java.util.Optional; +import java.util.jar.Attributes; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.jar.JarInputStream; +import java.util.jar.Manifest; +import java.util.spi.ToolProvider; +import java.util.stream.Collectors; +import java.util.zip.ZipEntry; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertNull; +import static org.testng.Assert.assertTrue; + +/** + * @test + * @bug 8211917 + * @summary Test that irrespective of which order the META-INF/MANIFEST.MF gets added using the FileSystem APIs, + * the underlying zipfs implementation always places it at the start of the zip to allow for java.util.jar.JarInputStream + * to be able to locate it. + * @run testng ZipFSManifestOrderTest + */ +public class ZipFSManifestOrderTest { + private static final String MANIFEST_SOME_ATTR = "test"; + private static final String MANIFEST_SOME_ATTR_VALUE = "8211917"; + + private static final String OS_FILE_MANIFEST_CONTENT = "Manifest-Version: 1.0\n" + + MANIFEST_SOME_ATTR + ": " + MANIFEST_SOME_ATTR_VALUE + "\n"; + + + @DataProvider(name = "zipOptions") + private Object[][] zipOptions() { + // each Object[] contains 3 entries: + // 1. The Map used to create a new zip/jar file using zipfs + // 2. The Map used to update an existing zip/jar using zipfs + // 3. The compression method used for entries in the zip/jar + return new Object[][]{ + {Map.of("create", "true"), Map.of(), ZipEntry.DEFLATED}, + {Map.of("create", "true", "compressionMethod", "STORED"), Map.of("compressionMethod", "STORED"), ZipEntry.STORED}, + {Map.of("create", "true", "compressionMethod", "DEFLATED"), Map.of("compressionMethod", "DEFLATED"), ZipEntry.DEFLATED} + }; + } + + /** + * Use the FileSystem APIs to first add the manifest and then the rest of the files, to a jar. Then verify that the + * JarInputStream finds the manifest and the other expected entries. + * + * @throws Exception + */ + @Test(dataProvider = "zipOptions") + public void testJarWithManifestAddedFirst(final Map zipOptions, + final Map zipUpdateOptions, final int compressionMethod) + throws Exception { + final Path jarPath = Paths.get("manifest-first.jar"); + Files.deleteIfExists(jarPath); + final Entry[] entries = newEntries(compressionMethod); + try (final FileSystem zipFs = FileSystems.newFileSystem(jarPath, zipOptions)) { + // first write the manifest + final Path manifestPath = zipFs.getPath("META-INF", "MANIFEST.MF"); + writeManifest(manifestPath); + // now write some other files + for (final Entry entry : entries) { + writeEntry(zipFs, entry); + } + } + verify(jarPath, true, entries, compressionMethod); + + // now repeat the whole test again, on the same jar, but this time update the jar instead of creating it new + try (final FileSystem zipFs = FileSystems.newFileSystem(jarPath, zipUpdateOptions)) { + // first write the manifest + final Path manifestPath = zipFs.getPath("META-INF", "MANIFEST.MF"); + writeManifest(manifestPath); + // now write some other files + for (final Entry entry : entries) { + writeEntry(zipFs, entry); + } + } + verify(jarPath, true, entries, compressionMethod); + } + + /** + * Use the FileSystem APIs to add the manifest last, to a jar. Then verify that the JarInputStream finds the + * manifest and the other expected entries. + * + * @throws Exception + */ + @Test(dataProvider = "zipOptions") + public void testJarWithManifestAddedLast(final Map zipOptions, + final Map zipUpdateOptions, final int compressionMethod) + throws Exception { + final Path jarPath = Paths.get("manifest-last.jar"); + Files.deleteIfExists(jarPath); + final Entry[] entries = newEntries(compressionMethod); + try (final FileSystem zipFs = FileSystems.newFileSystem(jarPath, zipOptions)) { + // first write other files + for (final Entry entry : entries) { + writeEntry(zipFs, entry); + } + // now write the manifest + final Path manifestPath = zipFs.getPath("META-INF", "MANIFEST.MF"); + writeManifest(manifestPath); + + } + verify(jarPath, true, entries, compressionMethod); + + // now repeat the whole test again, on the same jar, but this time update the jar instead of creating it new + try (final FileSystem zipFs = FileSystems.newFileSystem(jarPath, zipUpdateOptions)) { + // first write other files + for (final Entry entry : entries) { + writeEntry(zipFs, entry); + } + // now write the manifest + final Path manifestPath = zipFs.getPath("META-INF", "MANIFEST.MF"); + writeManifest(manifestPath); + + } + verify(jarPath, true, entries, compressionMethod); + } + + /** + * Use the FileSystem APIs to add the manifest ordered somewhere in between among other files in a jar. Then verify that + * the JarInputStream finds the manifest and the other expected entries. + * + * @throws Exception + */ + @Test(dataProvider = "zipOptions") + public void testJarWithManifestAddedInBetween(final Map zipOptions, + final Map zipUpdateOptions, final int compressionMethod) + throws Exception { + final Path jarPath = Paths.get("manifest-in-between.jar"); + Files.deleteIfExists(jarPath); + final Entry[] entries = newEntries(compressionMethod); + try (final FileSystem zipFs = FileSystems.newFileSystem(jarPath, zipOptions)) { + for (int i = 0; i < entries.length; i++) { + writeEntry(zipFs, entries[i]); + if (i == entries.length - 2) { + // write out the manifest, now that some non-manifest entries have been added + final Path manifestPath = zipFs.getPath("META-INF", "MANIFEST.MF"); + writeManifest(manifestPath); + } + } + } + verify(jarPath, true, entries, compressionMethod); + + // now repeat the whole test again, on the same jar, but this time update the jar instead of creating it new + try (final FileSystem zipFs = FileSystems.newFileSystem(jarPath, zipUpdateOptions)) { + for (int i = 0; i < entries.length; i++) { + writeEntry(zipFs, entries[i]); + if (i == entries.length - 2) { + // write out the manifest, now that some non-manifest entries have been added + final Path manifestPath = zipFs.getPath("META-INF", "MANIFEST.MF"); + writeManifest(manifestPath); + } + } + } + verify(jarPath, true, entries, compressionMethod); + } + + /** + * Use the FileSystem APIs to create a jar without any manifest. Then verify that the JarInputStream can find the + * expected entries in the jar and also doesn't find any manifest in it. + * + * @throws Exception + */ + @Test(dataProvider = "zipOptions") + public void testJarWithNoManifest(final Map zipOptions, + final Map zipUpdateOptions, final int compressionMethod) + throws Exception { + final Path jarPath = Paths.get("no-manifest.jar"); + Files.deleteIfExists(jarPath); + final Entry[] entries = newEntries(compressionMethod); + try (final FileSystem zipFs = FileSystems.newFileSystem(jarPath, zipOptions)) { + for (final Entry entry : entries) { + writeEntry(zipFs, entry); + } + } + verify(jarPath, false, entries, compressionMethod); + + // now repeat the whole test again, on the same jar, but this time update the jar instead of creating it new + try (final FileSystem zipFs = FileSystems.newFileSystem(jarPath, zipUpdateOptions)) { + for (final Entry entry : entries) { + writeEntry(zipFs, entry); + } + } + verify(jarPath, false, entries, compressionMethod); + } + + /** + * Use the FileSystem APIs to add some entries to a jar and then copy over a file into the jar as + * the manifest file. Then verify that the JarInputStream finds the manifest and the other expected entries. + * + * @throws Exception + */ + @Test(dataProvider = "zipOptions") + public void testManifestCopiedFromOSFile(final Map zipOptions, + final Map zipUpdateOptions, final int compressionMethod) + throws Exception { + // write out a file containing manifest contents + final Path osFile = Paths.get(".", "test-MANIFEST.MF"); + Files.deleteIfExists(osFile); + Files.writeString(osFile, OS_FILE_MANIFEST_CONTENT); + + // copy over that file into a jar + final Path jarPath = Paths.get("manifest-copied-from-os-file.jar"); + Files.deleteIfExists(jarPath); + final Entry[] entries = newEntries(compressionMethod); + try (final FileSystem zipFs = FileSystems.newFileSystem(jarPath, zipOptions)) { + // first write other files + for (final Entry entry : entries) { + writeEntry(zipFs, entry); + } + // now copy over the manifest file + final Path manifestPath = zipFs.getPath("META-INF", "MANIFEST.MF"); + Files.copy(osFile, manifestPath); + } + verify(jarPath, true, entries, compressionMethod); + + // now repeat the whole test again, on the same jar, but this time update the jar instead of creating it new + try (final FileSystem zipFs = FileSystems.newFileSystem(jarPath, zipUpdateOptions)) { + // first write other files + for (final Entry entry : entries) { + writeEntry(zipFs, entry); + } + // now copy over the manifest file + final Path manifestPath = zipFs.getPath("META-INF", "MANIFEST.MF"); + Files.copy(osFile, manifestPath, StandardCopyOption.REPLACE_EXISTING); + } + verify(jarPath, true, entries, compressionMethod); + } + + /** + * Using zip filesystem APIs, creates a jar containing a manifest file, then copies it over to a new jar file + * and then verifies that the manifest file and other expected entries are available in the copied over jar. + * + * @throws Exception + */ + @Test(dataProvider = "zipOptions") + public void testDuplicateJar(final Map zipOptions, + final Map zipUpdateOptions, final int compressionMethod) + throws Exception { + final Path jarPath = Paths.get("manifest-in-duplicate-jar.jar"); + Files.deleteIfExists(jarPath); + final Entry[] entries = newEntries(compressionMethod); + try (final FileSystem zipFs = FileSystems.newFileSystem(jarPath, zipOptions)) { + for (int i = 0; i < entries.length; i++) { + writeEntry(zipFs, entries[i]); + if (i == entries.length - 2) { + // write out the manifest, now that some non-manifest entries have been added + final Path manifestPath = zipFs.getPath("META-INF", "MANIFEST.MF"); + writeManifest(manifestPath); + } + } + } + verify(jarPath, true, entries, compressionMethod); + // copy over the jar to a new one + final Path duplicateJar = Paths.get("duplicate.jar"); + Files.deleteIfExists(duplicateJar); + Files.copy(jarPath, duplicateJar); + // verify the duplicate jar + verify(duplicateJar, true, entries, compressionMethod); + } + + /** + * Creates a jar using the {@code jar} tool and then updates it using zip filesystem + * and then verifies that the updated jar contains the expected manifest and other entries + * + * @throws Exception + */ + @Test(dataProvider = "zipOptions") + public void testJarToolGeneratedJarWithManifest(final Map zipOptions, + final Map zipUpdateOptions, final int compressionMethod) + throws Exception { + final Optional jarToolProvider = ToolProvider.findFirst("jar"); + assertTrue(jarToolProvider.isPresent(), "jar tool is missing"); + final ToolProvider jarTool = jarToolProvider.get(); + final Path jarPath = Paths.get("jar-created-from-jar-tool.jar"); + Files.deleteIfExists(jarPath); + final Entry[] entries = newEntries(compressionMethod); + final Path srcDir = Paths.get("src"); + recursiveDelete(srcDir); + Files.createDirectory(srcDir); + final FileSystem currentFileSystem = Paths.get(".").getFileSystem(); + // first create files on the local filesystem, so that they can then + // be used by the jar tool to create the jar + for (final Entry entry : entries) { + writeEntry(currentFileSystem, srcDir, entry); + } + // write out a file containing manifest contents + final Path manifestFile = Paths.get(".", "test-jar-MANIFEST.MF"); + Files.deleteIfExists(manifestFile); + Files.writeString(manifestFile, OS_FILE_MANIFEST_CONTENT); + + // create the jar + final int exitCode = jarTool.run(System.out, System.err, "-cvfm" + + (compressionMethod == ZipEntry.STORED ? "0" : ""), + jarPath.getFileName().toString(), + manifestFile.toAbsolutePath().toString(), + "-C", srcDir.toAbsolutePath().toString(), "."); + assertEquals(exitCode, 0, "jar tool exited with failure"); + // verify the created jar + verify(jarPath, true, entries, compressionMethod); + // now update the jar using ZipFS + try (final FileSystem zipFs = FileSystems.newFileSystem(jarPath, zipUpdateOptions)) { + for (int i = 0; i < entries.length; i++) { + writeEntry(zipFs, entries[i]); + if (i == entries.length - 2) { + // write out the manifest, now that some non-manifest entries have been added + final Path manifestPath = zipFs.getPath("META-INF", "MANIFEST.MF"); + writeManifest(manifestPath); + } + } + } + verify(jarPath, true, entries, compressionMethod); + } + + private static Entry[] newEntries(final int compressionMethod) { + return new Entry[]{new Entry("hello.txt", compressionMethod, "hello"), + new Entry("META-INF/bar/world.txt", compressionMethod, "world"), + new Entry("META-INF/greeting.txt", compressionMethod, "greeting")}; + } + + private static void writeEntry(final FileSystem fs, final Entry entry) throws IOException { + final Path path = fs.getPath(entry.name); + if (path.getParent() != null) { + Files.createDirectories(path.getParent()); + } + try (final OutputStream os = Files.newOutputStream(path)) { + os.write(entry.bytes); + } + } + + private static void writeEntry(final FileSystem fs, final Path rootPath, final Entry entry) throws IOException { + final Path path = fs.getPath(Paths.get(rootPath.toString(), entry.name).toString()); + if (path.getParent() != null) { + Files.createDirectories(path.getParent()); + } + try (final OutputStream os = Files.newOutputStream(path)) { + os.write(entry.bytes); + } + } + + private static void writeManifest(final Path manifestPath) throws IOException { + if (manifestPath.getParent() != null) { + Files.createDirectories(manifestPath.getParent()); + } + try (final OutputStream os = Files.newOutputStream(manifestPath)) { + final Manifest manifest = new Manifest(); + final Attributes attributes = manifest.getMainAttributes(); + attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0"); + attributes.putValue(MANIFEST_SOME_ATTR, MANIFEST_SOME_ATTR_VALUE); + manifest.write(os); + } + } + + private static void recursiveDelete(final Path dir) throws IOException { + if (!Files.exists(dir)) { + return; + } + Files.walkFileTree(dir, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(final Path dir, final IOException exc) throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + }); + } + + /** + * Verify that the given jar can be opened using JarInputStream and the JarInputStream is + * able to {@link JarInputStream#getManifest() locate the manifest}. Additionally, also + * verify that certain other expected entries in the jar, are found. + * + * @param jar Path to the jar + * @throws IOException + */ + private static void verify(final Path jar, final boolean expectManifest, + final Entry[] expectedEntries, final int compressionMethod) throws IOException { + // Zip/jar files with compression method STORED cannot be opened using Jar/ZipInputStream + // due to a known issue (JDK-8143613) which leads to exception "only DEFLATED entries can have EXT descriptor" + if (ZipEntry.STORED != compressionMethod) { + // use JarInputStream to verify + try (final JarInputStream jaris = new JarInputStream(Files.newInputStream(jar))) { + // verify the manifest is present + final Manifest manifest = jaris.getManifest(); + if (expectManifest) { + assertNotNull(manifest, "JarInputStream couldn't locate a manifest in " + jar); + final String attrVal = manifest.getMainAttributes().getValue(MANIFEST_SOME_ATTR); + assertEquals(attrVal, MANIFEST_SOME_ATTR_VALUE, "Unexpected value for manifest attribute " + + MANIFEST_SOME_ATTR + " in " + jar); + } else { + assertNull(manifest, "JarInputStream unexpectedly found a manifest in " + jar); + } + // verify the rest of the expected entries are present + final Map expected = Arrays.stream(expectedEntries).collect(Collectors.toMap(entry -> entry.name, entry -> entry)); + JarEntry jarEntry = jaris.getNextJarEntry(); + assertNotNull(jarEntry, "Jar " + jar + " is unexpectedly empty"); + while (jarEntry != null) { + if (jarEntry.isDirectory()) { + // skip directories + jarEntry = jaris.getNextJarEntry(); + continue; + } + final Entry expectedEntry = expected.remove(jarEntry.getName()); + assertNotNull(expectedEntry, "Unexpected entry " + jarEntry.getName() + " in jar " + jar); + assertEquals(jarEntry.getMethod(), expectedEntry.method, "Unexpected compression method on entry " + + jarEntry.getName() + " in jar " + jar); + + jarEntry = jaris.getNextJarEntry(); + } + assertEquals(expected.size(), 0, "Missing entries " + expected.keySet() + " in jar " + jar); + } + } + // use JarFile to verify + try (final JarFile jarFile = new JarFile(jar.toFile())) { + final Manifest manifest = jarFile.getManifest(); + if (expectManifest) { + assertNotNull(manifest, "JarFile couldn't locate a manifest in " + jar); + final String attrVal = manifest.getMainAttributes().getValue(MANIFEST_SOME_ATTR); + assertEquals(attrVal, MANIFEST_SOME_ATTR_VALUE, "Unexpected value for manifest attribute " + + MANIFEST_SOME_ATTR + " in " + jar); + } else { + assertNull(manifest, "JarFile unexpectedly found a manifest in " + jar); + } + // verify the rest of the expected entries are present + final Map expected = Arrays.stream(expectedEntries).collect(Collectors.toMap(entry -> entry.name, entry -> entry)); + final Enumeration jarEntries = jarFile.entries(); + assertTrue(jarEntries.hasMoreElements(), "Jar " + jar + " is unexpectedly empty"); + while (jarEntries.hasMoreElements()) { + final JarEntry jarEntry = jarEntries.nextElement(); + if (jarEntry.isDirectory()) { + // skip directories + continue; + } + if (jarEntry.getName().equals("META-INF/MANIFEST.MF")) { + // manifest has already been verified, so skip it + continue; + } + final Entry expectedEntry = expected.remove(jarEntry.getName()); + assertNotNull(expectedEntry, "Unexpected entry " + jarEntry.getName() + " in jar " + jar); + assertEquals(jarEntry.getMethod(), expectedEntry.method, "Unexpected compression method on entry " + + jarEntry.getName() + " in jar " + jar); + try (final InputStream is = jarFile.getInputStream(jarEntry)) { + assertEquals(is.readAllBytes(), expectedEntry.bytes, "Unexpected content in entry " + + jarEntry.getName() + " in jar " + jar); + } + } + assertEquals(expected.size(), 0, "Missing entries " + expected.keySet() + " in jar " + jar); + } + } + + /** + * Represents an entry in a Zip file. An entry encapsulates a name, a + * compression method, and its contents/data. + */ + private static class Entry { + private final String name; + private final int method; + private final byte[] bytes; + + private Entry(String name, int method, String contents) { + this.name = name; + this.method = method; + this.bytes = contents == null ? null : contents.getBytes(StandardCharsets.UTF_8); + } + } + +}