1 /*
   2  * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 
  26 package org.openjdk.jigsaw;
  27 
  28 import java.nio.file.attribute.BasicFileAttributes;
  29 import java.io.*;
  30 import java.nio.ByteBuffer;
  31 import java.nio.channels.*;
  32 import java.nio.file.*;
  33 import java.nio.file.Files;
  34 import java.security.MessageDigest;
  35 import java.util.*;
  36 import java.util.jar.*;
  37 import java.util.zip.*;
  38 import static org.openjdk.jigsaw.FileConstants.ModuleFile.*;
  39 import static org.openjdk.jigsaw.ModuleFile.*;
  40 
  41 
  42 /**
  43  * <p> A writer of module files </p>
  44  */
  45 
  46 public class ModuleFileWriter {
  47 
  48     private final File outfile;
  49     private final HashType hashtype;
  50     private final boolean fastestCompression;
  51     private long usize;
  52 
  53     public ModuleFileWriter(File outfile) {
  54         this(outfile, false);
  55     }
  56 
  57     public ModuleFileWriter(File outfile, boolean useFastestCompression) {
  58         this.outfile = outfile;
  59         this.hashtype = HashType.SHA256;
  60         this.fastestCompression = useFastestCompression;
  61     }
  62 
  63     /**
  64      * Generates an unsigned module file.
  65      */
  66     public void writeModule(File mdir,
  67                             File nativelibs,
  68                             File nativecmds,
  69                             File config)
  70             throws IOException
  71     {
  72         if (!mdir.isDirectory()) {
  73             throw new IOException("Not a directory: " + mdir);
  74         }
  75 
  76         try (RandomAccessFile file = new RandomAccessFile(outfile, "rw")) {
  77             // Truncate the file if it already exists
  78             file.setLength(0);
  79 
  80             // Reset module file to right after module header
  81             file.seek(ModuleFileHeader.LENGTH);
  82 
  83             // TODO: Why was this after the module info???
  84             long remainderStart = file.getFilePointer();
  85 
  86             // Write out the Module-Info Section
  87             File miclass = new File(mdir, "module-info.class");
  88             if (!miclass.exists()) {
  89                 throw new IOException(miclass + " does not exist");
  90             }
  91             writeSection(file,
  92                          SectionType.MODULE_INFO,
  93                          mdir,
  94                          Collections.singletonList(miclass.toPath()),
  95                          Compressor.NONE);
  96 
  97             // Write out the optional file sections
  98             writeOptionalSections(file, mdir, nativelibs, nativecmds, config);
  99 
 100             // Write out the module file header
 101             writeModuleFileHeader(file, remainderStart);
 102         }
 103     }
 104 
 105     /*
 106      * Write a section to the given module file.
 107      *
 108      * @params file RandomAccessFile for the resulting jmod file
 109      * @params type section type
 110      * @params sourcedir the directory containing the files to be written
 111      * @params files list of files to be written
 112      * @params compressor compression type
 113      */
 114     private void writeSection(RandomAccessFile file,
 115                               SectionType type,
 116                               File sourcedir,
 117                               List<Path> files,
 118                               Compressor compressor) throws IOException {
 119         // Start of section header
 120         final long start = file.getFilePointer();
 121 
 122         // Start of section content
 123         final long cstart = start + SectionHeader.LENGTH;
 124         // Seek to start of section content
 125         file.seek(cstart);
 126 
 127         writeSectionContent(file, type, sourcedir, files, compressor);
 128 
 129         // End of section
 130         final long end = file.getFilePointer();
 131         final int csize = (int) (end - cstart);
 132 
 133         // A section type that has no files has a section count of 0.
 134         int count = type.hasFiles() ? files.size() : 0;
 135         if (count > Short.MAX_VALUE) {
 136             throw new IOException("Too many files: " + count);
 137         }
 138         writeSectionHeader(file, type, compressor, start, csize, (short) count);
 139     }
 140 
 141     private void writeSectionContent(RandomAccessFile file,
 142                                      SectionType type,
 143                                      File sourcedir,
 144                                      List<Path> files,
 145                                      Compressor compressor) throws IOException {
 146 
 147         if (type.hasFiles()) {
 148             for (Path p : files) {
 149                 writeSubSection(file, sourcedir, p, compressor);
 150             }
 151         } else if (type == SectionType.CLASSES) {
 152             writeClassesContent(file, sourcedir, files, compressor);
 153         } else if (files.size() == 1) {
 154             writeFileContent(file, files.get(0), compressor);
 155         } else {
 156             throw new IllegalArgumentException("Section type: " + type
 157                     + " can only have one single file but given " + files);
 158         }
 159     }
 160 
 161     private void writeSectionHeader(RandomAccessFile file,
 162                                     SectionType type,
 163                                     Compressor compressor,
 164                                     long start, int csize,
 165                                     short subsections) throws IOException {
 166 
 167         // Compute hash of content
 168         MessageDigest md = getHashInstance(hashtype);
 169         FileChannel channel = file.getChannel();
 170         ByteBuffer content = ByteBuffer.allocate(csize);
 171 
 172         final long cstart = start + SectionHeader.LENGTH;
 173         int n = channel.read(content, cstart);
 174         if (n != csize) {
 175             throw new IOException("too few bytes read");
 176         }
 177         content.position(0);
 178         md.update(content);
 179         final byte[] hash = md.digest();
 180 
 181         // Write section header at section header start,
 182         // and seek to end of section.
 183         SectionHeader header =
 184                 new SectionHeader(type, compressor, csize, subsections, hash);
 185         file.seek(start);
 186         header.write(file);
 187         file.seek(cstart + csize);
 188     }
 189 
 190     private void writeClassesContent(DataOutput out,
 191                                      File sourcedir,
 192                                      List<Path> files,
 193                                      Compressor compressor) throws IOException {
 194         CompressedClassOutputStream cos =
 195             CompressedClassOutputStream.newInstance(sourcedir.toPath(),
 196                                                     files,
 197                                                     compressor,
 198                                                     fastestCompression);
 199         usize += cos.getUSize();
 200         cos.writeTo(out);
 201     }
 202 
 203     private void writeFileContent(DataOutput out,
 204                                   Path p,
 205                                   Compressor compressor) throws IOException {
 206         CompressedOutputStream cos = CompressedOutputStream.newInstance(p, compressor);
 207         usize += cos.getUSize();
 208         cos.writeTo(out);
 209     }
 210 
 211     /*
 212      * Write a subsection to the given module file.
 213      *
 214      * @params file RandomAccessFile for the resulting jmod file
 215      * @params sourcedir the directory containing the file to be written
 216      * @params p Path of a file to be written
 217      * @params compressor compression type
 218      */
 219     private void writeSubSection(RandomAccessFile file,
 220                                  File sourcedir,
 221                                  Path p,
 222                                  Compressor compressor) throws IOException {
 223         CompressedOutputStream cos = CompressedOutputStream.newInstance(p, compressor);
 224         usize += cos.getUSize();
 225 
 226         String storedpath = relativePath(sourcedir.toPath(), p);
 227         SubSectionFileHeader header = new SubSectionFileHeader((int)cos.getCSize(), storedpath);
 228         header.write(file);
 229         cos.writeTo(file);
 230     }
 231 
 232 
 233     /*
 234      * Processes each of the optional file sections.
 235      */
 236     private void writeOptionalSections(RandomAccessFile file,
 237                                        File mdir,
 238                                        File nativelibs,
 239                                        File nativecmds,
 240                                        File config)
 241             throws IOException
 242    {
 243         List<Path> classes = new ArrayList<>();
 244         List<Path> resources = new ArrayList<>();
 245         listClassesResources(mdir.toPath(), classes, resources);
 246 
 247         if (!classes.isEmpty()) {
 248             writeSection(file,
 249                          SectionType.CLASSES,
 250                          mdir,
 251                          classes,
 252                          Compressor.PACK200_GZIP);
 253         }
 254         if (!resources.isEmpty()) {
 255             writeSection(file,
 256                          SectionType.RESOURCES,
 257                          mdir,
 258                          resources,
 259                          Compressor.GZIP);
 260         }
 261 
 262         if (nativelibs != null && directoryIsNotEmpty(nativelibs)) {
 263             writeSection(file,
 264                          SectionType.NATIVE_LIBS,
 265                          nativelibs,
 266                          listFiles(nativelibs.toPath()),
 267                          Compressor.GZIP);
 268         }
 269         if (nativecmds != null && directoryIsNotEmpty(nativecmds)) {
 270             writeSection(file,
 271                          SectionType.NATIVE_CMDS,
 272                          nativecmds,
 273                          listFiles(nativecmds.toPath()),
 274                          Compressor.GZIP);
 275         }
 276         if (config != null && directoryIsNotEmpty(config)) {
 277             writeSection(file,
 278                          SectionType.CONFIG,
 279                          config,
 280                          listFiles(config.toPath()),
 281                          Compressor.GZIP);
 282         }
 283     }
 284 
 285     /*
 286      * Writes out the module file header.
 287      */
 288     private void writeModuleFileHeader(RandomAccessFile file,
 289                                        long remainderStart)
 290             throws IOException
 291     {
 292 
 293         long csize = file.length() - remainderStart;
 294 
 295         // Header Step 1
 296         // Write out the module file header (using a dummy file hash value)
 297         ModuleFileHeader header =
 298                 new ModuleFileHeader(csize, usize, hashtype,
 299                                      new byte[hashtype.length()]);
 300         file.seek(0);
 301         header.write(file);
 302 
 303         // Generate the module file hash
 304         byte[] fileHash = generateFileHash(file);
 305 
 306         // Header Step 2
 307         // Write out the module file header (using correct file hash value)
 308         header = new ModuleFileHeader(csize, usize, hashtype, fileHash);
 309         file.seek(0);
 310         header.write(file);
 311     }
 312 
 313     private void listClassesResources(Path dir,
 314                                       final List<Path> classes,
 315                                       final List<Path> resources)
 316             throws IOException
 317     {
 318 
 319         Files.walkFileTree(dir, new SimpleFileVisitor<Path>() {
 320             @Override
 321             public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
 322                     throws IOException {
 323                 if (!file.endsWith("module-info.class")) {
 324                     if (file.toFile().getName().endsWith(".class")) {
 325                         classes.add(file);
 326                     } else {
 327                         resources.add(file);
 328                     }
 329                 }
 330                 return FileVisitResult.CONTINUE;
 331             }
 332         });
 333     }
 334 
 335     // CompressedOutputStream and CompressedClassOutputStream are
 336     // subclass of ByteArrayOutputStream to avoid the array copy
 337     // from the compressed bytes and write buf directly to DataOutput
 338     static class CompressedOutputStream extends ByteArrayOutputStream {
 339         protected long size = 0;   // uncompressed size
 340         protected CompressedOutputStream() {
 341         }
 342 
 343         private CompressedOutputStream(Path p) throws IOException {
 344             // no compression
 345             size += Files.copy(p, this);
 346         }
 347 
 348         long getUSize() {
 349             return size;
 350         }
 351 
 352         long getCSize() {
 353             return count;
 354         }
 355 
 356         void writeTo(DataOutput out) throws IOException {
 357             out.write(buf, 0, count);
 358         }
 359 
 360         static CompressedOutputStream newInstance(Path p, Compressor type)
 361                 throws IOException
 362         {
 363             switch (type) {
 364                 case NONE:
 365                     return new CompressedOutputStream(p);
 366                 case GZIP:
 367                     return new GZIPCompressedOutputStream(p);
 368                 case PACK200_GZIP:
 369                 default:
 370                     throw new IllegalArgumentException("Unsupported type: " + type);
 371             }
 372         }
 373 
 374         static class GZIPCompressedOutputStream extends CompressedOutputStream {
 375             GZIPCompressedOutputStream(Path p) throws IOException {
 376                 super();
 377                 size += p.toFile().length();
 378                 try (GZIPOutputStream gos = new GZIPOutputStream(this)) {
 379                     Files.copy(p, gos);
 380                     gos.finish();
 381                 }
 382             }
 383         }
 384     }
 385 
 386     static abstract class CompressedClassOutputStream extends ByteArrayOutputStream {
 387         protected long size = 0;
 388         long getUSize() {
 389             return size;
 390         }
 391 
 392         long getCSize() {
 393             return count;
 394         }
 395 
 396         void writeTo(DataOutput out) throws IOException {
 397             out.write(buf, 0, count);
 398         }
 399 
 400         static CompressedClassOutputStream newInstance(Path sourcepath,
 401                                                        List<Path> classes,
 402                                                        Compressor type,
 403                                                        boolean fastestCompression)
 404                 throws IOException
 405         {
 406             switch (type) {
 407                 case PACK200_GZIP:
 408                     Pack200GZipOutputStream pos =
 409                             new Pack200GZipOutputStream(fastestCompression);
 410                     pos.compress(sourcepath, classes);
 411                     return pos;
 412                 default:
 413                     throw new IllegalArgumentException("Unsupported type: " + type);
 414             }
 415         }
 416 
 417         static class Pack200GZipOutputStream extends CompressedClassOutputStream {
 418             final Pack200.Packer packer = Pack200.newPacker();
 419             Pack200GZipOutputStream(boolean fastestCompression) {
 420                 Map<String, String> p = packer.properties();
 421                 p.put(Pack200.Packer.SEGMENT_LIMIT, "-1");
 422                 if (fastestCompression) {
 423                     p.put(Pack200.Packer.EFFORT, "1");
 424                 }
 425                 p.put(Pack200.Packer.KEEP_FILE_ORDER, Pack200.Packer.FALSE);
 426                 p.put(Pack200.Packer.MODIFICATION_TIME, Pack200.Packer.LATEST);
 427                 p.put(Pack200.Packer.DEFLATE_HINT, Pack200.Packer.FALSE);
 428             }
 429 
 430             void compress(Path sourcepath, List<Path> classes) throws IOException {
 431                 ByteArrayOutputStream os = new ByteArrayOutputStream();
 432                 try (JarOutputStream jos = new JarOutputStream(os)) {
 433                     jos.setLevel(0);
 434                     for (Path file : classes) {
 435                         // write to the JarInputStream for later pack200 compression
 436                         String name = file.toFile().getName().toLowerCase();
 437                         String p = sourcepath.relativize(file).toString();
 438                         if (!p.equals("module-info.class")) {
 439                             JarEntry entry = new JarEntry(p);
 440                             jos.putNextEntry(entry);
 441                             Files.copy(file, jos);
 442                             jos.closeEntry();
 443                         }
 444                         size += file.toFile().length();
 445                     }
 446                 }
 447 
 448                 byte[] data = os.toByteArray();
 449                 try (JarInputStream jin =
 450                         new JarInputStream(new ByteArrayInputStream(data));
 451                      GZIPOutputStream gout = new GZIPOutputStream(this)) {
 452                     packer.pack(jin, gout);
 453                 }
 454             }
 455         }
 456     }
 457 
 458     private String relativePath(Path sourcepath, Path path) throws IOException {
 459         Path relativepath = sourcepath.relativize(path);
 460 
 461         // The '/' character separates nested directories in path names.
 462         String pathseparator = relativepath.getFileSystem().getSeparator();
 463         String stored = relativepath.toString().replace(pathseparator, "/");
 464 
 465         // The path names of native-code files
 466         // must not include more than one element.
 467         // ## Temporarily turn off this check until the location of
 468         // ## the native libraries in jdk modules are changed
 469         // ensureShortNativePath(realpath, stored);
 470         return stored;
 471     }
 472 
 473     private List<Path> listFiles(Path path) throws IOException {
 474         final List<Path> list = new ArrayList<>();
 475         Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
 476 
 477             @Override
 478             public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
 479                     throws IOException {
 480                 list.add(file);
 481                 return FileVisitResult.CONTINUE;
 482             }
 483         });
 484         return list;
 485     }
 486 
 487     /*
 488      * Generates the hash value for a module file.
 489      * Excludes itself (the hash bytes in the module file header).
 490      */
 491     private byte[] generateFileHash(RandomAccessFile file)
 492             throws IOException
 493     {
 494         MessageDigest md = getHashInstance(hashtype);
 495 
 496         long remainderSize = file.length() - ModuleFileHeader.LENGTH;
 497         FileChannel channel = file.getChannel();
 498 
 499         // Module file header without the hash bytes
 500         ByteBuffer content = ByteBuffer.allocate(ModuleFileHeader.LENGTH_WITHOUT_HASH);
 501         int n = channel.read(content, 0);
 502         if (n != ModuleFileHeader.LENGTH_WITHOUT_HASH) {
 503             throw new IOException("too few bytes read");
 504         }
 505         content.position(0);
 506         md.update(content);
 507 
 508         // Remainder of file (read in chunks)
 509         content = ByteBuffer.allocate(8192);
 510         channel.position(ModuleFileHeader.LENGTH);
 511         n = channel.read(content);
 512         while (n != -1) {
 513             content.limit(n);
 514             content.position(0);
 515             md.update(content);
 516             content = ByteBuffer.allocate(8192);
 517             n = channel.read(content);
 518         }
 519 
 520         return md.digest();
 521     }
 522 
 523     /*
 524      * Check if a given directory is not empty.
 525      */
 526     private static boolean directoryIsNotEmpty(File dir)
 527             throws IOException {
 528         try (DirectoryStream<Path> ds =
 529                         Files.newDirectoryStream(dir.toPath())) {
 530             return ds.iterator().hasNext();
 531         }
 532     }
 533 
 534 }