Author: krosenvold Date: Tue Dec 30 19:23:43 2014 New Revision: 1648585 URL: http://svn.apache.org/r1648585 Log: Removed duplication between ZipArchiveOutputStream and StreamCompressor
Reatained all compatibility in ZipArchiveOutputStream Made StreamCompressor and all instantiation of this package private. Modified: commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/archivers/zip/ScatterZipOutputStream.java commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/archivers/zip/StreamCompressor.java commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java Modified: commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/archivers/zip/ScatterZipOutputStream.java URL: http://svn.apache.org/viewvc/commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/archivers/zip/ScatterZipOutputStream.java?rev=1648585&r1=1648584&r2=1648585&view=diff ============================================================================== --- commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/archivers/zip/ScatterZipOutputStream.java (original) +++ commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/archivers/zip/ScatterZipOutputStream.java Tue Dec 30 19:23:43 2014 @@ -21,9 +21,7 @@ package org.apache.commons.compress.arch import org.apache.commons.compress.utils.BoundedInputStream; import java.io.File; -import java.io.FileInputStream; import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.util.Queue; Modified: commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/archivers/zip/StreamCompressor.java URL: http://svn.apache.org/viewvc/commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/archivers/zip/StreamCompressor.java?rev=1648585&r1=1648584&r2=1648585&view=diff ============================================================================== --- commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/archivers/zip/StreamCompressor.java (original) +++ commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/archivers/zip/StreamCompressor.java Tue Dec 30 19:23:43 2014 @@ -17,6 +17,7 @@ */ package org.apache.commons.compress.archivers.zip; +import java.io.Closeable; import java.io.DataOutput; import java.io.IOException; import java.io.InputStream; @@ -30,7 +31,7 @@ import java.util.zip.Deflater; * * @since 1.10 */ -public abstract class StreamCompressor { +abstract class StreamCompressor implements Closeable { /* * Apparently Deflater.setInput gets slowed down a lot on Sun JVMs @@ -45,27 +46,26 @@ public abstract class StreamCompressor { private final CRC32 crc = new CRC32(); - int writtenToOutputStream = 0; - int sourcePayloadLength = 0; - long actualCrc; + private long writtenToOutputStream = 0; + private long sourcePayloadLength = 0; + private long totalWrittenToOutputStream = 0; private final int bufferSize = 4096; private final byte[] outputBuffer = new byte[bufferSize]; private final byte[] readerBuf = new byte[bufferSize]; - protected StreamCompressor(Deflater deflater) { + StreamCompressor(Deflater deflater) { this.def = deflater; } /** * Create a stream compressor with the given compression level. * - * @param compressionLevel The #Deflater compression level - * @param os The #OutputStream stream to receive output + * @param os The #OutputStream stream to receive output + * @param deflater The deflater to use * @return A stream compressor */ - public static StreamCompressor create(int compressionLevel, OutputStream os) { - final Deflater deflater = new Deflater(compressionLevel, true); + static StreamCompressor create(OutputStream os, Deflater deflater) { return new OutputStreamCompressor(deflater, os); } @@ -75,19 +75,18 @@ public abstract class StreamCompressor { * @param os The #OutputStream stream to receive output * @return A stream compressor */ - public static StreamCompressor create( OutputStream os) { - return create(Deflater.DEFAULT_COMPRESSION, os); + static StreamCompressor create(OutputStream os) { + return create(os, new Deflater(Deflater.DEFAULT_COMPRESSION, true)); } /** * Create a stream compressor with the given compression level. * - * @param compressionLevel The #Deflater compression level - * @param os The #DataOutput to receive output + * @param os The #DataOutput to receive output + * @param deflater The deflater to use for the compressor * @return A stream compressor */ - public static StreamCompressor create(int compressionLevel, DataOutput os) { - final Deflater deflater = new Deflater(compressionLevel, true); + static StreamCompressor create(DataOutput os, Deflater deflater) { return new DataOutputCompressor(deflater, os); } @@ -95,7 +94,7 @@ public abstract class StreamCompressor { * Create a stream compressor with the given compression level. * * @param compressionLevel The #Deflater compression level - * @param bs The #ScatterGatherBackingStore to receive output + * @param bs The #ScatterGatherBackingStore to receive output * @return A stream compressor */ public static StreamCompressor create(int compressionLevel, ScatterGatherBackingStore bs) { @@ -109,37 +108,51 @@ public abstract class StreamCompressor { * @param bs The #ScatterGatherBackingStore to receive output * @return A stream compressor */ - public static StreamCompressor create( ScatterGatherBackingStore bs) { + public static StreamCompressor create(ScatterGatherBackingStore bs) { return create(Deflater.DEFAULT_COMPRESSION, bs); } /** * The crc32 of the last deflated file + * * @return the crc32 */ public long getCrc32() { - return actualCrc; + return crc.getValue(); } /** * Return the number of bytes read from the source stream + * * @return The number of bytes read, never negative */ - public int getBytesRead() { + public long getBytesRead() { return sourcePayloadLength; } /** * The number of bytes written to the output + * * @return The number of bytes, never negative */ - public int getBytesWritten() { + public long getBytesWritten() { return writtenToOutputStream; } /** + * The total number of bytes written to the output for all files + * + * @return The number of bytes, never negative + */ + public long getTotalBytesWritten() { + return totalWrittenToOutputStream; + } + + + /** * Deflate the given source using the supplied compression method + * * @param source The source to compress * @param method The #ZipArchiveEntry compression method * @throws IOException When failures happen @@ -149,39 +162,56 @@ public abstract class StreamCompressor { reset(); int length; - while(( length = source.read(readerBuf, 0, readerBuf.length)) >= 0){ - crc.update(readerBuf, 0, length); - if (method == ZipArchiveEntry.DEFLATED) { - writeDeflated(readerBuf, 0, length); - } else { - writeOut(readerBuf, 0, length); - writtenToOutputStream += length; - } - sourcePayloadLength += length; + while ((length = source.read(readerBuf, 0, readerBuf.length)) >= 0) { + write(readerBuf, 0, length, method); } if (method == ZipArchiveEntry.DEFLATED) { flushDeflater(); } - actualCrc = crc.getValue(); - + } + /** + * Writes bytes to ZIP entry. + * + * @param b the byte array to write + * @param offset the start position to write from + * @param length the number of bytes to write + * @param method the comrpession method to use + * @return the number of bytes written to the stream this time + * @throws IOException on error + */ + long write(byte[] b, int offset, int length, int method) throws IOException { + long current = writtenToOutputStream; + crc.update(b, offset, length); + if (method == ZipArchiveEntry.DEFLATED) { + writeDeflated(b, offset, length); + } else { + writeCounted(b, offset, length); + } + sourcePayloadLength += length; + return writtenToOutputStream - current; } - private void reset(){ + + void reset() { crc.reset(); def.reset(); sourcePayloadLength = 0; writtenToOutputStream = 0; } - private void flushDeflater() throws IOException { + public void close() throws IOException { + def.end(); + } + + void flushDeflater() throws IOException { def.finish(); while (!def.finished()) { deflate(); } } - private void writeDeflated(byte[]b, int offset, int length) + private void writeDeflated(byte[] b, int offset, int length) throws IOException { if (length > 0 && !def.finished()) { if (length <= DEFLATER_BLOCK_SIZE) { @@ -212,12 +242,21 @@ public abstract class StreamCompressor { private void deflate() throws IOException { int len = def.deflate(outputBuffer, 0, outputBuffer.length); if (len > 0) { - writeOut(outputBuffer, 0, len); - writtenToOutputStream += len; + writeCounted(outputBuffer, 0, len); } } - protected abstract void writeOut(byte[] data, int offset, int length) throws IOException ; + public void writeCounted(byte[] data) throws IOException { + writeCounted(data, 0, data.length); + } + + public void writeCounted(byte[] data, int offset, int length) throws IOException { + writeOut(data, offset, length); + writtenToOutputStream += length; + totalWrittenToOutputStream += length; + } + + protected abstract void writeOut(byte[] data, int offset, int length) throws IOException; private static final class ScatterGatherBackingStoreCompressor extends StreamCompressor { private final ScatterGatherBackingStore bs; @@ -229,7 +268,7 @@ public abstract class StreamCompressor { protected final void writeOut(byte[] data, int offset, int length) throws IOException { - bs.writeOut(data, offset, length); + bs.writeOut(data, offset, length); } } @@ -243,12 +282,13 @@ public abstract class StreamCompressor { protected final void writeOut(byte[] data, int offset, int length) throws IOException { - os.write(data, offset, length); + os.write(data, offset, length); } } private static final class DataOutputCompressor extends StreamCompressor { private final DataOutput raf; + public DataOutputCompressor(Deflater deflater, DataOutput raf) { super(deflater); this.raf = raf; Modified: commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java URL: http://svn.apache.org/viewvc/commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java?rev=1648585&r1=1648584&r2=1648585&view=diff ============================================================================== --- commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java (original) +++ commons/proper/compress/trunk/src/main/java/org/apache/commons/compress/archivers/zip/ZipArchiveOutputStream.java Tue Dec 30 19:23:43 2014 @@ -28,7 +28,6 @@ import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.zip.CRC32; import java.util.zip.Deflater; import java.util.zip.ZipException; @@ -181,15 +180,7 @@ public class ZipArchiveOutputStream exte private final List<ZipArchiveEntry> entries = new LinkedList<ZipArchiveEntry>(); - /** - * CRC instance to avoid parsing DEFLATED data twice. - */ - private final CRC32 crc = new CRC32(); - - /** - * Count the bytes written to out. - */ - private long written = 0; + private final StreamCompressor streamCompressor; /** * Start of central directory. @@ -235,18 +226,12 @@ public class ZipArchiveOutputStream exte private ZipEncoding zipEncoding = ZipEncodingHelper.getZipEncoding(DEFAULT_ENCODING); - /** - * This Deflater object is used for output. - * - */ - protected final Deflater def = new Deflater(level, true); /** - * This buffer serves as a Deflater. + * This Deflater object is used for output. * */ - private final byte[] buf = new byte[BUFFER_SIZE]; - + protected final Deflater def; /** * Optional random access output. */ @@ -286,6 +271,8 @@ public class ZipArchiveOutputStream exte public ZipArchiveOutputStream(OutputStream out) { this.out = out; this.raf = null; + def = new Deflater(level, true); + streamCompressor = StreamCompressor.create(out, def); } /** @@ -305,6 +292,8 @@ public class ZipArchiveOutputStream exte _raf = null; o = new FileOutputStream(file); } + def = new Deflater(level, true); + streamCompressor = StreamCompressor.create(_raf, def); out = o; raf = _raf; } @@ -441,16 +430,16 @@ public class ZipArchiveOutputStream exte throw new IOException("This archive contains unclosed entries."); } - cdOffset = written; + cdOffset = streamCompressor.getTotalBytesWritten(); for (ZipArchiveEntry ze : entries) { writeCentralFileHeader(ze); } - cdLength = written - cdOffset; + cdLength = streamCompressor.getTotalBytesWritten() - cdOffset; writeZip64CentralDirectory(); writeCentralDirectoryEnd(); offsets.clear(); entries.clear(); - def.end(); + streamCompressor.close(); finished = true; } @@ -467,11 +456,11 @@ public class ZipArchiveOutputStream exte flushDeflater(); - long bytesWritten = written - entry.dataStart; - long realCrc = crc.getValue(); - crc.reset(); - + long bytesWritten = streamCompressor.getTotalBytesWritten() - entry.dataStart; + long realCrc = streamCompressor.getCrc32(); + entry.bytesRead = streamCompressor.getBytesRead(); doCloseEntry(realCrc, bytesWritten); + streamCompressor.reset(); } /** @@ -484,16 +473,24 @@ public class ZipArchiveOutputStream exte */ private void closeCopiedEntry() throws IOException { preClose(); - long realCrc = entry.entry.getCrc(); entry.bytesRead = entry.entry.getSize(); - doCloseEntry(realCrc, entry.entry.getCompressedSize()); + doCloseCopiedEntry(); } private void doCloseEntry(long realCrc, long bytesWritten) throws IOException { final Zip64Mode effectiveMode = getEffectiveZip64Mode(entry.entry); - final boolean actuallyNeedsZip64 = - handleSizesAndCrc(bytesWritten, realCrc, effectiveMode); + final boolean actuallyNeedsZip64 = handleSizesAndCrc(bytesWritten, realCrc, effectiveMode); + + closeEntry(actuallyNeedsZip64); + } + private void doCloseCopiedEntry() throws IOException { + Zip64Mode effectiveMode = getEffectiveZip64Mode(entry.entry); + boolean actuallyNeedsZip64 = checkIfNeedsZip64(effectiveMode); + closeEntry(actuallyNeedsZip64); + } + + private void closeEntry(boolean actuallyNeedsZip64) throws IOException { if (raf != null) { rewriteSizesAndCrc(actuallyNeedsZip64); } @@ -538,10 +535,7 @@ public class ZipArchiveOutputStream exte */ private void flushDeflater() throws IOException { if (entry.entry.getMethod() == DEFLATED) { - def.finish(); - while (!def.finished()) { - deflate(); - } + streamCompressor.flushDeflater(); } } @@ -563,7 +557,6 @@ public class ZipArchiveOutputStream exte entry.entry.setCompressedSize(bytesWritten); entry.entry.setCrc(crc); - def.reset(); } else if (raf == null) { if (entry.entry.getCrc() != crc) { throw new ZipException("bad CRC checksum for entry " @@ -586,16 +579,25 @@ public class ZipArchiveOutputStream exte entry.entry.setCrc(crc); } + return checkIfNeedsZip64(effectiveMode); + } + + /** + * Ensures the current entry's size and CRC information is set to + * the values just written, verifies it isn't too big in the + * Zip64Mode.Never case and returns whether the entry would + * require a Zip64 extra field. + */ + private boolean checkIfNeedsZip64(Zip64Mode effectiveMode) + throws ZipException { final boolean actuallyNeedsZip64 = effectiveMode == Zip64Mode.Always - || entry.entry.getSize() >= ZIP64_MAGIC - || entry.entry.getCompressedSize() >= ZIP64_MAGIC; + || entry.entry.getSize() >= ZIP64_MAGIC + || entry.entry.getCompressedSize() >= ZIP64_MAGIC; if (actuallyNeedsZip64 && effectiveMode == Zip64Mode.Never) { - throw new Zip64RequiredException(Zip64RequiredException - .getEntryTooBigMessage(entry.entry)); + throw new Zip64RequiredException(Zip64RequiredException.getEntryTooBigMessage(entry.entry)); } return actuallyNeedsZip64; } - /** * When using random access output, write the local file header * and potentiall the ZIP64 extra containing the correct CRC and @@ -692,7 +694,7 @@ public class ZipArchiveOutputStream exte def.setLevel(level); hasCompressionLevelChanged = false; } - writeLocalFileHeader((ZipArchiveEntry)archiveEntry); + writeLocalFileHeader((ZipArchiveEntry) archiveEntry); } /** @@ -828,17 +830,23 @@ public class ZipArchiveOutputStream exte throw new IllegalStateException("No current entry"); } ZipUtil.checkRequestedFeatures(entry.entry); - entry.hasWritten = true; - if (entry.entry.getMethod() == DEFLATED) { - writeDeflated(b, offset, length); - } else { - writeOut(b, offset, length); - written += length; - } - crc.update(b, offset, length); - count(length); + long writtenThisTime = streamCompressor.write(b, offset, length, entry.entry.getMethod()); + count(writtenThisTime); + } + + /** + * Write bytes to output or random access file. + * @param data the byte array to write + * @throws IOException on error + */ + private void writeCounted(byte[] data) throws IOException { + streamCompressor.writeCounted(data); } + + + + private void copyFromZipInputStream(InputStream src) throws IOException { if (entry == null) { throw new IllegalStateException("No current entry"); @@ -846,41 +854,11 @@ public class ZipArchiveOutputStream exte ZipUtil.checkRequestedFeatures(entry.entry); entry.hasWritten = true; byte[] tmpBuf = new byte[4096]; - int length = src.read( tmpBuf ); - while ( length >= 0 ) + int length; + while ((length = src.read( tmpBuf )) >= 0 ) { - writeOut( tmpBuf, 0, length ); - written += length; - crc.update( tmpBuf, 0, length ); - + streamCompressor.writeCounted(tmpBuf, 0, length); count( length ); - length = src.read( tmpBuf ); - } - } - - /** - * write implementation for DEFLATED entries. - */ - private void writeDeflated(byte[]b, int offset, int length) - throws IOException { - if (length > 0 && !def.finished()) { - entry.bytesRead += length; - if (length <= DEFLATER_BLOCK_SIZE) { - def.setInput(b, offset, length); - deflateUntilInputIsNeeded(); - } else { - final int fullblocks = length / DEFLATER_BLOCK_SIZE; - for (int i = 0; i < fullblocks; i++) { - def.setInput(b, offset + i * DEFLATER_BLOCK_SIZE, - DEFLATER_BLOCK_SIZE); - deflateUntilInputIsNeeded(); - } - final int done = fullblocks * DEFLATER_BLOCK_SIZE; - if (done < length) { - def.setInput(b, offset + done, length - done); - deflateUntilInputIsNeeded(); - } - } } } @@ -943,18 +921,6 @@ public class ZipArchiveOutputStream exte static final byte[] ZIP64_EOCD_LOC_SIG = ZipLong.getBytes(0X07064B50L); /** - * Writes next block of compressed data to the output stream. - * @throws IOException on error - */ - protected final void deflate() throws IOException { - int len = def.deflate(buf, 0, buf.length); - if (len > 0) { - writeOut(buf, 0, len); - written += len; - } - } - - /** * Writes the local file header entry * @param ze the entry to write * @throws IOException on error @@ -968,11 +934,11 @@ public class ZipArchiveOutputStream exte } final byte[] localHeader = createLocalFileHeader(ze, name, encodable); - offsets.put(ze, written); - entry.localDataStart = written + 14; // Right before crc - writeOut( localHeader); - written += localHeader.length; - entry.dataStart = written; + long localHeaderStart = streamCompressor.getTotalBytesWritten(); + offsets.put(ze, localHeaderStart); + entry.localDataStart = localHeaderStart + LFH_CRC_OFFSET; // At crc offset + writeCounted(localHeader); + entry.dataStart = streamCompressor.getTotalBytesWritten(); } @@ -1084,18 +1050,15 @@ public class ZipArchiveOutputStream exte if (ze.getMethod() != DEFLATED || raf != null) { return; } - writeOut(DD_SIG); - writeOut(ZipLong.getBytes(ze.getCrc())); - int sizeFieldSize = WORD; + writeCounted(DD_SIG); + writeCounted(ZipLong.getBytes(ze.getCrc())); if (!hasZip64Extra(ze)) { - writeOut(ZipLong.getBytes(ze.getCompressedSize())); - writeOut(ZipLong.getBytes(ze.getSize())); + writeCounted(ZipLong.getBytes(ze.getCompressedSize())); + writeCounted(ZipLong.getBytes(ze.getSize())); } else { - sizeFieldSize = DWORD; - writeOut(ZipEightByteInteger.getBytes(ze.getCompressedSize())); - writeOut(ZipEightByteInteger.getBytes(ze.getSize())); + writeCounted(ZipEightByteInteger.getBytes(ze.getCompressedSize())); + writeCounted(ZipEightByteInteger.getBytes(ze.getSize())); } - written += 2 * WORD + 2 * sizeFieldSize; } /** @@ -1125,10 +1088,8 @@ public class ZipArchiveOutputStream exte handleZip64Extra(ze, lfhOffset, needsZip64Extra); - byte[] centralFileHeader = createCentralFileHeader(ze, getName(ze), lfhOffset, - needsZip64Extra); - writeOut(centralFileHeader); - written += centralFileHeader.length; + byte[] centralFileHeader = createCentralFileHeader(ze, getName(ze), lfhOffset, needsZip64Extra); + writeCounted(centralFileHeader); } /** @@ -1249,11 +1210,11 @@ public class ZipArchiveOutputStream exte * and {@link Zip64Mode #setUseZip64} is {@link Zip64Mode#Never}. */ protected void writeCentralDirectoryEnd() throws IOException { - writeOut(EOCD_SIG); + writeCounted(EOCD_SIG); // disk numbers - writeOut(ZERO); - writeOut(ZERO); + writeCounted(ZERO); + writeCounted(ZERO); // number of entries int numberOfEntries = entries.size(); @@ -1269,18 +1230,17 @@ public class ZipArchiveOutputStream exte byte[] num = ZipShort.getBytes(Math.min(numberOfEntries, ZIP64_MAGIC_SHORT)); - writeOut(num); - writeOut(num); + writeCounted(num); + writeCounted(num); // length and location of CD - writeOut(ZipLong.getBytes(Math.min(cdLength, ZIP64_MAGIC))); - writeOut(ZipLong.getBytes(Math.min(cdOffset, ZIP64_MAGIC))); + writeCounted(ZipLong.getBytes(Math.min(cdLength, ZIP64_MAGIC))); + writeCounted(ZipLong.getBytes(Math.min(cdOffset, ZIP64_MAGIC))); // ZIP file comment ByteBuffer data = this.zipEncoding.encode(comment); - writeOut(ZipShort.getBytes(data.limit())); - writeOut(data.array(), data.arrayOffset(), - data.limit() - data.position()); + writeCounted(ZipShort.getBytes(data.limit())); + streamCompressor.writeCounted(data.array(), data.arrayOffset(), data.limit() - data.position()); } private static final byte[] ONE = ZipLong.getBytes(1L); @@ -1307,7 +1267,7 @@ public class ZipArchiveOutputStream exte return; } - long offset = written; + long offset = streamCompressor.getBytesWritten(); writeOut(ZIP64_EOCD_SIG); // size, we don't have any variable length as we don't support @@ -1359,9 +1319,10 @@ public class ZipArchiveOutputStream exte * @throws IOException on error */ protected final void writeOut(byte[] data) throws IOException { - writeOut(data, 0, data.length); + streamCompressor.writeOut(data, 0, data.length); } + /** * Write bytes to output or random access file. * @param data the byte array to write @@ -1370,19 +1331,10 @@ public class ZipArchiveOutputStream exte * @throws IOException on error */ protected final void writeOut(byte[] data, int offset, int length) - throws IOException { - if (raf != null) { - raf.write(data, offset, length); - } else { - out.write(data, offset, length); - } + throws IOException { + streamCompressor.writeOut(data, offset, length); } - private void deflateUntilInputIsNeeded() throws IOException { - while (!def.needsInput()) { - deflate(); - } - } private GeneralPurposeBit getGeneralPurposeBits(final int zipMethod, final boolean utfFallback) { GeneralPurposeBit b = new GeneralPurposeBit();