Repository: commons-compress Updated Branches: refs/heads/COMPRESS-207 3cddfaaff -> 8b1908cbb
COMPRESS-207 also record compressed bytes processed Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/8b1908cb Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/8b1908cb Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/8b1908cb Branch: refs/heads/COMPRESS-207 Commit: 8b1908cbba7012ed693636926bb1fd2057340d7b Parents: 3cddfaa Author: Stefan Bodewig <bode...@apache.org> Authored: Mon Mar 28 10:37:06 2016 +0200 Committer: Stefan Bodewig <bode...@apache.org> Committed: Mon Mar 28 10:37:06 2016 +0200 ---------------------------------------------------------------------- .../compressors/CompressionProgressEvent.java | 31 +++++++++++++----- .../compressors/CompressorInputStream.java | 6 ++-- .../compressors/CompressorOutputStream.java | 6 ++-- .../bzip2/BZip2CompressorInputStream.java | 34 +++++++++++++------- .../compress/compressors/BZip2TestCase.java | 23 ++++++++----- 5 files changed, 68 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/commons-compress/blob/8b1908cb/src/main/java/org/apache/commons/compress/compressors/CompressionProgressEvent.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressionProgressEvent.java b/src/main/java/org/apache/commons/compress/compressors/CompressionProgressEvent.java index 122d1ca..b4c8943 100644 --- a/src/main/java/org/apache/commons/compress/compressors/CompressionProgressEvent.java +++ b/src/main/java/org/apache/commons/compress/compressors/CompressionProgressEvent.java @@ -27,7 +27,7 @@ import java.util.EventObject; public class CompressionProgressEvent extends EventObject { private final int blockNumber, streamNumber; - private final long bytesProcessed; + private final long compressedBytesProcessed, uncompressedBytesProcessed; /** * Creates a new event. @@ -36,14 +36,19 @@ public class CompressionProgressEvent extends EventObject { * @param blockNumber number of the block that is getting processed now * @param streamNumer number of the stream that is getting * processed now - * @param bytesProcessed number of bytes read or written when the event is triggered + * @param compressedBytesProcessed number of compressed bytes read + * or written when the event is triggered + * @param uncompressedBytesProcessed number of uncompressed bytes read + * or written when the event is triggered */ public CompressionProgressEvent(Object source, int blockNumber, int streamNumber, - long bytesProcessed) { + long compressedBytesProcessed, + long uncompressedBytesProcessed) { super(source); this.blockNumber = blockNumber; this.streamNumber = streamNumber; - this.bytesProcessed = bytesProcessed; + this.compressedBytesProcessed = compressedBytesProcessed; + this.uncompressedBytesProcessed = uncompressedBytesProcessed; } /** @@ -69,10 +74,20 @@ public class CompressionProgressEvent extends EventObject { } /** - * The number of bytes processed so far. - * @return number of bytes read or written when the event is triggered + * The number of compressed bytes processed so far. + * @return number of compressed bytes read or written when the + * event is triggered */ - public long getBytesProcessed() { - return bytesProcessed; + public long getCompressedBytesProcessed() { + return compressedBytesProcessed; + } + + /** + * The number of uncompressed bytes processed so far. + * @return number of uncompressed bytes read or written when the + * event is triggered + */ + public long getUncompressedBytesProcessed() { + return uncompressedBytesProcessed; } } http://git-wip-us.apache.org/repos/asf/commons-compress/blob/8b1908cb/src/main/java/org/apache/commons/compress/compressors/CompressorInputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/CompressorInputStream.java index 45ceafe..4d0e46a 100644 --- a/src/main/java/org/apache/commons/compress/compressors/CompressorInputStream.java +++ b/src/main/java/org/apache/commons/compress/compressors/CompressorInputStream.java @@ -108,9 +108,11 @@ public abstract class CompressorInputStream extends InputStream { * @param blockNumber number of the block that is getting processed now * @param streamNumer number of the stream that is getting * processed now + * @param compressedBytesRead number of compressed bytes read */ - protected void fireProgress(int blockNumber, int streamNumber) { - CompressionProgressEvent e = new CompressionProgressEvent(this, blockNumber, streamNumber, getBytesRead()); + protected void fireProgress(int blockNumber, int streamNumber, long compressedBytesRead) { + CompressionProgressEvent e = new CompressionProgressEvent(this, blockNumber, streamNumber, + compressedBytesRead, getBytesRead()); for (CompressionProgressListener l : listeners) { l.notify(e); } http://git-wip-us.apache.org/repos/asf/commons-compress/blob/8b1908cb/src/main/java/org/apache/commons/compress/compressors/CompressorOutputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressorOutputStream.java b/src/main/java/org/apache/commons/compress/compressors/CompressorOutputStream.java index b0f7bf9..9dd18a1 100644 --- a/src/main/java/org/apache/commons/compress/compressors/CompressorOutputStream.java +++ b/src/main/java/org/apache/commons/compress/compressors/CompressorOutputStream.java @@ -54,9 +54,11 @@ public abstract class CompressorOutputStream extends OutputStream { * @param blockNumber number of the block that is getting processed now * @param streamNumer number of the stream that is getting * processed now + * @param compressedBytesWritten number of compressed bytes written */ - protected void fireProgress(int blockNumber, int streamNumber) { - CompressionProgressEvent e = new CompressionProgressEvent(this, blockNumber, streamNumber, getBytesWritten()); + protected void fireProgress(int blockNumber, int streamNumber, long compressedBytesWritten) { + CompressionProgressEvent e = new CompressionProgressEvent(this, blockNumber, streamNumber, + compressedBytesWritten, getBytesWritten()); for (CompressionProgressListener l : listeners) { l.notify(e); } http://git-wip-us.apache.org/repos/asf/commons-compress/blob/8b1908cb/src/main/java/org/apache/commons/compress/compressors/bzip2/BZip2CompressorInputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/bzip2/BZip2CompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/bzip2/BZip2CompressorInputStream.java index bcfe577..69f395c 100644 --- a/src/main/java/org/apache/commons/compress/compressors/bzip2/BZip2CompressorInputStream.java +++ b/src/main/java/org/apache/commons/compress/compressors/bzip2/BZip2CompressorInputStream.java @@ -91,6 +91,7 @@ public class BZip2CompressorInputStream extends CompressorInputStream implements private char su_z; private int currentBlock, currentStream; + private long compressedBytesRead; /** * All memory intensive stuff. This field is initialized by initBlock(). @@ -230,17 +231,25 @@ public class BZip2CompressorInputStream extends CompressorInputStream implements } } + private int read(InputStream in) throws IOException { + int r = in.read(); + if (r != -1) { + compressedBytesRead++; + } + return r; + } + private boolean init(boolean isFirstStream) throws IOException { if (null == in) { throw new IOException("No InputStream"); } - int magic0 = this.in.read(); + int magic0 = read(this.in); if (magic0 == -1 && !isFirstStream) { return false; } - int magic1 = this.in.read(); - int magic2 = this.in.read(); + int magic1 = read(this.in); + int magic2 = read(this.in); if (magic0 != 'B' || magic1 != 'Z' || magic2 != 'h') { throw new IOException(isFirstStream @@ -248,7 +257,7 @@ public class BZip2CompressorInputStream extends CompressorInputStream implements : "Garbage after a valid BZip2 stream"); } - int blockSize = this.in.read(); + int blockSize = read(this.in); if ((blockSize < '1') || (blockSize > '9')) { throw new IOException("BZip2 block size is invalid"); } @@ -302,6 +311,8 @@ public class BZip2CompressorInputStream extends CompressorInputStream implements this.currentState = EOF; throw new IOException("bad block header"); } else { + // subtract block start magic bytes from compressedBytesRead + fireProgress(currentBlock++, currentStream, compressedBytesRead - 6); this.storedBlockCRC = bsGetInt(); this.blockRandomised = bsR(1) == 1; @@ -313,7 +324,6 @@ public class BZip2CompressorInputStream extends CompressorInputStream implements this.data = new Data(this.blockSize100k); } - fireProgress(currentBlock++, currentStream); getAndMoveToFrontDecode(); this.crc.initialiseCRC(); @@ -379,7 +389,7 @@ public class BZip2CompressorInputStream extends CompressorInputStream implements if (bsLiveShadow < n) { final InputStream inShadow = this.in; do { - int thech = inShadow.read(); + int thech = read(inShadow); if (thech < 0) { throw new IOException("unexpected end of stream"); @@ -401,7 +411,7 @@ public class BZip2CompressorInputStream extends CompressorInputStream implements int bsBuffShadow = this.bsBuff; if (bsLiveShadow < 1) { - int thech = this.in.read(); + int thech = read(this.in); if (thech < 0) { throw new IOException("unexpected end of stream"); @@ -646,7 +656,7 @@ public class BZip2CompressorInputStream extends CompressorInputStream implements // Inlined: // int zvec = bsR(zn); while (bsLiveShadow < zn) { - final int thech = inShadow.read(); + final int thech = read(inShadow); if (thech >= 0) { bsBuffShadow = (bsBuffShadow << 8) | thech; bsLiveShadow += 8; @@ -662,7 +672,7 @@ public class BZip2CompressorInputStream extends CompressorInputStream implements while (zvec > limit_zt[zn]) { zn++; while (bsLiveShadow < 1) { - final int thech = inShadow.read(); + final int thech = read(inShadow); if (thech >= 0) { bsBuffShadow = (bsBuffShadow << 8) | thech; bsLiveShadow += 8; @@ -729,7 +739,7 @@ public class BZip2CompressorInputStream extends CompressorInputStream implements // Inlined: // int zvec = bsR(zn); while (bsLiveShadow < zn) { - final int thech = inShadow.read(); + final int thech = read(inShadow); if (thech >= 0) { bsBuffShadow = (bsBuffShadow << 8) | thech; bsLiveShadow += 8; @@ -745,7 +755,7 @@ public class BZip2CompressorInputStream extends CompressorInputStream implements while (zvec > limit_zt[zn]) { zn++; while (bsLiveShadow < 1) { - final int thech = inShadow.read(); + final int thech = read(inShadow); if (thech >= 0) { bsBuffShadow = (bsBuffShadow << 8) | thech; bsLiveShadow += 8; @@ -779,7 +789,7 @@ public class BZip2CompressorInputStream extends CompressorInputStream implements while (zvec > limit_zt[zn]) { zn++; while (bsLiveShadow < 1) { - final int thech = inShadow.read(); + final int thech = read(inShadow); if (thech >= 0) { bsBuffShadow = (bsBuffShadow << 8) | thech; http://git-wip-us.apache.org/repos/asf/commons-compress/blob/8b1908cb/src/test/java/org/apache/commons/compress/compressors/BZip2TestCase.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/commons/compress/compressors/BZip2TestCase.java b/src/test/java/org/apache/commons/compress/compressors/BZip2TestCase.java index e736125..17e6554 100644 --- a/src/test/java/org/apache/commons/compress/compressors/BZip2TestCase.java +++ b/src/test/java/org/apache/commons/compress/compressors/BZip2TestCase.java @@ -140,7 +140,8 @@ public final class BZip2TestCase extends AbstractTestCase { File inputFile = getFile("COMPRESS-207.bz2"); FileInputStream fInputStream = new FileInputStream(inputFile); final List<Integer> blockNumbers = new ArrayList<Integer>(); - final List<Long> readPositions = new ArrayList<Long>(); + final List<Long> uncompressedBytes = new ArrayList<Long>(); + final List<Long> compressedBytes = new ArrayList<Long>(); final BZip2CompressorInputStream in = new BZip2CompressorInputStream(fInputStream); CompressionProgressListener blockListener = new CompressionProgressListener() { @@ -148,7 +149,8 @@ public final class BZip2TestCase extends AbstractTestCase { public void notify(CompressionProgressEvent e) { assertSame(in, e.getSource()); blockNumbers.add(e.getBlockNumber()); - readPositions.add(e.getBytesProcessed()); + uncompressedBytes.add(e.getUncompressedBytesProcessed()); + compressedBytes.add(e.getCompressedBytesProcessed()); } }; in.addCompressionProgressListener(blockListener); @@ -158,16 +160,21 @@ public final class BZip2TestCase extends AbstractTestCase { // we miss the initial block event which is triggered by the constructor assertEquals(4, blockNumbers.size()); - System.err.println(blockNumbers); for (int i = 0; i < 4; i++) { assertEquals(i + 1, blockNumbers.get(i).intValue()); } - assertEquals(4, readPositions.size()); - assertEquals(Long.valueOf(899907), readPositions.get(0)); - assertEquals(Long.valueOf(1799817), readPositions.get(1)); - assertEquals(Long.valueOf(2699710), readPositions.get(2)); - assertEquals(Long.valueOf(3599604), readPositions.get(3)); + assertEquals(4, uncompressedBytes.size()); + assertEquals(Long.valueOf(899907), uncompressedBytes.get(0)); + assertEquals(Long.valueOf(1799817), uncompressedBytes.get(1)); + assertEquals(Long.valueOf(2699710), uncompressedBytes.get(2)); + assertEquals(Long.valueOf(3599604), uncompressedBytes.get(3)); + + assertEquals(4, compressedBytes.size()); + assertEquals(Long.valueOf(457766), compressedBytes.get(0)); + assertEquals(Long.valueOf(915639), compressedBytes.get(1)); + assertEquals(Long.valueOf(1373360), compressedBytes.get(2)); + assertEquals(Long.valueOf(1831305), compressedBytes.get(3)); } }
