Repository: commons-compress Updated Branches: refs/heads/master 3af95ce3a -> 25ecc9b17
COMPRESS-271 support for decompressConcatenated in lz4 frame input Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/25ecc9b1 Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/25ecc9b1 Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/25ecc9b1 Branch: refs/heads/master Commit: 25ecc9b177ef21bcc35a82776b71cbdae4a7f708 Parents: 3af95ce Author: Stefan Bodewig <bode...@apache.org> Authored: Sun Feb 5 12:55:12 2017 +0100 Committer: Stefan Bodewig <bode...@apache.org> Committed: Sun Feb 5 12:55:12 2017 +0100 ---------------------------------------------------------------------- .../compressors/CompressorStreamFactory.java | 2 +- .../lz4/FramedLZ4CompressorInputStream.java | 45 ++++++-- .../lz4/FramedLZ4CompressorInputStreamTest.java | 105 +++++++++++++++++++ 3 files changed, 143 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/commons-compress/blob/25ecc9b1/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java index 29ba9fd..4f0685b 100644 --- a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java +++ b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java @@ -518,7 +518,7 @@ public class CompressorStreamFactory implements CompressorStreamProvider { } if (LZ4_FRAMED.equalsIgnoreCase(name)) { - return new FramedLZ4CompressorInputStream(in); + return new FramedLZ4CompressorInputStream(in, actualDecompressConcatenated); } } catch (final IOException e) { http://git-wip-us.apache.org/repos/asf/commons-compress/blob/25ecc9b1/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java index 418f320..e597b5a 100644 --- a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java +++ b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java @@ -40,9 +40,7 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream { /* * TODO before releasing 1.14: * - * + xxhash32 checksum validation * + skippable frames - * + decompressConcatenated * + block dependence */ @@ -71,6 +69,7 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream { }; private final InputStream in; + private final boolean decompressConcatenated; private boolean expectBlockChecksum; private boolean expectContentSize; @@ -84,15 +83,29 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream { /** * Creates a new input stream that decompresses streams compressed - * using the LZ4 frame format. + * using the LZ4 frame format and stops after decompressing the + * first frame. * @param in the InputStream from which to read the compressed data * @throws IOException if reading fails */ public FramedLZ4CompressorInputStream(InputStream in) throws IOException { + this(in, false); + } + + /** + * Creates a new input stream that decompresses streams compressed + * using the LZ4 frame format. + * @param in the InputStream from which to read the compressed data + * @param decompressConcatenated if true, decompress until the end + * of the input; if false, stop after the first LZ4 frame + * and leave the input position to point to the next byte + * after the frame stream + * @throws IOException if reading fails + */ + public FramedLZ4CompressorInputStream(InputStream in, boolean decompressConcatenated) throws IOException { this.in = in; - readSignature(); - readFrameDescriptor(); - nextBlock(); + this.decompressConcatenated = decompressConcatenated; + init(true); } /** {@inheritDoc} */ @@ -130,13 +143,25 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream { return r; } - private void readSignature() throws IOException { + private void init(boolean firstFrame) throws IOException { + if (readSignature(firstFrame)) { + readFrameDescriptor(); + nextBlock(); + } + } + + private boolean readSignature(boolean firstFrame) throws IOException { final byte[] b = new byte[4]; final int read = IOUtils.readFully(in, b); count(read); + if (4 != read && !firstFrame) { + endReached = true; + return false; + } if (4 != read || !matches(b, 4)) { throw new IOException("Not a LZ4 frame stream"); } + return true; } private void readFrameDescriptor() throws IOException { @@ -185,8 +210,12 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream { boolean uncompressed = (len & UNCOMPRESSED_FLAG_MASK) != 0; int realLen = (int) (len & (~UNCOMPRESSED_FLAG_MASK)); if (realLen == 0) { - endReached = true; verifyContentChecksum(); + if (!decompressConcatenated) { + endReached = true; + } else { + init(false); + } return; } InputStream capped = new BoundedInputStream(in, realLen); http://git-wip-us.apache.org/repos/asf/commons-compress/blob/25ecc9b1/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java index 3f5c701..0363e5e 100644 --- a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java +++ b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java @@ -27,6 +27,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.InputStream; import java.io.IOException; +import java.util.Arrays; import org.apache.commons.compress.AbstractTestCase; import org.apache.commons.compress.compressors.CompressorStreamFactory; @@ -82,6 +83,86 @@ public final class FramedLZ4CompressorInputStreamTest } } + @Test + public void readBlaLz4WithDecompressConcatenated() throws IOException { + try (InputStream a = new FramedLZ4CompressorInputStream(new FileInputStream(getFile("bla.tar.lz4")), true); + FileInputStream e = new FileInputStream(getFile("bla.tar"))) { + byte[] expected = IOUtils.toByteArray(e); + byte[] actual = IOUtils.toByteArray(a); + assertArrayEquals(expected, actual); + } + } + + @Test + public void readDoubledBlaLz4WithDecompressConcatenatedTrue() throws Exception { + readDoubledBlaLz4(new StreamWrapper() { + public InputStream wrap(InputStream in) throws Exception { + return new FramedLZ4CompressorInputStream(in, true); + } + }, true); + } + + @Test + public void readDoubledBlaLz4WithDecompressConcatenatedFalse() throws Exception { + readDoubledBlaLz4(new StreamWrapper() { + public InputStream wrap(InputStream in) throws Exception { + return new FramedLZ4CompressorInputStream(in, false); + } + }, false); + } + + @Test + public void readDoubledBlaLz4WithoutExplicitDecompressConcatenated() throws Exception { + readDoubledBlaLz4(new StreamWrapper() { + public InputStream wrap(InputStream in) throws Exception { + return new FramedLZ4CompressorInputStream(in); + } + }, false); + } + + @Test + public void readBlaLz4ViaFactoryWithDecompressConcatenated() throws Exception { + try (InputStream a = new CompressorStreamFactory() + .createCompressorInputStream(CompressorStreamFactory.getLZ4Framed(), + new FileInputStream(getFile("bla.tar.lz4")), + true); + FileInputStream e = new FileInputStream(getFile("bla.tar"))) { + byte[] expected = IOUtils.toByteArray(e); + byte[] actual = IOUtils.toByteArray(a); + assertArrayEquals(expected, actual); + } + } + + @Test + public void readDoubledBlaLz4ViaFactoryWithDecompressConcatenatedTrue() throws Exception { + readDoubledBlaLz4(new StreamWrapper() { + public InputStream wrap(InputStream in) throws Exception { + return new CompressorStreamFactory() + .createCompressorInputStream(CompressorStreamFactory.getLZ4Framed(), in, true); + } + }, true); + } + + @Test + public void readDoubledBlaLz4ViaFactoryWithDecompressConcatenatedFalse() throws Exception { + readDoubledBlaLz4(new StreamWrapper() { + public InputStream wrap(InputStream in) throws Exception { + return new CompressorStreamFactory() + .createCompressorInputStream(CompressorStreamFactory.getLZ4Framed(), in, false); + } + }, false); + } + + @Test + public void readDoubledBlaLz4ViaFactoryWithoutExplicitDecompressConcatenated() throws Exception { + readDoubledBlaLz4(new StreamWrapper() { + public InputStream wrap(InputStream in) throws Exception { + return new CompressorStreamFactory() + .createCompressorInputStream(CompressorStreamFactory.getLZ4Framed(), in); + } + }, false); + } + @Test(expected = IOException.class) public void rejectsNonLZ4Stream() throws IOException { try (InputStream a = new FramedLZ4CompressorInputStream(new FileInputStream(getFile("bla.tar")))) { @@ -280,4 +361,28 @@ public final class FramedLZ4CompressorInputStreamTest assertThat(ex.getMessage(), containsString("content checksum mismatch")); } } + + interface StreamWrapper { + InputStream wrap(InputStream in) throws Exception; + } + + private void readDoubledBlaLz4(StreamWrapper wrapper, boolean expectDuplicateOutput) throws Exception { + byte[] singleInput; + try (InputStream i = new FileInputStream(getFile("bla.tar.lz4"))) { + singleInput = IOUtils.toByteArray(i); + } + byte[] input = duplicate(singleInput); + try (InputStream a = wrapper.wrap(new ByteArrayInputStream(input)); + FileInputStream e = new FileInputStream(getFile("bla.tar"))) { + byte[] expected = IOUtils.toByteArray(e); + byte[] actual = IOUtils.toByteArray(a); + assertArrayEquals(expectDuplicateOutput ? duplicate(expected) : expected, actual); + } + } + + private static byte[] duplicate(byte[] from) { + byte[] to = Arrays.copyOf(from, 2 * from.length); + System.arraycopy(from, 0, to, from.length, from.length); + return to; + } }
