COMPRESS-271 content checksum for LZ4 frames
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/68fa6892 Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/68fa6892 Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/68fa6892 Branch: refs/heads/master Commit: 68fa68928bf3d431209dc0342126b9023de02c93 Parents: 9dc8d30 Author: Stefan Bodewig <bode...@apache.org> Authored: Thu Jan 26 06:33:44 2017 +0100 Committer: Stefan Bodewig <bode...@apache.org> Committed: Thu Jan 26 06:33:44 2017 +0100 ---------------------------------------------------------------------- .../lz4/FramedLZ4CompressorInputStream.java | 15 ++++++++++--- .../lz4/FramedLZ4CompressorOutputStream.java | 17 ++++++++++----- .../lz4/FramedLZ4CompressorInputStreamTest.java | 22 ++++++++++++++++++++ .../lz4/FramedLZ4CompressorRoundtripTest.java | 8 ++++--- 4 files changed, 51 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/commons-compress/blob/68fa6892/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java index 7cbd688..cc8ffed 100644 --- a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java +++ b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java @@ -125,6 +125,9 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream { r = readOnce(b, off, len); } } + if (expectContentChecksum && r != -1) { + contentHash.update(b, off, r); + } return r; } @@ -213,11 +216,17 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream { private void verifyContentChecksum() throws IOException { if (expectContentChecksum) { - int skipped = (int) IOUtils.skip(in, 4); - count(skipped); - if (4 != skipped) { + byte[] checksum = new byte[4]; + int read = IOUtils.readFully(in, checksum); + count(read); + if (4 != read) { throw new IOException("Premature end of stream while reading content checksum"); } + long expectedHash = contentHash.getValue(); + if (expectedHash != ByteUtils.fromLittleEndian(checksum)) { + throw new IOException("content checksum mismatch."); + } + contentHash.reset(); } } http://git-wip-us.apache.org/repos/asf/commons-compress/blob/68fa6892/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java index b7ad0d0..525e05b 100644 --- a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java +++ b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java @@ -40,7 +40,7 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream { /* * TODO before releasing 1.14: * - * + xxhash32 checksum creation for content, blocks + * + xxhash32 checksum creation for blocks * + block dependence */ @@ -89,6 +89,7 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream { */ public static class Parameters { private final BlockSize blockSize; + private final boolean withContentChecksum; /** * The default parameters of 4M block size, enabled content @@ -96,18 +97,20 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream { * * <p>This matches the defaults of the lz4 command line utility.</p> */ - public static Parameters DEFAULT = new Parameters(BlockSize.M4); + public static Parameters DEFAULT = new Parameters(BlockSize.M4, true); /** * Sets up custom parameters for the LZ4 stream. * @param blockSize the size of a single block. + * @param withContentChecksum whether to write a content checksum */ - public Parameters(BlockSize blockSize) { + public Parameters(BlockSize blockSize, boolean withContentChecksum) { this.blockSize = blockSize; + this.withContentChecksum = withContentChecksum; } @Override public String toString() { - return "LZ4 Parameters with BlockSize " + blockSize; + return "LZ4 Parameters with BlockSize " + blockSize + ", withContentChecksum " + withContentChecksum; } } @@ -144,6 +147,7 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream { @Override public void write(byte[] data, int off, int len) throws IOException { + contentHash.update(data, off, len); if (currentIndex + len > blockData.length) { flushBlock(); while (len > blockData.length) { @@ -182,6 +186,9 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream { private void writeFrameDescriptor() throws IOException { int flags = FramedLZ4CompressorInputStream.SUPPORTED_VERSION | FramedLZ4CompressorInputStream.BLOCK_INDEPENDENCE_MASK; + if (params.withContentChecksum) { + flags |= FramedLZ4CompressorInputStream.CONTENT_CHECKSUM_MASK; + } out.write(flags); contentHash.update(flags); int bd = params.blockSize.getIndex() << 4; @@ -211,7 +218,7 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream { private void writeTrailer() throws IOException { out.write(END_MARK); - // TODO content checksum + ByteUtils.toLittleEndian(out, contentHash.getValue(), 4); } } http://git-wip-us.apache.org/repos/asf/commons-compress/blob/68fa6892/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java index 0893fbe..3f5c701 100644 --- a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java +++ b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java @@ -258,4 +258,26 @@ public final class FramedLZ4CompressorInputStreamTest assertThat(ex.getMessage(), containsString("content checksum")); } } + + @Test + public void rejectsStreamsWithBadContentChecksum() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x64, // flag - Version 01, block independent, no block checksum, no content size, with content checksum + 0x70, // block size 4MB + (byte) 185, // checksum + 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set + 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content + 0, 0, 0, 0, // empty block marker + 1, 2, 3, 4, + }; + try { + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input))) { + IOUtils.toByteArray(a); + fail("expected exception"); + } + } catch (IOException ex) { + assertThat(ex.getMessage(), containsString("content checksum mismatch")); + } + } } http://git-wip-us.apache.org/repos/asf/commons-compress/blob/68fa6892/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java index 9744df7..a6aa5aa 100644 --- a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java +++ b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorRoundtripTest.java @@ -39,10 +39,12 @@ public final class FramedLZ4CompressorRoundtripTest extends AbstractTestCase { @Parameters(name = "using {0}") public static Collection<Object[]> factory() { return Arrays.asList(new Object[][] { - new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.K64) }, - new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.K256) }, - new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.M1) }, + new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.K64, true) }, + new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.K256, true) }, + new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.M1, true) }, new Object[] { FramedLZ4CompressorOutputStream.Parameters.DEFAULT }, + // default without content checksum + new Object[] { new FramedLZ4CompressorOutputStream.Parameters(FramedLZ4CompressorOutputStream.BlockSize.M4, false) }, }); }
