Repository: commons-compress Updated Branches: refs/heads/master b5d6f1f62 -> 76d913a3c
COMPRESS-271 frame header checksums Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/76d913a3 Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/76d913a3 Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/76d913a3 Branch: refs/heads/master Commit: 76d913a3cd2224a1e6a13af77f77709ee1b265c6 Parents: b5d6f1f Author: Stefan Bodewig <bode...@apache.org> Authored: Wed Jan 25 06:36:12 2017 +0100 Committer: Stefan Bodewig <bode...@apache.org> Committed: Wed Jan 25 06:36:12 2017 +0100 ---------------------------------------------------------------------- .../lz4/FramedLZ4CompressorInputStream.java | 20 +++++++++++++--- .../lz4/FramedLZ4CompressorOutputStream.java | 18 ++++++++++---- .../lz4/FramedLZ4CompressorInputStreamTest.java | 25 ++++++++++++++++---- 3 files changed, 51 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/commons-compress/blob/76d913a3/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java index a7f4a4d..7cbd688 100644 --- a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java +++ b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java @@ -80,6 +80,9 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream { private InputStream currentBlock; private boolean endReached, inUncompressed; + // used for frame header checksum and content checksum, if present + private final XXHash32 contentHash = new XXHash32(); + /** * Creates a new input stream that decompresses streams compressed * using the LZ4 frame format. @@ -139,6 +142,7 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream { if (flags == -1) { throw new IOException("Premature end of stream while reading frame flags"); } + contentHash.update(flags); if ((flags & VERSION_MASK) != SUPPORTED_VERSION) { throw new IOException("Unsupported version " + (flags >> 6)); } @@ -148,19 +152,29 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream { expectBlockChecksum = (flags & BLOCK_CHECKSUM_MASK) != 0; expectContentSize = (flags & CONTENT_SIZE_MASK) != 0; expectContentChecksum = (flags & CONTENT_CHECKSUM_MASK) != 0; - if (readOneByte() == -1) { // max size is irrelevant for this implementation + int bdByte = readOneByte(); + if (bdByte == -1) { // max size is irrelevant for this implementation throw new IOException("Premature end of stream while reading frame BD byte"); } + contentHash.update(bdByte); if (expectContentSize) { // for now we don't care, contains the uncompressed size - int skipped = (int) IOUtils.skip(in, 8); + byte[] contentSize = new byte[8]; + int skipped = (int) IOUtils.readFully(in, contentSize); count(skipped); if (8 != skipped) { throw new IOException("Premature end of stream while reading content size"); } + contentHash.update(contentSize, 0, contentSize.length); } - if (readOneByte() == -1) { // partial hash of header. not supported, yet + int headerHash = readOneByte(); + if (headerHash == -1) { // partial hash of header. throw new IOException("Premature end of stream while reading frame header checksum"); } + int expectedHash = (int) ((contentHash.getValue() >> 8) & 0xff); + contentHash.reset(); + if (headerHash != expectedHash) { + throw new IOException("frame header checksum mismatch."); + } } private void nextBlock() throws IOException { http://git-wip-us.apache.org/repos/asf/commons-compress/blob/76d913a3/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java index f240e0b..e0622e1 100644 --- a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java +++ b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorOutputStream.java @@ -40,7 +40,7 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream { /* * TODO before releasing 1.14: * - * + xxhash32 checksum creation for headers, content, blocks + * + xxhash32 checksum creation for content, blocks * + block dependence */ @@ -57,6 +57,9 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream { private boolean finished = false; private int currentIndex = 0; + // used for frame header checksum and content checksum, if present + private final XXHash32 contentHash = new XXHash32(); + /** * Constructs a new output stream that compresses data using the * LZ4 frame format using the default block size of 4MB. @@ -129,10 +132,15 @@ public class FramedLZ4CompressorOutputStream extends CompressorOutputStream { } private void writeFrameDescriptor() throws IOException { - out.write(FramedLZ4CompressorInputStream.SUPPORTED_VERSION - | FramedLZ4CompressorInputStream.BLOCK_INDEPENDENCE_MASK); - out.write(BLOCK_SIZES.indexOf(blockData.length) << 4); - out.write(0); // TODO header checksum + int flags = FramedLZ4CompressorInputStream.SUPPORTED_VERSION + | FramedLZ4CompressorInputStream.BLOCK_INDEPENDENCE_MASK; + out.write(flags); + contentHash.update(flags); + int bd = BLOCK_SIZES.indexOf(blockData.length) << 4; + out.write(bd); + contentHash.update(bd); + out.write((int) ((contentHash.getValue() >> 8) & 0xff)); + contentHash.reset(); } private void flushBlock() throws IOException { http://git-wip-us.apache.org/repos/asf/commons-compress/blob/76d913a3/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java index 3727518..0893fbe 100644 --- a/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java +++ b/src/test/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStreamTest.java @@ -166,12 +166,29 @@ public final class FramedLZ4CompressorInputStreamTest } @Test + public void rejectsFileWithBadHeaderChecksum() throws IOException { + byte[] input = new byte[] { + 4, 0x22, 0x4d, 0x18, // signature + 0x64, // flag - Version 01, block independent, no block checksum, no content size, with content checksum + 0x70, // block size 4MB + 0, + }; + try { + try (InputStream a = new FramedLZ4CompressorInputStream(new ByteArrayInputStream(input))) { + fail("expected exception"); + } + } catch (IOException ex) { + assertThat(ex.getMessage(), containsString("header checksum mismatch")); + } + } + + @Test public void readsUncompressedBlocks() throws IOException { byte[] input = new byte[] { 4, 0x22, 0x4d, 0x18, // signature 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum 0x70, // block size 4MB - 0x00, // checksum, revisit once it gets validated + 115, // checksum 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content 0, 0, 0, 0, // empty block marker @@ -190,7 +207,7 @@ public final class FramedLZ4CompressorInputStreamTest 4, 0x22, 0x4d, 0x18, // signature 0x60, // flag - Version 01, block independent, no block checksum, no content size, no content checksum 0x70, // block size 4MB - 0x00, // checksum, revisit once it gets validated + 115, // checksum 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content 0, 0, 0, 0, // empty block marker @@ -207,7 +224,7 @@ public final class FramedLZ4CompressorInputStreamTest 4, 0x22, 0x4d, 0x18, // signature 0x70, // flag - Version 01, block independent, with block checksum, no content size, no content checksum 0x70, // block size 4MB - 0x00, // checksum, revisit once it gets validated + 114, // checksum 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content }; @@ -227,7 +244,7 @@ public final class FramedLZ4CompressorInputStreamTest 4, 0x22, 0x4d, 0x18, // signature 0x64, // flag - Version 01, block independent, no block checksum, no content size, with content checksum 0x70, // block size 4MB - 0x00, // checksum, revisit once it gets validated + (byte) 185, // checksum 13, 0, 0, (byte) 0x80, // 13 bytes length and uncompressed bit set 'H', 'e', 'l', 'l', 'o', ',', ' ', 'w', 'o', 'r', 'l', 'd', '!', // content 0, 0, 0, 0, // empty block marker
