Repository: commons-compress Updated Branches: refs/heads/master 72fec65e1 -> e79465bbe
COMPRESS-271 verify block checksums Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/982ce0ec Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/982ce0ec Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/982ce0ec Branch: refs/heads/master Commit: 982ce0eca9935e65915278ae07febd3a1d52273d Parents: 72fec65 Author: Stefan Bodewig <bode...@apache.org> Authored: Thu Feb 9 17:37:01 2017 +0100 Committer: Stefan Bodewig <bode...@apache.org> Committed: Thu Feb 9 17:37:01 2017 +0100 ---------------------------------------------------------------------- .../lz4/FramedLZ4CompressorInputStream.java | 38 +++++--- .../utils/ChecksumCalculatingInputStream.java | 97 ++++++++++++++++++++ 2 files changed, 120 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/commons-compress/blob/982ce0ec/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java index 9b47d7b..8bf49ac 100644 --- a/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java +++ b/src/main/java/org/apache/commons/compress/compressors/lz4/FramedLZ4CompressorInputStream.java @@ -25,6 +25,7 @@ import java.util.Arrays; import org.apache.commons.compress.compressors.CompressorInputStream; import org.apache.commons.compress.utils.BoundedInputStream; import org.apache.commons.compress.utils.ByteUtils; +import org.apache.commons.compress.utils.ChecksumCalculatingInputStream; import org.apache.commons.compress.utils.IOUtils; /** @@ -80,6 +81,9 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream { // used for frame header checksum and content checksum, if present private final XXHash32 contentHash = new XXHash32(); + // used for block checksum, if present + private final XXHash32 blockHash = new XXHash32(); + // only created if the frame doesn't set the block independence flag private byte[] blockDependencyBuffer; @@ -239,6 +243,9 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream { return; } InputStream capped = new BoundedInputStream(in, realLen); + if (expectBlockChecksum) { + capped = new ChecksumCalculatingInputStream(blockHash, capped); + } if (uncompressed) { inUncompressed = true; currentBlock = capped; @@ -257,31 +264,32 @@ public class FramedLZ4CompressorInputStream extends CompressorInputStream { currentBlock.close(); currentBlock = null; if (expectBlockChecksum) { - int skipped = (int) IOUtils.skip(in, 4); - count(skipped); - if (4 != skipped) { - throw new IOException("Premature end of stream while reading block checksum"); - } + verifyChecksum(blockHash, "block"); + blockHash.reset(); } } } private void verifyContentChecksum() throws IOException { if (expectContentChecksum) { - byte[] checksum = new byte[4]; - int read = IOUtils.readFully(in, checksum); - count(read); - if (4 != read) { - throw new IOException("Premature end of stream while reading content checksum"); - } - long expectedHash = contentHash.getValue(); - if (expectedHash != ByteUtils.fromLittleEndian(checksum)) { - throw new IOException("content checksum mismatch."); - } + verifyChecksum(contentHash, "content"); } contentHash.reset(); } + private void verifyChecksum(XXHash32 hash, String kind) throws IOException { + byte[] checksum = new byte[4]; + int read = IOUtils.readFully(in, checksum); + count(read); + if (4 != read) { + throw new IOException("Premature end of stream while reading " + kind + " checksum"); + } + long expectedHash = hash.getValue(); + if (expectedHash != ByteUtils.fromLittleEndian(checksum)) { + throw new IOException(kind + " checksum mismatch."); + } + } + private int readOneByte() throws IOException { final int b = in.read(); if (b != -1) { http://git-wip-us.apache.org/repos/asf/commons-compress/blob/982ce0ec/src/main/java/org/apache/commons/compress/utils/ChecksumCalculatingInputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/utils/ChecksumCalculatingInputStream.java b/src/main/java/org/apache/commons/compress/utils/ChecksumCalculatingInputStream.java new file mode 100644 index 0000000..a698196 --- /dev/null +++ b/src/main/java/org/apache/commons/compress/utils/ChecksumCalculatingInputStream.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.commons.compress.utils; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.zip.Checksum; + +/** + * A stream that calculates the checksum of the data read. + * @NotThreadSafe + * @since 1.14 + */ +public class ChecksumCalculatingInputStream extends InputStream { + private final InputStream in; + private final Checksum checksum; + + public ChecksumCalculatingInputStream(final Checksum checksum, final InputStream in) { + this.checksum = checksum; + this.in = in; + } + + /** + * Reads a single byte from the stream + * @throws IOException if the underlying stream throws or the + * stream is exhausted and the Checksum doesn't match the expected + * value + */ + @Override + public int read() throws IOException { + final int ret = in.read(); + if (ret >= 0) { + checksum.update(ret); + } + return ret; + } + + /** + * Reads a byte array from the stream + * @throws IOException if the underlying stream throws or the + * stream is exhausted and the Checksum doesn't match the expected + * value + */ + @Override + public int read(final byte[] b) throws IOException { + return read(b, 0, b.length); + } + + /** + * Reads from the stream into a byte array. + * @throws IOException if the underlying stream throws or the + * stream is exhausted and the Checksum doesn't match the expected + * value + */ + @Override + public int read(final byte[] b, final int off, final int len) throws IOException { + final int ret = in.read(b, off, len); + if (ret >= 0) { + checksum.update(b, off, ret); + } + return ret; + } + + @Override + public long skip(final long n) throws IOException { + // Can't really skip, we have to hash everything to verify the checksum + if (read() >= 0) { + return 1; + } + return 0; + } + + /** + * Returns the calculated checksum. + * @return the calculated checksum. + */ + public long getValue() { + return checksum.getValue(); + } + +}
