COMPRESS-382 and COMPRESS-386 -- break out unit tests; add memory limit for xz.
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/b10528a6 Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/b10528a6 Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/b10528a6 Branch: refs/heads/master Commit: b10528a62e51b2c5fdc0c4b7884cc93f03f8ce96 Parents: 7d73baf Author: tballison <talli...@mitre.org> Authored: Mon Apr 24 10:10:30 2017 -0400 Committer: tballison <talli...@mitre.org> Committed: Mon Apr 24 10:10:30 2017 -0400 ---------------------------------------------------------------------- .../commons/compress/MemoryLimitException.java | 40 +++++++++++++ .../CompressorMemoryLimitException.java | 36 ------------ .../compressors/CompressorStreamFactory.java | 19 ++---- .../lzma/LZMACompressorInputStream.java | 9 ++- .../compressors/lzw/LZWInputStream.java | 8 +-- .../compressors/xz/XZCompressorInputStream.java | 62 +++++++++++++++++--- .../compressors/z/ZCompressorInputStream.java | 15 +---- .../compressors/DetectCompressorTestCase.java | 54 +++++++++++++---- 8 files changed, 154 insertions(+), 89 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b10528a6/src/main/java/org/apache/commons/compress/MemoryLimitException.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/MemoryLimitException.java b/src/main/java/org/apache/commons/compress/MemoryLimitException.java new file mode 100644 index 0000000..8922ed2 --- /dev/null +++ b/src/main/java/org/apache/commons/compress/MemoryLimitException.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.commons.compress; + +import java.io.IOException; + +/** + * If a stream checks for estimated memory allocation, and the estimate + * goes above the memory limit, this is thrown. This can also be thrown + * if a stream tries to allocate a byte array that is larger than + * the allowable limit. + * + * @since 1.14 + */ +public class MemoryLimitException extends IOException { + + public MemoryLimitException(String message) { + super(message); + } + + public MemoryLimitException(String message, Exception e) { + super(message, e); + } +} http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b10528a6/src/main/java/org/apache/commons/compress/compressors/CompressorMemoryLimitException.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressorMemoryLimitException.java b/src/main/java/org/apache/commons/compress/compressors/CompressorMemoryLimitException.java deleted file mode 100644 index 4c87d1e..0000000 --- a/src/main/java/org/apache/commons/compress/compressors/CompressorMemoryLimitException.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.commons.compress.compressors; - -/** - * If a stream checks for estimated memory allocation, and the estimate - * goes above the memory limit, this is thrown. - * - * @since 1.14 - */ -public class CompressorMemoryLimitException extends CompressorException { - - public CompressorMemoryLimitException(String message) { - super(message); - } - - public CompressorMemoryLimitException(String message, Exception e) { - super(message, e); - } -} http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b10528a6/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java index a0a816f..f3433d9 100644 --- a/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java +++ b/src/main/java/org/apache/commons/compress/compressors/CompressorStreamFactory.java @@ -57,7 +57,6 @@ import org.apache.commons.compress.utils.IOUtils; import org.apache.commons.compress.utils.Lists; import org.apache.commons.compress.utils.ServiceLoaderIterator; import org.apache.commons.compress.utils.Sets; -import org.tukaani.xz.MemoryLimitException; /** * <p> @@ -498,7 +497,9 @@ public class CompressorStreamFactory implements CompressorStreamProvider { * the input stream * @return compressor input stream * @throws CompressorException - * if the compressor name is not known or not available + * if the compressor name is not known or not available, + * or if there's an IOException or MemoryLimitException thrown + * during initialization * @throws IllegalArgumentException * if the name or input stream is null */ @@ -528,18 +529,14 @@ public class CompressorStreamFactory implements CompressorStreamProvider { if (!XZUtils.isXZCompressionAvailable()) { throw new CompressorException("XZ compression is not available."); } - return new XZCompressorInputStream(in, actualDecompressConcatenated); + return new XZCompressorInputStream(in, actualDecompressConcatenated, memoryLimitInKb); } if (LZMA.equalsIgnoreCase(name)) { if (!LZMAUtils.isLZMACompressionAvailable()) { throw new CompressorException("LZMA compression is not available"); } - try { - return new LZMACompressorInputStream(in, memoryLimitInKb); - } catch (MemoryLimitException e) { - throw new CompressorMemoryLimitException("exceeded calculated memory limit", e); - } + return new LZMACompressorInputStream(in, memoryLimitInKb); } if (PACK200.equalsIgnoreCase(name)) { @@ -555,11 +552,7 @@ public class CompressorStreamFactory implements CompressorStreamProvider { } if (Z.equalsIgnoreCase(name)) { - try { - return new ZCompressorInputStream(in, memoryLimitInKb); - } catch (ZCompressorInputStream.IOExceptionWrappingMemoryLimitException e) { - throw new CompressorMemoryLimitException(e.getMessage()); - } + return new ZCompressorInputStream(in, memoryLimitInKb); } if (DEFLATE.equalsIgnoreCase(name)) { http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b10528a6/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java index 0520e62..7782be8 100644 --- a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java +++ b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java @@ -20,6 +20,8 @@ package org.apache.commons.compress.compressors.lzma; import java.io.IOException; import java.io.InputStream; + +import org.apache.commons.compress.MemoryLimitException; import org.tukaani.xz.LZMAInputStream; import org.apache.commons.compress.compressors.CompressorInputStream; @@ -55,7 +57,12 @@ public class LZMACompressorInputStream extends CompressorInputStream { */ public LZMACompressorInputStream(final InputStream inputStream, int memoryLimitInKb) throws IOException { - in = new LZMAInputStream(inputStream, memoryLimitInKb); + try { + in = new LZMAInputStream(inputStream, memoryLimitInKb); + } catch (org.tukaani.xz.MemoryLimitException e) { + //convert to commons-compress exception + throw new MemoryLimitException("exceeded calculated memory limit", e); + } } /** {@inheritDoc} */ http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b10528a6/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java index b61d9a1..350b4b0 100644 --- a/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java +++ b/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java @@ -22,8 +22,8 @@ import java.io.IOException; import java.io.InputStream; import java.nio.ByteOrder; +import org.apache.commons.compress.MemoryLimitException; import org.apache.commons.compress.compressors.CompressorInputStream; -import org.apache.commons.compress.compressors.CompressorMemoryLimitException; import org.apache.commons.compress.utils.BitInputStream; /** @@ -116,13 +116,13 @@ public abstract class LZWInputStream extends CompressorInputStream { * Initializes the arrays based on the maximum code size. * @param maxCodeSize maximum code size * @param memoryLimitInKb maximum allowed table size in Kb - * @throws CompressorMemoryLimitException if maxTableSize is > memoryLimitInKb + * @throws MemoryLimitException if maxTableSize is > memoryLimitInKb */ protected void initializeTables(final int maxCodeSize, final int memoryLimitInKb) - throws CompressorMemoryLimitException { + throws MemoryLimitException { final int maxTableSize = 1 << maxCodeSize; if (memoryLimitInKb > -1 && maxTableSize > memoryLimitInKb*1024) { - throw new CompressorMemoryLimitException("Tried to allocate "+maxTableSize + + throw new MemoryLimitException("Tried to allocate "+maxTableSize + " but memoryLimitInKb only allows "+(memoryLimitInKb*1024)); } initializeTables(maxCodeSize); http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b10528a6/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java index 27d70d2..b378212 100644 --- a/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java +++ b/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java @@ -20,6 +20,8 @@ package org.apache.commons.compress.compressors.xz; import java.io.IOException; import java.io.InputStream; + +import org.apache.commons.compress.MemoryLimitException; import org.tukaani.xz.XZ; import org.tukaani.xz.SingleXZInputStream; import org.tukaani.xz.XZInputStream; @@ -92,30 +94,72 @@ public class XZCompressorInputStream extends CompressorInputStream { public XZCompressorInputStream(final InputStream inputStream, final boolean decompressConcatenated) throws IOException { + this(inputStream, decompressConcatenated, -1); + } + + /** + * Creates a new input stream that decompresses XZ-compressed data + * from the specified input stream. + * + * @param inputStream where to read the compressed data + * @param decompressConcatenated + * if true, decompress until the end of the + * input; if false, stop after the first .xz + * stream and leave the input position to point + * to the next byte after the .xz stream + * @param memoryLimitInKb memory limit used when reading blocks. If + * the estimated memory limit is exceeded on {@link #read()}, + * a {@link MemoryLimitException} is thrown. + * + * @throws IOException if the input is not in the .xz format, + * the input is corrupt or truncated, the .xz + * headers specify options that are not supported + * by this implementation, + * or the underlying <code>inputStream</code> throws an exception + * + * @since 1.14 + */ + public XZCompressorInputStream(InputStream inputStream, + boolean decompressConcatenated, int memoryLimitInKb) + throws IOException { if (decompressConcatenated) { - in = new XZInputStream(inputStream); + in = new XZInputStream(inputStream, memoryLimitInKb); } else { - in = new SingleXZInputStream(inputStream); + in = new SingleXZInputStream(inputStream, memoryLimitInKb); } } @Override public int read() throws IOException { - final int ret = in.read(); - count(ret == -1 ? -1 : 1); - return ret; + try { + final int ret = in.read(); + count(ret == -1 ? -1 : 1); + return ret; + } catch (org.tukaani.xz.MemoryLimitException e) { + throw new MemoryLimitException("Exceeded memory limit", e); + } } @Override public int read(final byte[] buf, final int off, final int len) throws IOException { - final int ret = in.read(buf, off, len); - count(ret); - return ret; + try { + final int ret = in.read(buf, off, len); + count(ret); + return ret; + } catch (org.tukaani.xz.MemoryLimitException e) { + //convert to commons-compress MemoryLimtException + throw new MemoryLimitException("Exceeded memory limit", e); + } } @Override public long skip(final long n) throws IOException { - return in.skip(n); + try { + return in.skip(n); + } catch (org.tukaani.xz.MemoryLimitException e) { + //convert to commons-compress MemoryLimtException + throw new MemoryLimitException("Excedded memory limit", e); + } } @Override http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b10528a6/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java index 22d7c0c..64387e3 100644 --- a/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java +++ b/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.io.InputStream; import java.nio.ByteOrder; -import org.apache.commons.compress.compressors.CompressorMemoryLimitException; import org.apache.commons.compress.compressors.lzw.LZWInputStream; /** @@ -53,11 +52,7 @@ public class ZCompressorInputStream extends LZWInputStream { if (blockMode) { setClearCode(DEFAULT_CODE_SIZE); } - try { - initializeTables(maxCodeSize, memoryLimitInKb); - } catch (CompressorMemoryLimitException e) { - throw new IOExceptionWrappingMemoryLimitException(e.getMessage()); - } + initializeTables(maxCodeSize, memoryLimitInKb); clearEntries(); } @@ -173,12 +168,4 @@ public class ZCompressorInputStream extends LZWInputStream { return length > 3 && signature[0] == MAGIC_1 && signature[1] == (byte) MAGIC_2; } - /** - * Wrapper that subclasses IOException to wrap a MemoryLimitException - */ - public static class IOExceptionWrappingMemoryLimitException extends IOException { - public IOExceptionWrappingMemoryLimitException(String message) { - super(message); - } - } } http://git-wip-us.apache.org/repos/asf/commons-compress/blob/b10528a6/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java b/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java index 7cec5df..b70d3c7 100644 --- a/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java +++ b/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java @@ -31,6 +31,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import org.apache.commons.compress.MemoryLimitException; import org.apache.commons.compress.MockEvilInputStream; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.commons.compress.compressors.deflate.DeflateCompressorInputStream; @@ -170,21 +171,50 @@ public final class DetectCompressorTestCase { return name; } - @Test - public void testMemoryLimit() throws Exception { - testMemoryLimit("COMPRESS-382"); - testMemoryLimit("COMPRESS-386"); + @Test(expected = MemoryLimitException.class) + public void testLZMAMemoryLimit() throws Exception { + getStreamFor("COMPRESS-382", 100); } - private void testMemoryLimit(String fileName) throws IOException, CompressorException { - CompressorStreamFactory fac = new CompressorStreamFactory(true, - 100); - try (InputStream is = new BufferedInputStream( - new FileInputStream(getFile(fileName)))) { - InputStream compressorInputStream = fac.createCompressorInputStream(is); - fail("Should have thrown CompressorMemoryLimitException"); - } catch (CompressorMemoryLimitException e) { + @Test(expected = MemoryLimitException.class) + public void testZMemoryLimit() throws Exception { + getStreamFor("COMPRESS-386", 100); + } + + @Test(expected = MemoryLimitException.class) + public void testXZMemoryLimitOnRead() throws Exception { + //Even though the file is very small, the memory limit + //has to be quite large (8296 KiB) because of the dictionary size + + //This is triggered on read(); not during initialization. + //This test is here instead of the xz unit test to make sure + //that the parameter is properly passed via the CompressorStreamFactory + try (InputStream compressorIs = getStreamFor("bla.tar.xz", 100)) { + int c = compressorIs.read(); + } + } + @Test(expected = MemoryLimitException.class) + public void testXZMemoryLimitOnSkip() throws Exception { + try (InputStream compressorIs = getStreamFor("bla.tar.xz", 100)) { + compressorIs.skip(10); + } + } + + private InputStream getStreamFor(final String fileName, final int memoryLimitInKb) throws Exception { + CompressorStreamFactory fac = new CompressorStreamFactory(true, + memoryLimitInKb); + InputStream is = new BufferedInputStream( + new FileInputStream(getFile(fileName))); + try { + return fac.createCompressorInputStream(is); + } catch (CompressorException e) { + if (e.getCause() != null && e.getCause() instanceof Exception) { + //unwrap cause to reveal MemoryLimiteException + throw (Exception)e.getCause(); + } else { + throw e; + } } }
