COMPRESS-382 and COMPRESS-386 -- hard reset to expunge static parameter proposal. Updated MemoryLimitException to include cause when it exists. Updated LZWInputStream to calculate estimated memory on maxTableSize*6 to account for int[] and byte[], byte[].
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/4e442bb4 Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/4e442bb4 Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/4e442bb4 Branch: refs/heads/master Commit: 4e442bb44280243f6e0e72d845fa8275d665ca84 Parents: b10528a Author: tballison <talli...@mitre.org> Authored: Tue Apr 25 08:44:21 2017 -0400 Committer: tballison <talli...@mitre.org> Committed: Tue Apr 25 08:44:21 2017 -0400 ---------------------------------------------------------------------- .../commons/compress/MemoryLimitException.java | 30 +++++++++++++++++--- .../lzma/LZMACompressorInputStream.java | 2 +- .../compressors/lzw/LZWInputStream.java | 20 +++++++++---- .../compressors/xz/XZCompressorInputStream.java | 8 +++--- .../compressors/z/ZCompressorInputStream.java | 2 +- .../compressors/DetectCompressorTestCase.java | 2 +- 6 files changed, 47 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4e442bb4/src/main/java/org/apache/commons/compress/MemoryLimitException.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/MemoryLimitException.java b/src/main/java/org/apache/commons/compress/MemoryLimitException.java index 8922ed2..122edb1 100644 --- a/src/main/java/org/apache/commons/compress/MemoryLimitException.java +++ b/src/main/java/org/apache/commons/compress/MemoryLimitException.java @@ -30,11 +30,33 @@ import java.io.IOException; */ public class MemoryLimitException extends IOException { - public MemoryLimitException(String message) { - super(message); + //long instead of int to account for overflow for corrupt files + private final long memoryNeededInKb; + private final int memoryLimitInKb; + + public MemoryLimitException(long memoryNeededInKb, int memoryLimitInKb) { + super(buildMessage(memoryNeededInKb, memoryLimitInKb)); + this.memoryNeededInKb = memoryNeededInKb; + this.memoryLimitInKb = memoryLimitInKb; + } + + public MemoryLimitException(long memoryNeededInKb, int memoryLimitInKb, Exception e) { + super(buildMessage(memoryNeededInKb, memoryLimitInKb), e); + this.memoryNeededInKb = memoryNeededInKb; + this.memoryLimitInKb = memoryLimitInKb; + } + + public long getMemoryNeededInKb() { + return memoryNeededInKb; + } + + public int getMemoryLimitInKb() { + return memoryLimitInKb; } - public MemoryLimitException(String message, Exception e) { - super(message, e); + private static String buildMessage(long memoryNeededInKb, int memoryLimitInKb) { + return "" + memoryNeededInKb + " kb of memory would be needed; limit was " + + memoryLimitInKb + " kb. " + + "If the file is not corrupt, consider increasing the memory limit."; } } http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4e442bb4/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java index 7782be8..5723b5c 100644 --- a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java +++ b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java @@ -61,7 +61,7 @@ public class LZMACompressorInputStream extends CompressorInputStream { in = new LZMAInputStream(inputStream, memoryLimitInKb); } catch (org.tukaani.xz.MemoryLimitException e) { //convert to commons-compress exception - throw new MemoryLimitException("exceeded calculated memory limit", e); + throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit(), e); } } http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4e442bb4/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java index 350b4b0..2a6127d 100644 --- a/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java +++ b/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java @@ -114,16 +114,24 @@ public abstract class LZWInputStream extends CompressorInputStream { /** * Initializes the arrays based on the maximum code size. + * First checks that the estimated memory usage is below memoryLimitInKb + * * @param maxCodeSize maximum code size - * @param memoryLimitInKb maximum allowed table size in Kb - * @throws MemoryLimitException if maxTableSize is > memoryLimitInKb + * @param memoryLimitInKb maximum allowed estimated memory usage in Kb + * @throws MemoryLimitException if estimated memory usage is greater than memoryLimitInKb */ protected void initializeTables(final int maxCodeSize, final int memoryLimitInKb) throws MemoryLimitException { - final int maxTableSize = 1 << maxCodeSize; - if (memoryLimitInKb > -1 && maxTableSize > memoryLimitInKb*1024) { - throw new MemoryLimitException("Tried to allocate "+maxTableSize + - " but memoryLimitInKb only allows "+(memoryLimitInKb*1024)); + + if (memoryLimitInKb > -1) { + final int maxTableSize = 1 << maxCodeSize; + //account for potential overflow + long memoryUsageInBytes = (long) maxTableSize * 6;//(4 (prefixes) + 1 (characters) +1 (outputStack)) + long memoryUsageInKb = memoryUsageInBytes >> 10; + + if (memoryUsageInKb > (long)memoryLimitInKb) { + throw new MemoryLimitException(memoryUsageInKb, memoryLimitInKb); + } } initializeTables(maxCodeSize); } http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4e442bb4/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java index b378212..fc0cbf5 100644 --- a/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java +++ b/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java @@ -120,7 +120,7 @@ public class XZCompressorInputStream extends CompressorInputStream { * @since 1.14 */ public XZCompressorInputStream(InputStream inputStream, - boolean decompressConcatenated, int memoryLimitInKb) + boolean decompressConcatenated, final int memoryLimitInKb) throws IOException { if (decompressConcatenated) { in = new XZInputStream(inputStream, memoryLimitInKb); @@ -136,7 +136,7 @@ public class XZCompressorInputStream extends CompressorInputStream { count(ret == -1 ? -1 : 1); return ret; } catch (org.tukaani.xz.MemoryLimitException e) { - throw new MemoryLimitException("Exceeded memory limit", e); + throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit(), e); } } @@ -148,7 +148,7 @@ public class XZCompressorInputStream extends CompressorInputStream { return ret; } catch (org.tukaani.xz.MemoryLimitException e) { //convert to commons-compress MemoryLimtException - throw new MemoryLimitException("Exceeded memory limit", e); + throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit(), e); } } @@ -158,7 +158,7 @@ public class XZCompressorInputStream extends CompressorInputStream { return in.skip(n); } catch (org.tukaani.xz.MemoryLimitException e) { //convert to commons-compress MemoryLimtException - throw new MemoryLimitException("Excedded memory limit", e); + throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit(), e); } } http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4e442bb4/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java index 64387e3..994e102 100644 --- a/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java +++ b/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java @@ -38,7 +38,7 @@ public class ZCompressorInputStream extends LZWInputStream { private final int maxCodeSize; private long totalCodesRead = 0; - public ZCompressorInputStream(final InputStream inputStream, int memoryLimitInKb) + public ZCompressorInputStream(final InputStream inputStream, final int memoryLimitInKb) throws IOException { super(inputStream, ByteOrder.LITTLE_ENDIAN); final int firstByte = (int) in.readBits(8); http://git-wip-us.apache.org/repos/asf/commons-compress/blob/4e442bb4/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java b/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java index b70d3c7..844b168 100644 --- a/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java +++ b/src/test/java/org/apache/commons/compress/compressors/DetectCompressorTestCase.java @@ -210,7 +210,7 @@ public final class DetectCompressorTestCase { return fac.createCompressorInputStream(is); } catch (CompressorException e) { if (e.getCause() != null && e.getCause() instanceof Exception) { - //unwrap cause to reveal MemoryLimiteException + //unwrap cause to reveal MemoryLimitException throw (Exception)e.getCause(); } else { throw e;
