COMPRESS-382 and COMPRESS-386 -- take 4, clean up and allow for overflow via longs.
Project: http://git-wip-us.apache.org/repos/asf/commons-compress/repo Commit: http://git-wip-us.apache.org/repos/asf/commons-compress/commit/26eab98e Tree: http://git-wip-us.apache.org/repos/asf/commons-compress/tree/26eab98e Diff: http://git-wip-us.apache.org/repos/asf/commons-compress/diff/26eab98e Branch: refs/heads/master Commit: 26eab98e3a0d403587cc915eaa8f7d4b7b894cef Parents: 0e8ff9c Author: tballison <talli...@mitre.org> Authored: Mon Apr 24 22:09:59 2017 -0400 Committer: tballison <talli...@mitre.org> Committed: Mon Apr 24 22:09:59 2017 -0400 ---------------------------------------------------------------------- .../apache/commons/compress/MemoryLimit.java | 15 +++++++++++ .../commons/compress/MemoryLimitException.java | 18 +++++++++----- .../lzma/LZMACompressorInputStream.java | 2 +- .../compressors/lzw/LZWInputStream.java | 26 ++++++++------------ .../compressors/xz/XZCompressorInputStream.java | 6 ++--- .../compressors/z/ZCompressorInputStream.java | 3 +-- 6 files changed, 42 insertions(+), 28 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/commons-compress/blob/26eab98e/src/main/java/org/apache/commons/compress/MemoryLimit.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/MemoryLimit.java b/src/main/java/org/apache/commons/compress/MemoryLimit.java index e03e8e7..b43eb86 100644 --- a/src/main/java/org/apache/commons/compress/MemoryLimit.java +++ b/src/main/java/org/apache/commons/compress/MemoryLimit.java @@ -62,4 +62,19 @@ public class MemoryLimit { public static int getMemoryLimitInKb() { return MEMORY_LIMIT_IN_KB; } + + public static void checkLimitInKb(long memoryNeeded) throws MemoryLimitException { + if (memoryNeeded < 0) { + throw new IllegalArgumentException("MemoryLimit must be > -1"); + } + + if (memoryNeeded >> 10 > Integer.MAX_VALUE) { + throw new MemoryLimitException(memoryNeeded, + (MEMORY_LIMIT_IN_KB < 0) ? Integer.MAX_VALUE : MEMORY_LIMIT_IN_KB); + } + + if (MEMORY_LIMIT_IN_KB > -1 && memoryNeeded > MEMORY_LIMIT_IN_KB) { + throw new MemoryLimitException(memoryNeeded, MEMORY_LIMIT_IN_KB); + } + } } http://git-wip-us.apache.org/repos/asf/commons-compress/blob/26eab98e/src/main/java/org/apache/commons/compress/MemoryLimitException.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/MemoryLimitException.java b/src/main/java/org/apache/commons/compress/MemoryLimitException.java index 8922ed2..a243a17 100644 --- a/src/main/java/org/apache/commons/compress/MemoryLimitException.java +++ b/src/main/java/org/apache/commons/compress/MemoryLimitException.java @@ -26,15 +26,21 @@ import java.io.IOException; * if a stream tries to allocate a byte array that is larger than * the allowable limit. * + * <p/> + * Set the global memory limit via {@link MemoryLimit#setMemoryLimitInKb(int)}. + * * @since 1.14 */ public class MemoryLimitException extends IOException { - public MemoryLimitException(String message) { - super(message); - } - - public MemoryLimitException(String message, Exception e) { - super(message, e); + /** + * + * @param memoryNeeded estimated memory needed + * @param memoryLimit memory limit applied + */ + public MemoryLimitException(long memoryNeeded, int memoryLimit) { + super("" + memoryNeeded + " KiB of memory would be needed; limit was " + + memoryLimit + " KiB. If the file is not corrupt, consider " + + "increasing MemoryLimit.MEMORY_LIMIT_IN_KB."); } } http://git-wip-us.apache.org/repos/asf/commons-compress/blob/26eab98e/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java index 6931541..35d7e12 100644 --- a/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java +++ b/src/main/java/org/apache/commons/compress/compressors/lzma/LZMACompressorInputStream.java @@ -56,7 +56,7 @@ public class LZMACompressorInputStream extends CompressorInputStream { in = new LZMAInputStream(inputStream, MemoryLimit.getMemoryLimitInKb()); } catch (org.tukaani.xz.MemoryLimitException e) { //convert to commons-compress exception - throw new MemoryLimitException("exceeded calculated memory limit", e); + throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit()); } } http://git-wip-us.apache.org/repos/asf/commons-compress/blob/26eab98e/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java b/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java index 350b4b0..ed2c476 100644 --- a/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java +++ b/src/main/java/org/apache/commons/compress/compressors/lzw/LZWInputStream.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.ByteOrder; +import org.apache.commons.compress.MemoryLimit; import org.apache.commons.compress.MemoryLimitException; import org.apache.commons.compress.compressors.CompressorInputStream; import org.apache.commons.compress.utils.BitInputStream; @@ -115,25 +116,18 @@ public abstract class LZWInputStream extends CompressorInputStream { /** * Initializes the arrays based on the maximum code size. * @param maxCodeSize maximum code size - * @param memoryLimitInKb maximum allowed table size in Kb - * @throws MemoryLimitException if maxTableSize is > memoryLimitInKb + * + * @throws MemoryLimitException + * if the calculated memory usage, based on the maxTableSize, + * is > {@link MemoryLimit#MEMORY_LIMIT_IN_KB} */ - protected void initializeTables(final int maxCodeSize, final int memoryLimitInKb) - throws MemoryLimitException { + protected void initializeTables(final int maxCodeSize) throws MemoryLimitException { final int maxTableSize = 1 << maxCodeSize; - if (memoryLimitInKb > -1 && maxTableSize > memoryLimitInKb*1024) { - throw new MemoryLimitException("Tried to allocate "+maxTableSize + - " but memoryLimitInKb only allows "+(memoryLimitInKb*1024)); - } - initializeTables(maxCodeSize); - } - /** - * Initializes the arrays based on the maximum code size. - * @param maxCodeSize maximum code size - */ - protected void initializeTables(final int maxCodeSize) { - final int maxTableSize = 1 << maxCodeSize; + //account for potential overflow + long memoryUsageInBytes = (long)maxTableSize * 6;//(4 (prefixes) + 1 (characters) +1 (outputStack)) + MemoryLimit.checkLimitInKb(memoryUsageInBytes >> 10); + prefixes = new int[maxTableSize]; characters = new byte[maxTableSize]; outputStack = new byte[maxTableSize]; http://git-wip-us.apache.org/repos/asf/commons-compress/blob/26eab98e/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java index 3f977ef..e80cbcf 100644 --- a/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java +++ b/src/main/java/org/apache/commons/compress/compressors/xz/XZCompressorInputStream.java @@ -110,7 +110,7 @@ public class XZCompressorInputStream extends CompressorInputStream { count(ret == -1 ? -1 : 1); return ret; } catch (org.tukaani.xz.MemoryLimitException e) { - throw new MemoryLimitException("Exceeded memory limit", e); + throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit()); } } @@ -122,7 +122,7 @@ public class XZCompressorInputStream extends CompressorInputStream { return ret; } catch (org.tukaani.xz.MemoryLimitException e) { //convert to commons-compress MemoryLimtException - throw new MemoryLimitException("Exceeded memory limit", e); + throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit()); } } @@ -132,7 +132,7 @@ public class XZCompressorInputStream extends CompressorInputStream { return in.skip(n); } catch (org.tukaani.xz.MemoryLimitException e) { //convert to commons-compress MemoryLimtException - throw new MemoryLimitException("Excedded memory limit", e); + throw new MemoryLimitException(e.getMemoryNeeded(), e.getMemoryLimit()); } } http://git-wip-us.apache.org/repos/asf/commons-compress/blob/26eab98e/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java b/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java index ca61cd3..28b69a2 100644 --- a/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java +++ b/src/main/java/org/apache/commons/compress/compressors/z/ZCompressorInputStream.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.io.InputStream; import java.nio.ByteOrder; -import org.apache.commons.compress.MemoryLimit; import org.apache.commons.compress.compressors.lzw.LZWInputStream; /** @@ -52,7 +51,7 @@ public class ZCompressorInputStream extends LZWInputStream { if (blockMode) { setClearCode(DEFAULT_CODE_SIZE); } - initializeTables(maxCodeSize, MemoryLimit.getMemoryLimitInKb()); + initializeTables(maxCodeSize); clearEntries(); }
