This is an automated email from the ASF dual-hosted git repository.
ggregory pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-compress.git
The following commit(s) were added to refs/heads/master by this push:
new 373ee83d [COMPRESS-649] Improve performance in
org.apache.commons.compress.compressors.lz4.BlockLZ4CompressorOutputStream.Pair.literalLength()
new 10bc9ff2 Merge branch 'master' of
https://gitbox.apache.org/repos/asf/commons-compress.git
373ee83d is described below
commit 373ee83d801cf635319cd74e35d8abbd0062e1ea
Author: Gary Gregory <garydgreg...@gmail.com>
AuthorDate: Fri Oct 27 07:04:40 2023 -0400
[COMPRESS-649] Improve performance in
org.apache.commons.compress.compressors.lz4.BlockLZ4CompressorOutputStream.Pair.literalLength()
---
src/changes/changes.xml | 2 +
.../lz4/BlockLZ4CompressorOutputStream.java | 14 ++++-
.../lz4/CompressionDegradationTest.java | 73 ++++++++++++++++++++++
.../compress/COMPRESS-649/some-900kb-text.txt | 1 +
4 files changed, 89 insertions(+), 1 deletion(-)
diff --git a/src/changes/changes.xml b/src/changes/changes.xml
index b560d1ca..4f81835e 100644
--- a/src/changes/changes.xml
+++ b/src/changes/changes.xml
@@ -68,6 +68,8 @@ The <action> type attribute can be add,update,fix,remove.
<action type="fix" dev="ggregory" due-to="Gary Gregory">Precompile
regular expression in ArArchiveInputStream.isGNULongName(String).</action>
<action type="fix" dev="ggregory" due-to="Gary Gregory">Precompile
regular expression in
TarArchiveEntry.parseInstantFromDecimalSeconds(String).</action>
<action type="fix" dev="ggregory" due-to="Gary Gregory">Precompile
regular expression in ChangeSet.addDeletion(Change).</action>
+ <action type="fix" dev="ggregory" due-to="Gary Gregory"
issue="COMPRESS-649">Improve performance in
BlockLZ4CompressorOutputStream.</action>
+
<!-- UPDATE -->
<action type="update" dev="ggregory" due-to="Dependabot">Bump
org.slf4j:slf4j-api from 2.0.8 to 2.0.9 #413.</action>
<action type="update" dev="ggregory" due-to="Gary Gregory">Bump
commons-io:commons-io from 2.13.0 to 2.15.0.</action>
diff --git
a/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStream.java
b/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStream.java
index 4190cb22..eea19f10 100644
---
a/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStream.java
+++
b/src/main/java/org/apache/commons/compress/compressors/lz4/BlockLZ4CompressorOutputStream.java
@@ -53,6 +53,8 @@ public class BlockLZ4CompressorOutputStream extends
CompressorOutputStream {
out.write(length);
}
private final Deque<byte[]> literals = new LinkedList<>();
+
+ private int literalLength;
private int brOffset, brLength;
@@ -62,6 +64,7 @@ public class BlockLZ4CompressorOutputStream extends
CompressorOutputStream {
final byte[] copy = Arrays.copyOfRange(block.getData(),
block.getOffset(),
block.getOffset() + block.getLength());
literals.add(copy);
+ literalLength += copy.length;
return copy;
}
@@ -87,11 +90,20 @@ public class BlockLZ4CompressorOutputStream extends
CompressorOutputStream {
}
private int literalLength() {
- return literals.stream().mapToInt(b -> b.length).sum();
+ // This method is performance sensitive
+ if (literalLength != 0) {
+ return literalLength;
+ }
+ int length = 0;
+ for (final byte[] b : literals) {
+ length += b.length;
+ }
+ return literalLength = length;
}
private void prependLiteral(final byte[] data) {
literals.addFirst(data);
+ literalLength += data.length;
}
private void prependTo(final Pair other) {
diff --git
a/src/test/java/org/apache/commons/compress/compressors/lz4/CompressionDegradationTest.java
b/src/test/java/org/apache/commons/compress/compressors/lz4/CompressionDegradationTest.java
new file mode 100644
index 00000000..653825dc
--- /dev/null
+++
b/src/test/java/org/apache/commons/compress/compressors/lz4/CompressionDegradationTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.commons.compress.compressors.lz4;
+
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+
+/**
+ * Tests COMPRESS-649.
+ */
+public class CompressionDegradationTest {
+
+ public static void main(final String[] args) throws Exception {
+ try (RandomAccessFile aFile = new
RandomAccessFile("src/test/resources/org/apache/commons/compress/COMPRESS-649/some-900kb-text.txt",
"r");
+ FileChannel inChannel = aFile.getChannel()) {
+ final long fileSize = inChannel.size();
+
+ final ByteBuffer buffer = ByteBuffer.allocate((int) fileSize);
+ inChannel.read(buffer);
+ buffer.flip();
+
+ final String rawPlan = new String(buffer.array(),
StandardCharsets.UTF_8);
+ final long start = System.currentTimeMillis();
+ for (int i = 0; i < 80; i++) {
+ assertNotNull(compress(rawPlan));
+ }
+ final long end = System.currentTimeMillis();
+ final float sec = (end - start) / 1000F;
+ System.out.println(sec + " seconds");
+ }
+ }
+
+ private static String compress(final String value) throws IOException {
+ try (ByteArrayOutputStream byteStream = new
ByteArrayOutputStream(value.length());
+ FramedLZ4CompressorOutputStream compress = new
FramedLZ4CompressorOutputStream(byteStream)) {
+ String compressedValue = null;
+ try {
+ compress.write(value.getBytes(StandardCharsets.UTF_8));
+ compress.finish();
+ compressedValue =
Base64.getEncoder().encodeToString(byteStream.toByteArray());
+ } finally {
+ compress.close();
+ byteStream.close();
+ }
+
+ return compressedValue;
+ }
+ }
+}
\ No newline at end of file
diff --git
a/src/test/resources/org/apache/commons/compress/COMPRESS-649/some-900kb-text.txt
b/src/test/resources/org/apache/commons/compress/COMPRESS-649/some-900kb-text.txt
new file mode 100644
index 00000000..dc323a58
--- /dev/null
+++
b/src/test/resources/org/apache/commons/compress/COMPRESS-649/some-900kb-text.txt
@@ -0,0 +1 @@
+5EUUBnKi7eAidloprJ6kOXt0C4Biy0GwHUnlThbHPe6hyNkGQTE9ZfFLOzqniL0BDKgaCVHa8GqWVDw6nZBoJC0i6RwjcbeHkk512gLbXji6sQ1D0oQvSBmGBONnRiaLcRShW3QOr1F4rlVCOxva92ENoiW2lclQN92YJLm5t1MRSAuxpon8qn52kvxImHyyBavAcujZC22GgZ8sIuPZwN6XC1JOlgDQVzNQYI0nFaBVVYTe4tLfbtTOpj10flv3DpewXRLzJIVjhFzPRy0SjoZgQBZFnZIFn52V6CaxqVLfMMosq93U0A4xqnmEvjnWIjhHVsltWItaHgd6DU5gVGLhClRFqqkbXUQou9XHmcGFmj5GP0wbXnvZIvcHVmO3lNaWrCyf1yf3aIGGkWRezOz9AYWK5hRf8r5LAdpu8mMlPc0PSO74uGK6sPRze6l5xzDpUtraiDcWfLfz1F81Rc4qs5rtfNciJXUN3Ltp8UsX
[...]
\ No newline at end of file
