This is an automated email from the ASF dual-hosted git repository.
caolu pushed a commit to branch kylin5
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push:
new 0327be455c KYLIN-6048 limit command line output size to avoid OOM
0327be455c is described below
commit 0327be455c33fb67f2f2c98a61cfd9b29ab8b736
Author: 夏旭晨 <[email protected]>
AuthorDate: Wed Nov 13 18:38:52 2024 +0800
KYLIN-6048 limit command line output size to avoid OOM
Co-authored-by: Xuchen Xia <[email protected]>
---
.../org/apache/kylin/common/KylinConfigBase.java | 7 ++
.../kylin/common/util/CliCommandExecutor.java | 7 +-
.../org/apache/kylin/common/util/HeadBuilder.java | 106 ++++++++++++++++
.../org/apache/kylin/common/util/SSHClient.java | 5 +-
.../kylin/common/util/StringBuilderHelper.java | 97 +++++++++++++++
.../org/apache/kylin/common/util/TailBuilder.java | 135 +++++++++++++++++++++
.../kylin/common/util/CliCommandExecutorTest.java | 57 +++++++++
.../apache/kylin/common/util/HeadBuilderTest.java | 111 +++++++++++++++++
.../apache/kylin/common/util/SSHClientTest.java | 66 ++++++++++
.../kylin/common/util/StringBuilderHelperTest.java | 125 +++++++++++++++++++
.../apache/kylin/common/util/TailBuilderTest.java | 104 ++++++++++++++++
11 files changed, 817 insertions(+), 3 deletions(-)
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index f9a7229318..947ffd8f20 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -134,6 +134,7 @@ public abstract class KylinConfigBase implements
Serializable {
public static final String SERVER_NAME_STRING = "spring.application.name";
protected static final Map<String, String> STATIC_SYSTEM_ENV = new
ConcurrentHashMap<>(System.getenv());
+ public static final int BYTES_PER_CHAR = 2;
/*
* DON'T DEFINE CONSTANTS FOR PROPERTY KEYS!
@@ -1813,6 +1814,12 @@ public abstract class KylinConfigBase implements
Serializable {
return Boolean.parseBoolean(getOptional("kylin.streaming.enabled",
FALSE));
}
+ public int getMaxCommandLineOutputLength() {
+ // default 10MB, if the command line output length over this value
+ // the output will be truncated as 5MB head and 5MB tail.
+ return Integer.parseInt(getOptional("kylin.command.max-output-bytes",
String.valueOf(10 * 1024 * 1024))) / BYTES_PER_CHAR;
+ }
+
public boolean isStreamingEnabled() {
return isStreamingConfigEnabled() &&
KylinInfoExtension.getFactory().checkKylinInfo();
}
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
b/src/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
index 487f2fd1e6..70d680636a 100644
---
a/src/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
+++
b/src/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
@@ -26,6 +26,7 @@ import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import org.apache.commons.io.FileUtils;
+import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.scheduler.EventBusFactory;
import org.apache.kylin.guava30.shaded.common.annotations.VisibleForTesting;
import org.slf4j.LoggerFactory;
@@ -191,8 +192,10 @@ public class CliCommandExecutor {
pid = ProcessUtils.getPid(proc);
logger.info("sub process {} on behalf of job {}, start to run...",
pid, jobId);
EventBusFactory.getInstance().postSync(new ProcessStart(pid,
jobId));
-
- StringBuilder result = new StringBuilder();
+ int maxCommandLineOutputLength =
KylinConfig.getInstanceFromEnv().getMaxCommandLineOutputLength();
+ int headSize = maxCommandLineOutputLength / 2;
+ StringBuilderHelper result = StringBuilderHelper.headTail(headSize,
+ maxCommandLineOutputLength - headSize);
try (BufferedReader reader = new BufferedReader(
new InputStreamReader(proc.getInputStream(),
StandardCharsets.UTF_8))) {
String line;
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/util/HeadBuilder.java
b/src/core-common/src/main/java/org/apache/kylin/common/util/HeadBuilder.java
new file mode 100644
index 0000000000..42133b12c3
--- /dev/null
+++
b/src/core-common/src/main/java/org/apache/kylin/common/util/HeadBuilder.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.util;
+
+import lombok.Getter;
+
+/**
+ * Limit the capacity of StringBuilder to avoid OOM
+ * If the capacity of StringBuilder would exceed the maxCapacity, the
StringBuilder will be recreated to maxCapacity
+ */
+public class HeadBuilder {
+
+ private StringBuilder builder;
+
+ @Getter
+ private final int maxCapacity;
+
+ public HeadBuilder(int initialCapacity, int maxCapacity) {
+ if (initialCapacity < 0) {
+ throw new IllegalArgumentException("initialCapacity must greater
than 0");
+ }
+ if (maxCapacity < 0) {
+ throw new IllegalArgumentException("maxCapacity must greater than
0");
+ }
+ this.builder = new StringBuilder(Math.min(initialCapacity,
maxCapacity));
+ this.maxCapacity = maxCapacity;
+ }
+
+ public HeadBuilder(int maxCapacity) {
+ this(16, maxCapacity);
+ }
+
+ private int ensureCapacity(int appendLen) {
+ if (builder.capacity() >= maxCapacity) {
+ return Math.min(builder.capacity() - builder.length(), appendLen);
+ }
+ if (builder.capacity() > builder.length() + appendLen) {
+ return appendLen;
+ }
+ // expand capacity
+ int newCapacity = builder.length() << 1 + 2;
+ if (newCapacity > maxCapacity) {
+ // create a new builder with maxCapacity to ensure maxCapacity
limit
+ String origin = builder.toString();
+ builder = new StringBuilder(maxCapacity);
+ builder.append(origin);
+ } else {
+ builder.ensureCapacity(Math.max(newCapacity, builder.length() +
appendLen));
+ }
+ return Math.min(builder.capacity() - builder.length(), appendLen);
+ }
+
+ public int capacity() {
+ return builder.capacity();
+ }
+
+ public int length() {
+ return builder.length();
+ }
+
+ public HeadBuilder append(String str) {
+ return append(str, 0, str.length());
+ }
+
+ public HeadBuilder append(CharSequence csq) {
+ return append(csq, 0, csq.length());
+ }
+
+ public HeadBuilder append(CharSequence csq, int start, int end) {
+ int appendSize = ensureCapacity(end - start);
+ if (appendSize == 0) {
+ return this;
+ }
+ builder.append(csq, start, start + appendSize);
+ return this;
+ }
+
+ public HeadBuilder append(char c) {
+ if (ensureCapacity(1) == 0) {
+ return this;
+ }
+ builder.append(c);
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ return builder.toString();
+ }
+
+}
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/util/SSHClient.java
b/src/core-common/src/main/java/org/apache/kylin/common/util/SSHClient.java
index a93363c0ff..86daf0e198 100644
--- a/src/core-common/src/main/java/org/apache/kylin/common/util/SSHClient.java
+++ b/src/core-common/src/main/java/org/apache/kylin/common/util/SSHClient.java
@@ -32,6 +32,7 @@ import java.nio.charset.Charset;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.KylinConfig;
import org.slf4j.LoggerFactory;
import com.jcraft.jsch.Channel;
@@ -314,7 +315,9 @@ public class SSHClient {
try {
logger.info("[" + username + "@" + hostname + "] Execute command:
" + command);
- StringBuilder text = new StringBuilder();
+ int maxCommandLineOutputLength =
KylinConfig.getInstanceFromEnv().getMaxCommandLineOutputLength();
+ StringBuilderHelper text =
StringBuilderHelper.headTail(maxCommandLineOutputLength / 2,
+ maxCommandLineOutputLength - maxCommandLineOutputLength /
2);
int exitCode = -1;
Session session = newJSchSession();
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/util/StringBuilderHelper.java
b/src/core-common/src/main/java/org/apache/kylin/common/util/StringBuilderHelper.java
new file mode 100644
index 0000000000..e165716b20
--- /dev/null
+++
b/src/core-common/src/main/java/org/apache/kylin/common/util/StringBuilderHelper.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.util;
+
+/**
+ * slice the string head and tail
+ * merge head and tail within the total size
+ */
+public class StringBuilderHelper {
+
+ private final int totalSize;
+
+ private final HeadBuilder headBuilder;
+
+ private final TailBuilder tailBuilder;
+
+ private int appendedSize = 0;
+
+ public static StringBuilderHelper head(int headSize) {
+ return new StringBuilderHelper(headSize, 0);
+ }
+
+ public static StringBuilderHelper tail(int tailSize) {
+ return new StringBuilderHelper(0, tailSize);
+ }
+
+ public static StringBuilderHelper headTail(int headSize, int tailSize) {
+ return new StringBuilderHelper(headSize, tailSize);
+ }
+
+ private StringBuilderHelper(int headSize, int tailSize) {
+ this.headBuilder = new HeadBuilder(headSize);
+ this.tailBuilder = new TailBuilder(tailSize);
+ this.totalSize = headSize + tailSize;
+ }
+
+ public StringBuilderHelper append(CharSequence csq) {
+ headBuilder.append(csq);
+ tailBuilder.append(csq);
+ appendedSize += csq.length();
+ return this;
+ }
+
+ public StringBuilderHelper append(char c) {
+ headBuilder.append(c);
+ tailBuilder.append(c);
+ appendedSize++;
+ return this;
+ }
+
+ private String merge() {
+ if (appendedSize < totalSize) {
+ // concat appended string
+ char[] res = new char[appendedSize];
+ headBuilder.toString().getChars(0, headBuilder.length(), res, 0);
+ int least = appendedSize - headBuilder.length();
+ tailBuilder.toString().getChars(tailBuilder.length() - least,
tailBuilder.length(),
+ res, headBuilder.length());
+ return new String(res);
+ } else {
+ // concat head and tail
+ char[] res = new char[totalSize];
+ int headLength = headBuilder.length();
+ headBuilder.toString().getChars(0, headLength, res, 0);
+ tailBuilder.toString().getChars(0, tailBuilder.length(),
+ res, headLength);
+ return new String(res);
+ }
+ }
+
+ public int length() {
+ return Math.min(appendedSize, totalSize);
+ }
+
+ /**
+ * if appended string size <= totalSize return total string
+ * if append string size > totalSize return head + tail
+ */
+ public String toString() {
+ return merge();
+ }
+}
diff --git
a/src/core-common/src/main/java/org/apache/kylin/common/util/TailBuilder.java
b/src/core-common/src/main/java/org/apache/kylin/common/util/TailBuilder.java
new file mode 100644
index 0000000000..8968604131
--- /dev/null
+++
b/src/core-common/src/main/java/org/apache/kylin/common/util/TailBuilder.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.util;
+
+import java.util.Arrays;
+import java.util.LinkedList;
+
+import lombok.Getter;
+
+public class TailBuilder {
+
+ static final int ENTRY_SIZE = 1024;
+
+ private LinkedList<char[]> data = new LinkedList<>();
+
+ @Getter
+ private final int maxCapacity;
+
+ // last segment left size
+ private int writableSize = 0;
+
+ private int length = 0;
+
+ public int length() {
+ return Math.min(length, maxCapacity);
+ }
+
+ public TailBuilder(int maxCapacity) {
+ if (maxCapacity < 0) {
+ throw new IllegalArgumentException("maxCapacity must >= 0");
+ }
+ this.maxCapacity = maxCapacity;
+ }
+
+ public TailBuilder append(CharSequence csq) {
+ if (maxCapacity == 0) {
+ return this;
+ }
+ if (csq == null || csq.length() == 0) {
+ return this;
+ }
+ // only need tail part
+ if (csq.length() > maxCapacity) {
+ csq = csq.subSequence(csq.length() - maxCapacity, csq.length());
+ }
+ int size2Append = csq.length();
+ char[] charArray = csq.toString().toCharArray();
+ while (size2Append > 0) {
+ int copySize = Math.min(writableSize, size2Append);
+ System.arraycopy(charArray, charArray.length - size2Append,
getWriteEntry(),
+ ENTRY_SIZE - writableSize, copySize);
+ writableSize -= copySize;
+ size2Append -= copySize;
+ length += copySize;
+ }
+ return this;
+ }
+
+ private char[] getWriteEntry() {
+ ensureWriteEntry();
+ return data.getLast();
+ }
+
+ public TailBuilder append(char c) {
+ if (maxCapacity == 0) {
+ return this;
+ }
+ getWriteEntry()[ENTRY_SIZE - writableSize] = c;
+ writableSize--;
+ length++;
+ return this;
+ }
+
+ /**
+ * ensure the last entry having space to write
+ * allocate a new entry if the last entry is full
+ * we would recycle the first entry to allocate
+ * if the (sizeof(data) - 1) * ENTRY_SIZE > maxCapacity so the first entry
is useless
+ * because data[0] is not include any char we want(remain last maxCapacity
chars)
+ */
+ private void ensureWriteEntry() {
+ if (writableSize > 0) {
+ return;
+ }
+ char[] allocated = null;
+ // recycle first entry
+ if (ENTRY_SIZE * (data.size() - 1) >= maxCapacity) {
+ allocated = data.removeFirst();
+ Arrays.fill(allocated, (char) 0);
+ length -= ENTRY_SIZE;
+ }
+ if (allocated == null) {
+ allocated = new char[ENTRY_SIZE];
+ }
+ data.add(allocated);
+ writableSize = ENTRY_SIZE;
+ }
+
+ /**
+ * @return a String contains tail maxCapacity chars, if length <
maxCapacity, return all
+ */
+ public String toString() {
+ int totalSize = Math.min(length, maxCapacity);
+ char[] result = new char[totalSize];
+ int skipSize = Math.max(0, length - maxCapacity);
+ int pos = 0;
+ for (int i = 0; i < data.size() - 1; i++) {
+ if (skipSize >= ENTRY_SIZE) {
+ skipSize -= ENTRY_SIZE;
+ continue;
+ }
+ char[] src = data.get(i);
+ System.arraycopy(src, skipSize, result, pos, ENTRY_SIZE -
skipSize);
+ pos += ENTRY_SIZE - skipSize;
+ skipSize = 0;
+ }
+ System.arraycopy(getWriteEntry(), skipSize, result, pos, ENTRY_SIZE -
writableSize - skipSize);
+ return new String(result);
+ }
+}
diff --git
a/src/core-common/src/test/java/org/apache/kylin/common/util/CliCommandExecutorTest.java
b/src/core-common/src/test/java/org/apache/kylin/common/util/CliCommandExecutorTest.java
index 8b22159738..7d83931961 100644
---
a/src/core-common/src/test/java/org/apache/kylin/common/util/CliCommandExecutorTest.java
+++
b/src/core-common/src/test/java/org/apache/kylin/common/util/CliCommandExecutorTest.java
@@ -17,10 +17,15 @@
*/
package org.apache.kylin.common.util;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
import java.io.File;
import java.lang.reflect.Method;
import org.apache.commons.io.FileUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.junit.annotation.MetadataInfo;
import org.junit.Assert;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
@@ -29,6 +34,7 @@ import org.mockito.Mockito;
import lombok.val;
+@MetadataInfo(onlyProps = true)
public class CliCommandExecutorTest {
@Test
@@ -77,6 +83,57 @@ public class CliCommandExecutorTest {
Assert.assertEquals(fileList[0].getName(), tempFile.getName());
}
+
+ @Test
+ void testLargeOutputSlice() throws Exception {
+ // These test assert the kylin.command.max-output-length is 5M chars
as 10 MB size
+ int maxCommandLineOutputLength =
KylinConfig.getInstanceFromEnv().getMaxCommandLineOutputLength();
+ assertEquals(5 * 1024 * 1024, maxCommandLineOutputLength);
+ CliCommandExecutor cliCommandExecutor = new CliCommandExecutor();
+ // generate a large output and expect the output is truncated
+ CliCommandExecutor.CliCmdExecResult res = cliCommandExecutor.execute(
+ "echo 'testhead' && line=$(printf 'a%.0s' {1..1251}) && yes
\"$line\" 2>/dev/null | head -n 5505 && echo 'testtail'",
+ null);
+ assertEquals(maxCommandLineOutputLength, res.getCmd().length());
+ assertTrue(res.getCmd().startsWith("testhead"));
+ assertTrue(res.getCmd().endsWith("testtail\n"));
+ // multiline long string
+ res = cliCommandExecutor
+ .execute("echo 'testhead' && line=$(printf 'a%.0s' {1..111})
&& yes \"$line\" 2>/dev/null | head -n " + 1024 * 52
+ + "&& echo 'testtail'", null);
+ assertEquals(maxCommandLineOutputLength, res.getCmd().length());
+ assertTrue(res.getCmd().startsWith("testhead"));
+ assertTrue(res.getCmd().endsWith("testtail\n"));
+ // multiline short string
+ res = cliCommandExecutor.execute("echo 'testhead' && line=$(printf
'a%.0s' {1..9}) && yes \"$line\" 2>/dev/null | head -n "
+ + 1024 * 513 + "&& echo 'testtail'", null);
+ assertEquals(maxCommandLineOutputLength, res.getCmd().length());
+ assertTrue(res.getCmd().startsWith("testhead"));
+ assertTrue(res.getCmd().endsWith("testtail\n"));
+
+ // generate large output less than 10 mb and expect the same output
+ res = cliCommandExecutor.execute(
+ "echo 'testhead' && line=$(printf 'a%.0s' {1..1251}) && yes
\"$line\" 2>/dev/null | head -n 1000 && echo 'testtail'",
+ null);
+ final int HEAD_TAIL_LEN = 18;
+ assertEquals(1252 * 1000 + HEAD_TAIL_LEN, res.getCmd().length());
+ assertTrue(res.getCmd().startsWith("testhead"));
+ assertTrue(res.getCmd().endsWith("testtail\n"));
+
+ // multiline long string
+ res = cliCommandExecutor
+ .execute("echo 'testhead' && line=$(printf 'a%.0s' {1..111})
&& yes \"$line\" 2>/dev/null | head -n " + 1024 * 10
+ + "&& echo 'testtail'", null);
+ assertEquals(112 * 10 * 1024 + HEAD_TAIL_LEN, res.getCmd().length());
+ assertTrue(res.getCmd().startsWith("testhead"));
+ assertTrue(res.getCmd().endsWith("testtail\n"));
+ // multiline short string
+ res = cliCommandExecutor.execute("echo 'testhead' && line=$(printf
'a%.0s' {1..9}) && yes \"$line\" 2>/dev/null | head -n "
+ + 1024 * 100 + "&& echo 'testtail'", null);
+ assertEquals(10 * 100 * 1024 + HEAD_TAIL_LEN, res.getCmd().length());
+ assertTrue(res.getCmd().startsWith("testhead"));
+ assertTrue(res.getCmd().endsWith("testtail\n"));
+ }
}
class MockSSHClient extends SSHClient {
diff --git
a/src/core-common/src/test/java/org/apache/kylin/common/util/HeadBuilderTest.java
b/src/core-common/src/test/java/org/apache/kylin/common/util/HeadBuilderTest.java
new file mode 100644
index 0000000000..0123d50b4b
--- /dev/null
+++
b/src/core-common/src/test/java/org/apache/kylin/common/util/HeadBuilderTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.util;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class HeadBuilderTest {
+
+ private static boolean isFull(HeadBuilder stringWriter) {
+ return stringWriter.length() == stringWriter.capacity();
+ }
+
+ @Test
+ void testWriteLimited() throws IOException {
+ HeadBuilder stringWriter = new HeadBuilder(10);
+ stringWriter.append('1');
+ assertEquals("1", stringWriter.toString());
+ assertEquals(10, stringWriter.capacity());
+ assertFalse(isFull(stringWriter));
+ stringWriter.append('2');
+ assertEquals("12", stringWriter.toString());
+ assertEquals(10, stringWriter.capacity());
+ assertEquals(2, stringWriter.length());
+ assertFalse(isFull(stringWriter));
+ stringWriter.append("345");
+ assertEquals("12345", stringWriter.toString());
+ assertFalse(isFull(stringWriter));
+ stringWriter.append("678910");
+ assertWhenFull(stringWriter);
+ stringWriter.append('a');
+ assertWhenFull(stringWriter);
+ stringWriter.append("aa", 1, 2);
+ assertWhenFull(stringWriter);
+ testFull(20, 12345);
+ testFull(16, 12345);
+ testFull(1204, 10024);
+ testFull(1, 52431);
+ }
+
+ @Test
+ void testConstructor() {
+ // legal capacity
+ new HeadBuilder(0);
+ Assertions.assertThrows(IllegalArgumentException.class, () -> new
HeadBuilder(-1));
+ Assertions.assertThrows(IllegalArgumentException.class, () -> new
HeadBuilder(-1, 0));
+ }
+
+ private static void testFull(int initCapacity, int maxCapacity) {
+ HeadBuilder stringWriter;
+ stringWriter = new HeadBuilder(initCapacity, maxCapacity);
+ final String APPEND_STR = new String(new char[123]);
+ final char APPEND_CHAR = 'a';
+ final int SUB_APPEND_LENGTH = 89;
+ while (!isFull(stringWriter)) {
+ append(stringWriter, APPEND_STR, APPEND_CHAR, SUB_APPEND_LENGTH);
+ }
+ // append when full to test capacity not increase
+ append(stringWriter, APPEND_STR, APPEND_CHAR, SUB_APPEND_LENGTH);
+ assertEquals(maxCapacity, stringWriter.capacity());
+ assertEquals(maxCapacity, stringWriter.length());
+ }
+
+ private static void append(HeadBuilder stringWriter, String appendStr,
char appendChar, int subAppendLength) {
+ int length;
+ length = stringWriter.length();
+ stringWriter.append(appendStr);
+ if (!isFull(stringWriter)) {
+ assertEquals(123, stringWriter.length() - length);
+ }
+ length = stringWriter.length();
+ stringWriter.append(appendChar);
+ if (!isFull(stringWriter)) {
+ assertEquals(1, stringWriter.length() - length);
+ }
+ length = stringWriter.length();
+ stringWriter.append(appendStr, 0, subAppendLength);
+ if (!isFull(stringWriter)) {
+ assertEquals(subAppendLength, stringWriter.length() - length);
+ }
+ }
+
+ private static void assertWhenFull(HeadBuilder stringWriter) {
+ assertEquals("1234567891", stringWriter.toString());
+ assertEquals(10, stringWriter.capacity());
+ assertEquals(10, stringWriter.length());
+ assertTrue(isFull(stringWriter));
+ }
+
+}
diff --git
a/src/core-common/src/test/java/org/apache/kylin/common/util/SSHClientTest.java
b/src/core-common/src/test/java/org/apache/kylin/common/util/SSHClientTest.java
index 0db4bcf395..e5af7f54c7 100644
---
a/src/core-common/src/test/java/org/apache/kylin/common/util/SSHClientTest.java
+++
b/src/core-common/src/test/java/org/apache/kylin/common/util/SSHClientTest.java
@@ -18,6 +18,9 @@
package org.apache.kylin.common.util;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Method;
@@ -72,6 +75,69 @@ public class SSHClientTest {
Assert.assertEquals("hello\n", output.getText());
}
+ @Test
+ public void testCmdWithLargeOutput() throws Exception {
+ if (!isRemote)
+ return;
+ // These tests assert that the output from SSH commands are truncated
correctly
+ // based on the 'kylin.command.max-output-length' configuration
which is 10 MB
+ final int maxCommandLineOutputLength =
KylinConfig.getInstanceFromEnv().getMaxCommandLineOutputLength();
+ assertEquals(5 * 1024 * 1024, maxCommandLineOutputLength);
+ final int HEAD_TAIL_LEN = 18; // considering the length of
'testhead\ntesttail\n'
+ SSHClient ssh = new SSHClient(this.hostname, this.port, this.username,
this.password);
+ SSHClientOutput sshClientOutput;
+
+ // Test large output and check it is truncated
+ sshClientOutput = ssh.execCommand(
+ "echo 'testhead' && line=$(printf 'a%.0s' {1..1251}) && yes
\"$line\" 2>/dev/null | head -n 5505 && echo 'testtail'");
+ String result = sshClientOutput.getText();
+ assertEquals(maxCommandLineOutputLength, result.length());
+ assertTrue(result.startsWith("testhead"));
+ assertTrue(result.endsWith("testtail\n"));
+
+ // Test multiline long string
+ sshClientOutput = ssh
+ .execCommand("echo 'testhead' && line=$(printf 'a%.0s'
{1..111}) && yes \"$line\" 2>/dev/null | head -n "
+ + 1024 * 52 + " && echo 'testtail'");
+ result = sshClientOutput.getText();
+ assertEquals(maxCommandLineOutputLength, result.length());
+ assertTrue(result.startsWith("testhead"));
+ assertTrue(result.endsWith("testtail\n"));
+
+ // Test multiline short string (larger than 10 MB)
+ sshClientOutput = ssh.execCommand("echo 'testhead' && line=$(printf
'a%.0s' {1..9}) && yes \"$line\" 2>/dev/null | head -n "
+ + 1024 * 513 + " && echo 'testtail'");
+ result = sshClientOutput.getText();
+ assertEquals(maxCommandLineOutputLength, result.length());
+ assertTrue(result.startsWith("testhead"));
+ assertTrue(result.endsWith("testtail\n"));
+
+ // Generate large output less than 10 MB and expect the same output
+ sshClientOutput = ssh.execCommand(
+ "echo 'testhead' && line=$(printf 'a%.0s' {1..1251}) && yes
\"$line\" 2>/dev/null | head -n 1000 && echo 'testtail'");
+ result = sshClientOutput.getText();
+ assertEquals(1252 * 1000 + HEAD_TAIL_LEN, result.length());
+ assertTrue(result.startsWith("testhead"));
+ assertTrue(result.endsWith("testtail\n"));
+
+ // Multiline long string, less than 10 MB
+ sshClientOutput = ssh
+ .execCommand("echo 'testhead' && line=$(printf 'a%.0s'
{1..111}) && yes \"$line\" 2>/dev/null | head -n "
+ + 1024 * 10 + " && echo 'testtail'");
+ result = sshClientOutput.getText();
+ assertEquals(112 * 10 * 1024 + HEAD_TAIL_LEN, result.length());
+ assertTrue(result.startsWith("testhead"));
+ assertTrue(result.endsWith("testtail\n"));
+
+ // Multiline short string, less than 10 MB
+ sshClientOutput = ssh.execCommand("echo 'testhead' && line=$(printf
'a%.0s' {1..9}) && yes \"$line\" 2>/dev/null | head -n "
+ + 1024 * 100 + " && echo 'testtail'");
+ result = sshClientOutput.getText();
+ assertEquals(10 * 100 * 1024 + HEAD_TAIL_LEN, result.length());
+ assertTrue(result.startsWith("testhead"));
+ assertTrue(result.endsWith("testtail\n"));
+ }
+
@Test
public void testScpFileToRemote() throws Exception {
if (!isRemote)
diff --git
a/src/core-common/src/test/java/org/apache/kylin/common/util/StringBuilderHelperTest.java
b/src/core-common/src/test/java/org/apache/kylin/common/util/StringBuilderHelperTest.java
new file mode 100644
index 0000000000..65f8f05a71
--- /dev/null
+++
b/src/core-common/src/test/java/org/apache/kylin/common/util/StringBuilderHelperTest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.util;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.Random;
+
+import org.junit.jupiter.api.Test;
+
+public class StringBuilderHelperTest {
+
+ @Test
+ public void testHead() {
+ StringBuilderHelper helper = StringBuilderHelper.head(4);
+ helper.append("abc");
+ assertEquals("abc", helper.toString());
+ helper.append('d');
+ assertEquals("abcd", helper.toString());
+
+ helper = StringBuilderHelper.head(4);
+ helper.append("abcd");
+ assertEquals("abcd", helper.toString());
+ // append after full
+ helper.append('e');
+ assertEquals("abcd", helper.toString());
+ helper.append("fg");
+ assertEquals("abcd", helper.toString());
+ }
+
+ @Test
+ public void testTail() {
+ StringBuilderHelper helper = StringBuilderHelper.tail(4);
+ helper.append("abc");
+ assertEquals("abc", helper.toString());
+ helper.append('d');
+ assertEquals("abcd", helper.toString());
+ // append after full
+ helper.append('e');
+ assertEquals("bcde", helper.toString());
+ helper.append("fg");
+ assertEquals("defg", helper.toString());
+ }
+
+ @Test
+ public void testHeadAndTail() {
+ StringBuilderHelper helper1 = StringBuilderHelper.headTail(3, 2);
+ helper1.append("abc");
+ assertEquals("abc", helper1.toString());
+ helper1.append('d');
+ assertEquals("abcd", helper1.toString());
+ helper1.append('e');
+ assertEquals("abcde", helper1.toString());
+ helper1.append("fg");
+ assertEquals("abcfg", helper1.toString());
+ helper1.append('h');
+ assertEquals("abcgh", helper1.toString());
+ }
+
+ @Test
+ public void testHeadAndTailWithLargeSize() {
+ // expected truncate
+ testWithLength(3000, 3000, 10000, 0, 1, "TEST HEAD", "TEST TAIL");
+ testWithLength(3000, 3000, 10000, 10, 100, "TEST HEAD", "TEST TAIL");
+ testWithLength(3000, 3000, 10000, 10, TailBuilder.ENTRY_SIZE + 10,
"TEST HEAD", "TEST TAIL");
+ testWithLength(3000, 3000, 10000, TailBuilder.ENTRY_SIZE,
TailBuilder.ENTRY_SIZE + 10, "TEST HEAD",
+ "TEST TAIL");
+
+ // expected not truncate
+ testWithLength(3000, 3000, 1500, 0, 1, "TEST HEAD", "TEST TAIL");
+ testWithLength(3000, 3000, 1500, 10, 100, "TEST HEAD", "TEST TAIL");
+ testWithLength(3000, 3000, 1500, 10, TailBuilder.ENTRY_SIZE + 10,
"TEST HEAD", "TEST TAIL");
+ testWithLength(3000, 3000, 1500, TailBuilder.ENTRY_SIZE,
TailBuilder.ENTRY_SIZE + 10, "TEST HEAD", "TEST TAIL");
+
+ }
+
+ private void testWithLength(int headSize, int tailSize, int appendTotal,
int strMinLengthExclusive,
+ int strMaxLength, String head, String tail) {
+ StringBuilder stringBuilder = new StringBuilder();
+ StringBuilderHelper helper1 = StringBuilderHelper.headTail(headSize,
tailSize);
+ helper1.append(head);
+ stringBuilder.append(head);
+ assertEquals(head, helper1.toString());
+ int totalSize = headSize + tailSize;
+ Random random = new Random();
+ int appended = 0;
+ while (appended < appendTotal) {
+ int length = Math.min(random.nextInt(strMaxLength -
strMinLengthExclusive + 1) + strMinLengthExclusive,
+ appendTotal - appended);
+ String str = random.ints('a', 'z' + 1).limit(length)
+ .collect(StringBuilder::new,
StringBuilder::appendCodePoint, StringBuilder::append).toString();
+ if (str.length() == 1) {
+ // test append char
+ helper1.append(str.charAt(0));
+ } else {
+ helper1.append(str);
+ }
+ stringBuilder.append(str);
+ appended += str.length();
+ }
+ helper1.append(tail);
+ stringBuilder.append(tail);
+ String string = stringBuilder.toString();
+ if (string.length() > totalSize) {
+ string = string.substring(0, headSize) +
string.substring(string.length() - tailSize);
+ }
+ assertEquals(string, helper1.toString());
+ }
+
+}
diff --git
a/src/core-common/src/test/java/org/apache/kylin/common/util/TailBuilderTest.java
b/src/core-common/src/test/java/org/apache/kylin/common/util/TailBuilderTest.java
new file mode 100644
index 0000000000..0a17c841d4
--- /dev/null
+++
b/src/core-common/src/test/java/org/apache/kylin/common/util/TailBuilderTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kylin.common.util;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+public class TailBuilderTest {
+
+ @Test
+ public void testConstructor() {
+ assertThrows(IllegalArgumentException.class, () -> new
TailBuilder(-1));
+ }
+
+ @Test
+ public void testAppendCharSequence() {
+ TailBuilder tailBuilder = new TailBuilder(10);
+
+ // empty string
+ tailBuilder.append("");
+ assertEquals(0, tailBuilder.length());
+
+ // str size < maxCapacity
+ tailBuilder.append("abc");
+ assertEquals(3, tailBuilder.length());
+
+ // total size > maxCapacity
+ tailBuilder.append("12345678");
+ assertEquals("bc12345678", tailBuilder.toString());
+ assertEquals(10, tailBuilder.length());
+
+ // str size = maxCapacity
+ tailBuilder = new TailBuilder(10);
+ tailBuilder.append("1234567890");
+ assertEquals("1234567890", tailBuilder.toString());
+ assertEquals(10, tailBuilder.length());
+
+ // str size > maxCapacity
+ tailBuilder = new TailBuilder(6);
+ tailBuilder.append("12345678901");
+ assertEquals("678901", tailBuilder.toString());
+ assertEquals(6, tailBuilder.length());
+ }
+
+ @Test
+ public void testAppendChar() {
+ TailBuilder tailBuilder = new TailBuilder(10);
+ tailBuilder.append('a');
+ assertEquals(1, tailBuilder.length());
+ }
+
+ @Test
+ public void testEnsureWriteEntry() {
+ TailBuilder tailBuilder = new TailBuilder(10);
+
+ for (int i = 0; i < 10; i++) {
+ tailBuilder.append('a');
+ }
+ tailBuilder.append('b');
+ assertEquals(10, tailBuilder.length());
+ assertEquals("aaaaaaaaab", tailBuilder.toString());
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ public void testLargeStr(boolean appendHead) {
+ StringBuilder stringBuilder = new StringBuilder();
+ TailBuilder tailBuilder = new TailBuilder(TailBuilder.ENTRY_SIZE / 2);
+ if (appendHead) {
+ stringBuilder.append("test head");
+ tailBuilder.append("test head");
+ }
+ char[] chars = new char[TailBuilder.ENTRY_SIZE + 100];
+ for (int i = 0; i < TailBuilder.ENTRY_SIZE + 100; i++) {
+ chars[i] = (char) ('a' + (i % 26));
+ }
+ String testStr = new String(chars);
+ stringBuilder.append(testStr);
+ tailBuilder.append(testStr);
+ assertEquals(TailBuilder.ENTRY_SIZE / 2, tailBuilder.length());
+ assertEquals(stringBuilder.substring(stringBuilder.length() -
TailBuilder.ENTRY_SIZE / 2),
+ tailBuilder.toString());
+ }
+
+}
