This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 88bdb053f3 [format] Fix corrupt output by overlapping custom line
delimiters (#9344)
88bdb053f3 is described below
commit 88bdb053f384d715ef73df5daf9f461456e8f66d
Author: Arnav Balyan <[email protected]>
AuthorDate: Sun Aug 23 20:51:07 2026 +0530
[format] Fix corrupt output by overlapping custom line delimiters (#9344)
---
.../paimon/format/text/CustomLineReader.java | 32 ++++++++++++++++------
.../paimon/format/text/TextFileReaderTest.java | 9 ++++++
2 files changed, 32 insertions(+), 9 deletions(-)
diff --git
a/paimon-format/src/main/java/org/apache/paimon/format/text/CustomLineReader.java
b/paimon-format/src/main/java/org/apache/paimon/format/text/CustomLineReader.java
index 1652798ed5..cfa980d6ed 100644
---
a/paimon-format/src/main/java/org/apache/paimon/format/text/CustomLineReader.java
+++
b/paimon-format/src/main/java/org/apache/paimon/format/text/CustomLineReader.java
@@ -32,10 +32,12 @@ public class CustomLineReader implements TextLineReader {
private final InputStream inputStream;
private final byte[] delimiter;
+ private final int[] prefixTable;
public CustomLineReader(InputStream inputStream, byte[] delimiter) {
this.inputStream = inputStream;
this.delimiter = delimiter;
+ this.prefixTable = buildPrefixTable(delimiter);
}
@Nullable
@@ -61,6 +63,12 @@ public class CustomLineReader implements TextLineReader {
}
byte current = (byte) b;
+ while (matchIndex > 0 && current != delimiter[matchIndex]) {
+ int fallback = prefixTable[matchIndex - 1];
+ out.write(delimiter, 0, matchIndex - fallback);
+ matchIndex = fallback;
+ }
+
if (current == delimiter[matchIndex]) {
// Current byte matches the next expected delimiter byte
matchIndex++;
@@ -68,15 +76,6 @@ public class CustomLineReader implements TextLineReader {
// Complete delimiter found, return the line without the
delimiter
return out.toString(StandardCharsets.UTF_8.name());
}
- } else if (matchIndex > 0) {
- // Mismatch: handle partial matches
- out.write(delimiter, 0, matchIndex);
- if (current == delimiter[0]) {
- matchIndex = 1;
- } else {
- out.write(current);
- matchIndex = 0;
- }
} else {
// just add the current byte to output
out.write(current);
@@ -84,6 +83,21 @@ public class CustomLineReader implements TextLineReader {
}
}
+ private static int[] buildPrefixTable(byte[] delimiter) {
+ int[] table = new int[delimiter.length];
+ int matched = 0;
+ for (int i = 1; i < delimiter.length; i++) {
+ while (matched > 0 && delimiter[i] != delimiter[matched]) {
+ matched = table[matched - 1];
+ }
+ if (delimiter[i] == delimiter[matched]) {
+ matched++;
+ }
+ table[i] = matched;
+ }
+ return table;
+ }
+
@Override
public void close() throws IOException {
inputStream.close();
diff --git
a/paimon-format/src/test/java/org/apache/paimon/format/text/TextFileReaderTest.java
b/paimon-format/src/test/java/org/apache/paimon/format/text/TextFileReaderTest.java
index d78ff1b3f0..37afed0b52 100644
---
a/paimon-format/src/test/java/org/apache/paimon/format/text/TextFileReaderTest.java
+++
b/paimon-format/src/test/java/org/apache/paimon/format/text/TextFileReaderTest.java
@@ -167,6 +167,15 @@ public class TextFileReaderTest {
assertThat(lines.get(1)).isEqualTo("bay");
}
+ @Test
+ public void testReadLineWithOverlappingDelimiterPrefix() throws
IOException {
+ writeFile("aaabtail");
+
+ List<String> lines = readAllLines("aab");
+
+ assertThat(lines).containsExactly("a", "tail");
+ }
+
@Test
public void testReadLineWithMultiByteUTF8Delimiter() throws IOException {
// Emoji delimiter
