This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new e3cd4df38b [format] Keep parsing remaining fields after a malformed
one in PERMISSIVE CSV mode (#9556)
e3cd4df38b is described below
commit e3cd4df38beebe13adf3f558fe054c12c400af42
Author: YangJie <[email protected]>
AuthorDate: Thu Sep 3 04:20:26 2026 -0400
[format] Keep parsing remaining fields after a malformed one in PERMISSIVE
CSV mode (#9556)
---
.../org/apache/paimon/format/csv/CsvParser.java | 3 +-
.../paimon/format/csv/CsvFileFormatTest.java | 38 ++++++++++++++++++++++
2 files changed, 40 insertions(+), 1 deletion(-)
diff --git
a/paimon-format/src/main/java/org/apache/paimon/format/csv/CsvParser.java
b/paimon-format/src/main/java/org/apache/paimon/format/csv/CsvParser.java
index 58f7b790c2..c72b72be8e 100644
--- a/paimon-format/src/main/java/org/apache/paimon/format/csv/CsvParser.java
+++ b/paimon-format/src/main/java/org/apache/paimon/format/csv/CsvParser.java
@@ -180,7 +180,8 @@ public class CsvParser {
row.setField(i, parseResult.getValue());
} else if (mode == PERMISSIVE
&& (parseResult == null || !parseResult.getLeft() ||
exception != null)) {
- break;
+ // Only this field is malformed, so keep parsing the rest of
the row.
+ row.setField(i, null);
} else if (mode == DROPMALFORMED
&& (parseResult == null || !parseResult.getLeft() ||
exception != null)) {
return null;
diff --git
a/paimon-format/src/test/java/org/apache/paimon/format/csv/CsvFileFormatTest.java
b/paimon-format/src/test/java/org/apache/paimon/format/csv/CsvFileFormatTest.java
index 47fc367726..a7033bfe2f 100644
---
a/paimon-format/src/test/java/org/apache/paimon/format/csv/CsvFileFormatTest.java
+++
b/paimon-format/src/test/java/org/apache/paimon/format/csv/CsvFileFormatTest.java
@@ -639,6 +639,44 @@ public class CsvFileFormatTest extends FormatReadWriteTest
{
assertThat(permissiveResult.get(3).getDouble(2)).isEqualTo(400.81);
}
+ @Test
+ public void testCsvPermissiveKeepsFieldsAfterMalformed() throws
IOException {
+ RowType rowType = DataTypes.ROW(DataTypes.INT(), DataTypes.STRING(),
DataTypes.DOUBLE());
+ Options options = new Options();
+ options.set(CsvOptions.MODE, CsvOptions.Mode.PERMISSIVE);
+ FileFormat format =
+ new CsvFileFormatFactory().create(new FormatContext(options,
1024, 1024));
+ Path testFile = new Path(parent, "permissive_first_" +
UUID.randomUUID() + ".csv");
+
+ // Malformed field in the first position: PERMISSIVE must null only the
+ // offending field and keep the valid fields after it.
+ fileIO.writeFile(testFile, "x,Alice,1.5\n3,Carol,3.5", false);
+ List<InternalRow> result = read(format, rowType, rowType, testFile);
+ assertThat(result).hasSize(2);
+ assertThat(result.get(0).isNullAt(0)).isTrue();
+ assertThat(result.get(0).getString(1)).isEqualTo(fromString("Alice"));
+ assertThat(result.get(0).getDouble(2)).isEqualTo(1.5);
+ assertThat(result.get(1).getInt(0)).isEqualTo(3);
+ assertThat(result.get(1).getString(1)).isEqualTo(fromString("Carol"));
+ assertThat(result.get(1).getDouble(2)).isEqualTo(3.5);
+
+ // PERMISSIVE is the default mode, so this format is built without
setting csv.mode.
+ // Covers a malformed field in the middle and two malformed fields in
one row.
+ RowType midRowType = DataTypes.ROW(DataTypes.INT(),
DataTypes.DOUBLE(), DataTypes.STRING());
+ FileFormat defaultFormat =
+ new CsvFileFormatFactory().create(new FormatContext(new
Options(), 1024, 1024));
+ Path midFile = new Path(parent, "permissive_middle_" +
UUID.randomUUID() + ".csv");
+ fileIO.writeFile(midFile, "1,oops,world\ny,bad,keep", false);
+ List<InternalRow> midResult = read(defaultFormat, midRowType,
midRowType, midFile);
+ assertThat(midResult).hasSize(2);
+ assertThat(midResult.get(0).getInt(0)).isEqualTo(1);
+ assertThat(midResult.get(0).isNullAt(1)).isTrue();
+
assertThat(midResult.get(0).getString(2)).isEqualTo(fromString("world"));
+ assertThat(midResult.get(1).isNullAt(0)).isTrue();
+ assertThat(midResult.get(1).isNullAt(1)).isTrue();
+
assertThat(midResult.get(1).getString(2)).isEqualTo(fromString("keep"));
+ }
+
@Test
public void testCsvParserParseField() {
RowType rowType =
