This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 9ad161e120 [format] Escape the escape character when writing CSV
(#9278)
9ad161e120 is described below
commit 9ad161e120d61cbbf194bf7fb9f48b1f978963ff
Author: ZIHAN DAI <[email protected]>
AuthorDate: Thu Aug 20 12:02:32 2026 +1000
[format] Escape the escape character when writing CSV (#9278)
---
.../apache/paimon/format/csv/CsvFormatWriter.java | 18 +++++++++------
.../paimon/format/csv/CsvFileFormatTest.java | 27 ++++++++++++++++++++++
2 files changed, 38 insertions(+), 7 deletions(-)
diff --git
a/paimon-format/src/main/java/org/apache/paimon/format/csv/CsvFormatWriter.java
b/paimon-format/src/main/java/org/apache/paimon/format/csv/CsvFormatWriter.java
index 2004eece23..6de063d0d7 100644
---
a/paimon-format/src/main/java/org/apache/paimon/format/csv/CsvFormatWriter.java
+++
b/paimon-format/src/main/java/org/apache/paimon/format/csv/CsvFormatWriter.java
@@ -98,22 +98,26 @@ public class CsvFormatWriter extends AbstractTextFileWriter
{
return csvOptions.nullLiteral();
}
+ String quote = csvOptions.quoteCharacter();
+ String escape = csvOptions.escapeCharacter();
+ boolean escapable = !escape.isEmpty();
+
// Optimized escaping with early exit checks
boolean needsQuoting =
field.indexOf(csvOptions.fieldDelimiter().charAt(0)) >= 0
|| field.indexOf(csvOptions.lineDelimiter().charAt(0))
>= 0
- ||
field.indexOf(csvOptions.quoteCharacter().charAt(0)) >= 0;
+ || field.indexOf(quote.charAt(0)) >= 0
+ || (escapable && field.indexOf(escape.charAt(0)) >= 0);
if (!needsQuoting) {
return field;
}
- // Only escape if needed
- String escaped =
- field.replace(
- csvOptions.quoteCharacter(),
- csvOptions.escapeCharacter() +
csvOptions.quoteCharacter());
- return csvOptions.quoteCharacter() + escaped +
csvOptions.quoteCharacter();
+ // Only escape if needed. The escape character goes first: CsvParser
drops an escape
+ // character that is not followed by a quote or another escape, and
escaping the quotes
+ // first would double the escape characters inserted for them.
+ String escaped = escapable ? field.replace(escape, escape + escape) :
field;
+ return quote + escaped.replace(quote, escape + quote) + quote;
}
/** Optimized string casting with caching and fast paths for common types.
*/
diff --git
a/paimon-format/src/test/java/org/apache/paimon/format/csv/CsvFileFormatTest.java
b/paimon-format/src/test/java/org/apache/paimon/format/csv/CsvFileFormatTest.java
index 2254931327..5e47ebae95 100644
---
a/paimon-format/src/test/java/org/apache/paimon/format/csv/CsvFileFormatTest.java
+++
b/paimon-format/src/test/java/org/apache/paimon/format/csv/CsvFileFormatTest.java
@@ -371,9 +371,11 @@ public class CsvFileFormatTest extends FormatReadWriteTest
{
// Verify results
assertThat(result).hasSize(3);
assertThat(result.get(0).getInt(0)).isEqualTo(1);
+
assertThat(result.get(0).getString(1).toString()).isEqualTo("Value\"With\"Quotes");
assertThat(result.get(1).getInt(0)).isEqualTo(2);
assertThat(result.get(1).getString(1).toString()).isEqualTo("Normal Value");
assertThat(result.get(2).getInt(0)).isEqualTo(3);
+
assertThat(result.get(2).getString(1).toString()).isEqualTo("Special\\Characters");
}
}
@@ -826,6 +828,31 @@ public class CsvFileFormatTest extends FormatReadWriteTest
{
* Performs a complete write-read test with the given options and test
data. Returns the data
* that was read back for further verification.
*/
+ @Test
+ public void testFieldsContainingTheEscapeCharacterRoundTrip() throws
IOException {
+ RowType rowType = DataTypes.ROW(DataTypes.INT().notNull(),
DataTypes.STRING());
+ // every one of these is written unquoted or half-quoted unless the
escape character is
+ // itself escaped, and CsvParser then drops it
+ String[] inputs = {
+ "Special\\Characters", "trailing\\", "a,b\\", "\\\\double",
"\\\"quoteAfterEscape"
+ };
+
+ List<InternalRow> testData = new ArrayList<>();
+ for (int i = 0; i < inputs.length; i++) {
+ testData.add(GenericRow.of(i, BinaryString.fromString(inputs[i])));
+ }
+
+ List<InternalRow> result =
+ writeThenRead(new Options(), rowType, rowType, testData,
"escape_round_trip");
+
+ assertThat(result).hasSize(inputs.length);
+ for (int i = 0; i < inputs.length; i++) {
+ assertThat(result.get(i).getInt(0)).isEqualTo(i);
+ assertThat(result.get(i).getString(1)).isNotNull();
+
assertThat(result.get(i).getString(1).toString()).isEqualTo(inputs[i]);
+ }
+ }
+
private List<InternalRow> writeThenRead(
Options options,
RowType fullRowType,
