This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 2d0db9efe5 [common] Make quoting mean literal in array and row string
cast rules (#9769)
2d0db9efe5 is described below
commit 2d0db9efe5527194764f305104149cdc3f21c7ac
Author: YangJie <[email protected]>
AuthorDate: Sun Sep 13 22:38:51 2026 -0400
[common] Make quoting mean literal in array and row string cast rules
(#9769)
---
.../paimon/casting/StringToArrayCastRule.java | 49 +-----
.../apache/paimon/casting/StringToRowCastRule.java | 57 +------
.../org/apache/paimon/casting/TokenSplitter.java | 141 +++++++++++++++++
.../apache/paimon/casting/CastExecutorTest.java | 169 +++++++++++++++++++++
4 files changed, 323 insertions(+), 93 deletions(-)
diff --git
a/paimon-common/src/main/java/org/apache/paimon/casting/StringToArrayCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/StringToArrayCastRule.java
index 2e48e0b825..45a1cb1d97 100644
---
a/paimon-common/src/main/java/org/apache/paimon/casting/StringToArrayCastRule.java
+++
b/paimon-common/src/main/java/org/apache/paimon/casting/StringToArrayCastRule.java
@@ -26,11 +26,9 @@ import org.apache.paimon.types.DataType;
import org.apache.paimon.types.DataTypeFamily;
import org.apache.paimon.types.DataTypeRoot;
import org.apache.paimon.types.VarCharType;
-import org.apache.paimon.utils.StringUtils;
import java.util.ArrayList;
import java.util.List;
-import java.util.Stack;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -114,52 +112,15 @@ class StringToArrayCastRule extends
AbstractCastRule<BinaryString, InternalArray
private List<Object> parseArrayElements(
String content, CastExecutor<BinaryString, Object>
elementCastExecutor) {
List<Object> elements = new ArrayList<>();
- for (String token : splitArrayElements(content)) {
- String trimmedToken = token.trim();
+ for (TokenSplitter.Token token : TokenSplitter.split(content)) {
+ String value = token.value();
+ // only an unquoted null is the null element; "null" is the
four-character string
Object element =
- "null".equals(trimmedToken)
+ !token.literal() && "null".equals(value)
? null
- :
elementCastExecutor.cast(BinaryString.fromString(trimmedToken));
+ :
elementCastExecutor.cast(BinaryString.fromString(value));
elements.add(element);
}
return elements;
}
-
- private List<String> splitArrayElements(String content) {
- List<String> elements = new ArrayList<>();
- StringBuilder current = new StringBuilder();
- Stack<Character> bracketStack = new Stack<>();
- boolean inQuotes = false;
- boolean escaped = false;
-
- for (char c : content.toCharArray()) {
- if (escaped) {
- escaped = false;
- } else if (c == '\\') {
- escaped = true;
- } else if (c == '"') {
- inQuotes = !inQuotes;
- } else if (!inQuotes) {
- if (StringUtils.isOpenBracket(c)) {
- bracketStack.push(c);
- } else if (StringUtils.isCloseBracket(c) &&
!bracketStack.isEmpty()) {
- bracketStack.pop();
- } else if (c == ',' && bracketStack.isEmpty()) {
- addCurrentElement(elements, current);
- continue;
- }
- }
- current.append(c);
- }
-
- addCurrentElement(elements, current);
- return elements;
- }
-
- private void addCurrentElement(List<String> elements, StringBuilder
current) {
- if (current.length() > 0) {
- elements.add(current.toString());
- current.setLength(0);
- }
- }
}
diff --git
a/paimon-common/src/main/java/org/apache/paimon/casting/StringToRowCastRule.java
b/paimon-common/src/main/java/org/apache/paimon/casting/StringToRowCastRule.java
index 5a2379d273..1ceaf1c2bc 100644
---
a/paimon-common/src/main/java/org/apache/paimon/casting/StringToRowCastRule.java
+++
b/paimon-common/src/main/java/org/apache/paimon/casting/StringToRowCastRule.java
@@ -26,11 +26,8 @@ import org.apache.paimon.types.DataTypeFamily;
import org.apache.paimon.types.DataTypeRoot;
import org.apache.paimon.types.RowType;
import org.apache.paimon.types.VarCharType;
-import org.apache.paimon.utils.StringUtils;
-import java.util.ArrayList;
import java.util.List;
-import java.util.Stack;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -93,7 +90,7 @@ class StringToRowCastRule extends
AbstractCastRule<BinaryString, InternalRow> {
if (content.isEmpty()) {
return createNullRow(fieldCount);
}
- List<String> fieldValues = splitRowFields(content);
+ List<TokenSplitter.Token> fieldValues =
TokenSplitter.split(content);
if (fieldValues.size() != fieldCount) {
throw new RuntimeException(
"Row field count mismatch. Expected: "
@@ -137,60 +134,22 @@ class StringToRowCastRule extends
AbstractCastRule<BinaryString, InternalRow> {
}
private GenericRow createRowFromFields(
- List<String> fieldValues,
+ List<TokenSplitter.Token> fieldValues,
CastExecutor<BinaryString, Object>[] fieldCastExecutors,
int fieldCount) {
GenericRow row = new GenericRow(fieldCount);
for (int i = 0; i < fieldCount; i++) {
- String fieldValue = fieldValues.get(i).trim();
- Object value = parseFieldValue(fieldValue, fieldCastExecutors[i]);
- row.setField(i, value);
+ row.setField(i, parseFieldValue(fieldValues.get(i),
fieldCastExecutors[i]));
}
return row;
}
private Object parseFieldValue(
- String fieldValue, CastExecutor<BinaryString, Object>
castExecutor) {
- return "null".equals(fieldValue)
+ TokenSplitter.Token token, CastExecutor<BinaryString, Object>
castExecutor) {
+ String value = token.value();
+ // only an unquoted null is the null field; "null" is the
four-character string
+ return !token.literal() && "null".equals(value)
? null
- : castExecutor.cast(BinaryString.fromString(fieldValue));
- }
-
- private List<String> splitRowFields(String content) {
- List<String> fields = new ArrayList<>();
- StringBuilder current = new StringBuilder();
- Stack<Character> bracketStack = new Stack<>();
- boolean inQuotes = false;
- boolean escaped = false;
-
- for (char c : content.toCharArray()) {
- if (escaped) {
- escaped = false;
- } else if (c == '\\') {
- escaped = true;
- } else if (c == '"') {
- inQuotes = !inQuotes;
- } else if (!inQuotes) {
- if (StringUtils.isOpenBracket(c)) {
- bracketStack.push(c);
- } else if (StringUtils.isCloseBracket(c) &&
!bracketStack.isEmpty()) {
- bracketStack.pop();
- } else if (c == ',' && bracketStack.isEmpty()) {
- addCurrentField(fields, current);
- continue;
- }
- }
- current.append(c);
- }
-
- addCurrentField(fields, current);
- return fields;
- }
-
- private void addCurrentField(List<String> fields, StringBuilder current) {
- if (current.length() > 0) {
- fields.add(current.toString());
- current.setLength(0);
- }
+ : castExecutor.cast(BinaryString.fromString(value));
}
}
diff --git
a/paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java
b/paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java
new file mode 100644
index 0000000000..54059675db
--- /dev/null
+++ b/paimon-common/src/main/java/org/apache/paimon/casting/TokenSplitter.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.casting;
+
+import org.apache.paimon.utils.StringUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Stack;
+
+/**
+ * Splits the comma-separated body of an array or row literal into its tokens,
honouring quotes,
+ * escapes and nesting.
+ *
+ * <p>A separator only separates outside quotes and at bracket depth zero, so
{@code "a,b"} and
+ * {@code [a, b]} each stay one token.
+ *
+ * <p>Quotes and backslashes are this level's syntax, so they are removed at
depth zero and kept
+ * verbatim inside a nested literal, where they are the inner level's syntax
and the rule for that
+ * element parses them again. Whether they appeared at depth zero is
remembered: quoting or escaping
+ * is how the literal text {@code null} and the empty string are written,
which are otherwise
+ * unrepresentable.
+ *
+ * <p>Whitespace around a token is dropped; whitespace inside quotes is kept.
+ */
+class TokenSplitter {
+
+ /** One token of a literal body, plus whether its value was written as a
literal. */
+ static class Token {
+
+ private final String value;
+ private final boolean literal;
+
+ Token(String value, boolean literal) {
+ this.value = value;
+ this.literal = literal;
+ }
+
+ String value() {
+ return value;
+ }
+
+ /** Whether quotes or an escape made this a literal string rather than
a bare word. */
+ boolean literal() {
+ return literal;
+ }
+ }
+
+ private TokenSplitter() {}
+
+ static List<Token> split(String content) {
+ List<Token> tokens = new ArrayList<>();
+ StringBuilder current = new StringBuilder();
+ Stack<Character> bracketStack = new Stack<>();
+ boolean inQuotes = false;
+ boolean escaped = false;
+ boolean literal = false;
+ // length of current up to the last character that was not unquoted
whitespace
+ int end = 0;
+
+ for (char c : content.toCharArray()) {
+ boolean nested = !bracketStack.isEmpty();
+ if (escaped) {
+ // the escapee stands for itself and is never read as syntax
+ escaped = false;
+ current.append(c);
+ end = current.length();
+ continue;
+ }
+ if (c == '\\') {
+ escaped = true;
+ if (nested) {
+ // the inner rule has to see the escape to protect its own
separators
+ current.append(c);
+ end = current.length();
+ } else {
+ literal = true;
+ }
+ continue;
+ }
+ if (c == '"') {
+ inQuotes = !inQuotes;
+ if (nested) {
+ current.append(c);
+ end = current.length();
+ } else {
+ literal = true;
+ }
+ continue;
+ }
+ if (!inQuotes) {
+ if (StringUtils.isOpenBracket(c)) {
+ bracketStack.push(c);
+ } else if (StringUtils.isCloseBracket(c) &&
!bracketStack.isEmpty()) {
+ bracketStack.pop();
+ } else if (c == ',' && bracketStack.isEmpty()) {
+ addToken(tokens, current, end, literal);
+ current.setLength(0);
+ end = 0;
+ literal = false;
+ continue;
+ } else if (Character.isWhitespace(c) && end == 0) {
+ // leading whitespace outside quotes is not part of the
token
+ continue;
+ }
+ }
+ current.append(c);
+ if (inQuotes || !Character.isWhitespace(c)) {
+ end = current.length();
+ }
+ }
+
+ addToken(tokens, current, end, literal);
+ return tokens;
+ }
+
+ private static void addToken(
+ List<Token> tokens, StringBuilder current, int end, boolean
literal) {
+ // whitespace is not part of a token, so one made only of whitespace
was never written;
+ // quoting is how an empty value is written
+ if (end > 0 || literal) {
+ tokens.add(new Token(current.substring(0, end), literal));
+ }
+ }
+}
diff --git
a/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java
b/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java
index 31a776c87d..8325687c21 100644
---
a/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java
+++
b/paimon-common/src/test/java/org/apache/paimon/casting/CastExecutorTest.java
@@ -975,6 +975,175 @@ public class CastExecutorTest {
BinaryString.fromString("{1, {2025-01-06, {1 -> [1, null, 2]},
null}}"));
}
+ @Test
+ public void testStringToArrayQuotingAndEscaping() {
+ ArrayType arrayType = new ArrayType(DataTypes.STRING());
+ CastExecutor<BinaryString, InternalArray> cast =
+ (CastExecutor<BinaryString, InternalArray>)
+ CastExecutors.resolve(VarCharType.STRING_TYPE,
arrayType);
+
+ // quotes group a token across the separator and do not survive into
the value
+ compareCastResult(
+ cast,
+ BinaryString.fromString("[\"a,b\", c]"),
+ new GenericArray(
+ new Object[] {
+ BinaryString.fromString("a,b"),
BinaryString.fromString("c")
+ }));
+
+ // quoting is how an empty string is written, so the element must be
kept
+ compareCastResult(
+ cast,
+ BinaryString.fromString("[\"\", a]"),
+ new GenericArray(
+ new Object[] {BinaryString.fromString(""),
BinaryString.fromString("a")}));
+
+ // an unquoted null is the null element; a quoted one is the
four-character string
+ compareCastResult(
+ cast,
+ BinaryString.fromString("[null, \"null\"]"),
+ new GenericArray(new Object[] {null,
BinaryString.fromString("null")}));
+
+ // a backslash escapes the next character and is itself syntax
+ compareCastResult(
+ cast,
+ BinaryString.fromString("[a\\,b, c]"),
+ new GenericArray(
+ new Object[] {
+ BinaryString.fromString("a,b"),
BinaryString.fromString("c")
+ }));
+ }
+
+ @Test
+ public void testStringToRowQuotingAndEscaping() {
+ RowType rowType =
+ DataTypes.ROW(
+ DataTypes.FIELD(0, "f0", DataTypes.STRING()),
+ DataTypes.FIELD(1, "f1", DataTypes.INT()));
+ CastExecutor<BinaryString, InternalRow> cast =
+ (CastExecutor<BinaryString, InternalRow>)
+ CastExecutors.resolve(VarCharType.STRING_TYPE,
rowType);
+
+ compareCastResult(
+ cast,
+ BinaryString.fromString("{\"a,b\", 2}"),
+ GenericRow.of(BinaryString.fromString("a,b"), 2));
+
+ // an empty quoted field stays a field, so the field count still
matches
+ compareCastResult(
+ cast,
+ BinaryString.fromString("{\"\", 2}"),
+ GenericRow.of(BinaryString.fromString(""), 2));
+
+ // a quoted null is the string, an unquoted one is SQL NULL
+ compareCastResult(
+ cast,
+ BinaryString.fromString("{\"null\", 2}"),
+ GenericRow.of(BinaryString.fromString("null"), 2));
+ compareCastResult(cast, BinaryString.fromString("{null, 2}"),
GenericRow.of(null, 2));
+
+ compareCastResult(
+ cast,
+ BinaryString.fromString("{a\\,b, 2}"),
+ GenericRow.of(BinaryString.fromString("a,b"), 2));
+ }
+
+ @Test
+ public void testStringToNestedArrayKeepsInnerSyntax() {
+ // quotes and escapes belong to whichever level wrote them: the outer
split must leave a
+ // nested literal's own syntax in place for the element rule to parse
again, or the inner
+ // separator stops being protected and the element count changes
+ ArrayType nested = new ArrayType(new ArrayType(DataTypes.STRING()));
+ CastExecutor<BinaryString, InternalArray> cast =
+ (CastExecutor<BinaryString, InternalArray>)
+ CastExecutors.resolve(VarCharType.STRING_TYPE, nested);
+
+ assertNestedElements(cast, "[[\"a,b\"], [c]]", new String[] {"a,b"},
new String[] {"c"});
+ assertNestedElements(cast, "[[a\\,b], [c]]", new String[] {"a,b"}, new
String[] {"c"});
+ assertNestedElements(cast, "[[\"null\"], [a]]", new String[] {"null"},
new String[] {"a"});
+ assertNestedElements(cast, "[[\"\"], [a]]", new String[] {""}, new
String[] {"a"});
+ assertNestedElements(cast, "[[\" a \"], [b]]", new String[] {" a "},
new String[] {"b"});
+ assertNestedElements(cast, "[[1, 2], [3]]", new String[] {"1", "2"},
new String[] {"3"});
+ }
+
+ private static void assertNestedElements(
+ CastExecutor<BinaryString, InternalArray> cast, String literal,
String[]... expected) {
+ InternalArray outer = cast.cast(BinaryString.fromString(literal));
+ assertThat(outer.size()).as("outer size of %s",
literal).isEqualTo(expected.length);
+ for (int i = 0; i < expected.length; i++) {
+ InternalArray inner = outer.getArray(i);
+ assertThat(inner.size())
+ .as("inner size of %s at %s", literal, i)
+ .isEqualTo(expected[i].length);
+ for (int j = 0; j < expected[i].length; j++) {
+ assertThat(inner.getString(j).toString())
+ .as("element %s.%s of %s", i, j, literal)
+ .isEqualTo(expected[i][j]);
+ }
+ }
+ }
+
+ @Test
+ public void testStringToArrayEscapedNullIsALiteral() {
+ ArrayType arrayType = new ArrayType(DataTypes.STRING());
+ CastExecutor<BinaryString, InternalArray> cast =
+ (CastExecutor<BinaryString, InternalArray>)
+ CastExecutors.resolve(VarCharType.STRING_TYPE,
arrayType);
+
+ // escaping, like quoting, says the token is written text rather than
the null literal
+ compareCastResult(
+ cast,
+ BinaryString.fromString("[\\null, x]"),
+ new GenericArray(
+ new Object[] {
+ BinaryString.fromString("null"),
BinaryString.fromString("x")
+ }));
+ }
+
+ @Test
+ public void testStringToRowWhitespaceOnlyFieldIsNotAField() {
+ RowType rowType =
+ DataTypes.ROW(
+ DataTypes.FIELD(0, "f0", DataTypes.STRING()),
+ DataTypes.FIELD(1, "f1", DataTypes.STRING()),
+ DataTypes.FIELD(2, "f2", DataTypes.STRING()));
+ CastExecutor<BinaryString, InternalRow> cast =
+ (CastExecutor<BinaryString, InternalRow>)
+ CastExecutors.resolve(VarCharType.STRING_TYPE,
rowType);
+
+ // whitespace is not part of a token, so a field made only of
whitespace was never
+ // written, and where it sits does not change that
+ for (String literal : new String[] {"{a, ,b}", "{ ,a,b}", "{a,b, }"})
{
+ assertThatThrownBy(() ->
cast.cast(BinaryString.fromString(literal)))
+ .as("%s", literal)
+ .hasMessageContaining("Row field count mismatch. Expected:
3, Actual: 2");
+ }
+
+ // quoting is how an empty field is written
+ compareCastResult(
+ cast,
+ BinaryString.fromString("{a, \"\", b}"),
+ GenericRow.of(
+ BinaryString.fromString("a"),
+ BinaryString.fromString(""),
+ BinaryString.fromString("b")));
+ }
+
+ @Test
+ public void testStringToArraySkipsAnEmptyElement() {
+ ArrayType arrayType = new ArrayType(DataTypes.INT());
+ CastExecutor<BinaryString, InternalArray> cast =
+ (CastExecutor<BinaryString, InternalArray>)
+ CastExecutors.resolve(VarCharType.STRING_TYPE,
arrayType);
+
+ // an element written as nothing, with or without whitespace, is no
element: handing the
+ // empty string to the int cast instead would fail the whole array
+ compareCastResult(
+ cast, BinaryString.fromString("[1,,3]"), new GenericArray(new
Integer[] {1, 3}));
+ compareCastResult(
+ cast, BinaryString.fromString("[1, , 3]"), new
GenericArray(new Integer[] {1, 3}));
+ }
+
@Test
public void testSplitMapEntriesWithQuotes() {
String content = "1, \"abc\"";
