This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 93efa6386c [common] Follow Spark's SUBSTRING position semantics and
count characters in TRIM (#9329)
93efa6386c is described below
commit 93efa6386ccdc066ca3f6a30e3e5ec546046a44f
Author: Jiajia Li <[email protected]>
AuthorDate: Wed Aug 26 11:25:40 2026 +0800
[common] Follow Spark's SUBSTRING position semantics and count characters
in TRIM (#9329)
---
.../java/org/apache/paimon/data/BinaryString.java | 71 ++++++++++++++++++++++
.../paimon/predicate/SubstringTransform.java | 18 +-----
.../org/apache/paimon/predicate/TrimTransform.java | 17 +++---
.../org/apache/paimon/data/BinaryStringTest.java | 63 +++++++++++++++++++
.../paimon/predicate/SubstringTransformTest.java | 33 ++++++++++
.../apache/paimon/predicate/TrimTransformTest.java | 23 +++++++
6 files changed, 202 insertions(+), 23 deletions(-)
diff --git
a/paimon-common/src/main/java/org/apache/paimon/data/BinaryString.java
b/paimon-common/src/main/java/org/apache/paimon/data/BinaryString.java
index 750456f3b5..73806e4ab6 100644
--- a/paimon-common/src/main/java/org/apache/paimon/data/BinaryString.java
+++ b/paimon-common/src/main/java/org/apache/paimon/data/BinaryString.java
@@ -411,6 +411,77 @@ public final class BinaryString extends BinarySection
implements Comparable<Bina
}
}
+ /** SQL positions: one-based, zero means one, negative counts back from
the end. */
+ public BinaryString substringSQL(int pos, int length) {
+ int len = numChars();
+ int start = 0;
+ if (pos > 0) {
+ start = pos - 1;
+ } else if (pos < 0) {
+ start = len + pos;
+ }
+ int end;
+ if ((long) start + length > Integer.MAX_VALUE) {
+ end = Integer.MAX_VALUE;
+ } else if ((long) start + length < Integer.MIN_VALUE) {
+ end = Integer.MIN_VALUE;
+ } else {
+ end = start + length;
+ }
+ return substring(start, end);
+ }
+
+ /** Removes leading whole characters contained in {@code trimString}, null
for null. */
+ public BinaryString trimLeft(BinaryString trimString) {
+ if (trimString == null) {
+ return null;
+ }
+ int start = 0;
+ while (start < sizeInBytes) {
+ int charBytes = numBytesForFirstByte(byteAt(start));
+ // a truncated sequence is not a character
+ if (charBytes > sizeInBytes - start) {
+ break;
+ }
+ if (!trimString.contains(fromAddress(segments, offset + start,
charBytes))) {
+ break;
+ }
+ start += charBytes;
+ }
+ return start >= sizeInBytes ? EMPTY_UTF8 : copyBinaryString(start,
sizeInBytes - 1);
+ }
+
+ /** Trailing counterpart of {@link #trimLeft(BinaryString)}. */
+ public BinaryString trimRight(BinaryString trimString) {
+ if (trimString == null) {
+ return null;
+ }
+ int end = sizeInBytes;
+ while (end > 0) {
+ int charStart = end - 1;
+ while (charStart > 0 && (byteAt(charStart) & 0xC0) == 0x80) {
+ charStart--;
+ }
+ // a truncated sequence is not a character
+ if (numBytesForFirstByte(byteAt(charStart)) != end - charStart) {
+ break;
+ }
+ if (!trimString.contains(fromAddress(segments, offset + charStart,
end - charStart))) {
+ break;
+ }
+ end = charStart;
+ }
+ return end == 0 ? EMPTY_UTF8 : copyBinaryString(0, end - 1);
+ }
+
+ /** Both-sides counterpart of {@link #trimLeft(BinaryString)}. */
+ public BinaryString trim(BinaryString trimString) {
+ if (trimString == null) {
+ return null;
+ }
+ return trimLeft(trimString).trimRight(trimString);
+ }
+
private BinaryString trimMultiSegs() {
int s = 0;
int e = this.sizeInBytes - 1;
diff --git
a/paimon-common/src/main/java/org/apache/paimon/predicate/SubstringTransform.java
b/paimon-common/src/main/java/org/apache/paimon/predicate/SubstringTransform.java
index cb2c0f425c..7be7e66e8e 100644
---
a/paimon-common/src/main/java/org/apache/paimon/predicate/SubstringTransform.java
+++
b/paimon-common/src/main/java/org/apache/paimon/predicate/SubstringTransform.java
@@ -75,21 +75,9 @@ public class SubstringTransform implements Transform {
return null;
}
- int sourceLength = sourceString.numChars();
- int beginIndex = readPosition(inputs.get(1), row);
- if (beginIndex > sourceLength) {
- return BinaryString.EMPTY_UTF8;
- }
-
- int endIndex = sourceLength;
- if (hasLength) {
- endIndex = beginIndex + readPosition(inputs.get(2), row) - 1;
- }
- endIndex = Math.min(endIndex, sourceLength);
- beginIndex--;
- checkArgument(beginIndex < endIndex);
-
- return sourceString.substring(beginIndex, endIndex);
+ int pos = readPosition(inputs.get(1), row);
+ int length = hasLength ? readPosition(inputs.get(2), row) :
Integer.MAX_VALUE;
+ return sourceString.substringSQL(pos, length);
}
private static boolean isNullPosition(Object position, InternalRow row) {
diff --git
a/paimon-common/src/main/java/org/apache/paimon/predicate/TrimTransform.java
b/paimon-common/src/main/java/org/apache/paimon/predicate/TrimTransform.java
index 01ba75e729..cd97456019 100644
--- a/paimon-common/src/main/java/org/apache/paimon/predicate/TrimTransform.java
+++ b/paimon-common/src/main/java/org/apache/paimon/predicate/TrimTransform.java
@@ -19,7 +19,6 @@
package org.apache.paimon.predicate;
import org.apache.paimon.data.BinaryString;
-import org.apache.paimon.utils.StringUtils;
import java.util.List;
@@ -32,6 +31,9 @@ public class TrimTransform extends StringTransform {
public static final String NAME = "TRIM";
+ /** The one-input form trims spaces only, not every whitespace character.
*/
+ private static final BinaryString SPACE = BinaryString.fromString(" ");
+
private final Flag trimFlag;
public TrimTransform(List<Object> inputs, Flag trimFlag) {
@@ -50,22 +52,21 @@ public class TrimTransform extends StringTransform {
if (inputs.get(0) == null) {
return null;
}
- String sourceString = inputs.get(0).toString();
- String charsToTrim = " ";
+ BinaryString sourceString = inputs.get(0);
+ BinaryString charsToTrim = SPACE;
if (inputs.size() == 2) {
if (inputs.get(1) == null) {
- // StringUtils.ltrim/rtrim treat a null charsToTrim as a null
result
return null;
}
- charsToTrim = inputs.get(1).toString();
+ charsToTrim = inputs.get(1);
}
switch (trimFlag) {
case BOTH:
- return BinaryString.fromString(StringUtils.trim(sourceString,
charsToTrim));
+ return sourceString.trim(charsToTrim);
case LEADING:
- return BinaryString.fromString(StringUtils.ltrim(sourceString,
charsToTrim));
+ return sourceString.trimLeft(charsToTrim);
case TRAILING:
- return BinaryString.fromString(StringUtils.rtrim(sourceString,
charsToTrim));
+ return sourceString.trimRight(charsToTrim);
default:
throw new IllegalArgumentException("Invalid trim way " +
trimFlag.name());
}
diff --git
a/paimon-common/src/test/java/org/apache/paimon/data/BinaryStringTest.java
b/paimon-common/src/test/java/org/apache/paimon/data/BinaryStringTest.java
index f7c2525f42..4fd497b1b3 100644
--- a/paimon-common/src/test/java/org/apache/paimon/data/BinaryStringTest.java
+++ b/paimon-common/src/test/java/org/apache/paimon/data/BinaryStringTest.java
@@ -151,6 +151,69 @@ public class BinaryStringTest {
assertThat(EMPTY_UTF8.getSizeInBytes()).isEqualTo(0);
}
+ @TestTemplate
+ public void substringSQL() {
+ BinaryString s = fromString("abcdef");
+ assertThat(s.substringSQL(0, 2)).isEqualTo(fromString("ab"));
+ assertThat(s.substringSQL(1, 2)).isEqualTo(fromString("ab"));
+ assertThat(s.substringSQL(-2, 2)).isEqualTo(fromString("ef"));
+ assertThat(s.substringSQL(-9, 2)).isEqualTo(EMPTY_UTF8);
+ assertThat(s.substringSQL(-9, 5)).isEqualTo(fromString("ab"));
+ assertThat(s.substringSQL(-9,
Integer.MAX_VALUE)).isEqualTo(fromString("abcdef"));
+ assertThat(s.substringSQL(Integer.MIN_VALUE,
Integer.MIN_VALUE)).isEqualTo(EMPTY_UTF8);
+ assertThat(s.substringSQL(2, 0)).isEqualTo(EMPTY_UTF8);
+ assertThat(s.substringSQL(9, 2)).isEqualTo(EMPTY_UTF8);
+ assertThat(s.substringSQL(1,
Integer.MAX_VALUE)).isEqualTo(fromString("abcdef"));
+ assertThat(s.substringSQL(-2,
Integer.MAX_VALUE)).isEqualTo(fromString("ef"));
+
+ assertThat(fromString("\uD83D\uDE00abc").substringSQL(2,
2)).isEqualTo(fromString("ab"));
+ }
+
+ @TestTemplate
+ public void trimWithTrimString() {
+
assertThat(fromString("xyzaxyz").trim(fromString("xyz"))).isEqualTo(fromString("a"));
+
assertThat(fromString("zyxaxyz").trim(fromString("xyz"))).isEqualTo(fromString("a"));
+
assertThat(fromString("xyzaxyz").trimLeft(fromString("xyz"))).isEqualTo(fromString("axyz"));
+ assertThat(fromString("xyzaxyz").trimRight(fromString("xyz")))
+ .isEqualTo(fromString("xyza"));
+
+
assertThat(fromString("abc").trim(fromString("xyz"))).isEqualTo(fromString("abc"));
+
assertThat(fromString("abc").trim(EMPTY_UTF8)).isEqualTo(fromString("abc"));
+
+
assertThat(fromString("xyx").trim(fromString("xyz"))).isEqualTo(EMPTY_UTF8);
+
assertThat(fromString("xyx").trimLeft(fromString("xyz"))).isEqualTo(EMPTY_UTF8);
+
assertThat(fromString("xyx").trimRight(fromString("xyz"))).isEqualTo(EMPTY_UTF8);
+ assertThat(fromString("").trim(fromString("x"))).isEqualTo(EMPTY_UTF8);
+
+ assertThat(fromString("abc").trim(null)).isNull();
+ assertThat(fromString("abc").trimLeft(null)).isNull();
+ assertThat(fromString("abc").trimRight(null)).isNull();
+ }
+
+ @TestTemplate
+ public void trimComparesWholeCharacters() {
+
assertThat(fromString("。x。").trim(fromString("、"))).isEqualTo(fromString("。x。"));
+
assertThat(fromString("中x中").trim(fromString("丁"))).isEqualTo(fromString("中x中"));
+
assertThat(fromString("中x中").trim(fromString("中"))).isEqualTo(fromString("x"));
+
+
assertThat(fromString("😁x😁").trim(fromString("😀"))).isEqualTo(fromString("😁x😁"));
+
assertThat(fromString("😁x😁").trim(fromString("😁"))).isEqualTo(fromString("x"));
+
assertThat(fromString("x").trimRight(fromString("😀"))).isEqualTo(fromString("x"));
+
+
assertThat(fromString("a中b").trim(fromString("ab中"))).isEqualTo(EMPTY_UTF8);
+ }
+
+ @TestTemplate
+ public void truncatedSequenceIsNotTakenFromTheNeighbouringBytes() {
+ byte[] buffer = new byte[16];
+ System.arraycopy("\u4e2dSECRET".getBytes(UTF_8), 0, buffer, 0, 9);
+ MemorySegment[] segments = {MemorySegment.wrap(buffer)};
+
+ BinaryString truncated = BinaryString.fromAddress(segments, 0, 1);
+
assertThat(truncated.trimLeft(fromString("\u4e2d")).getSizeInBytes()).isEqualTo(1);
+
assertThat(truncated.trimRight(fromString("\u4e2d")).getSizeInBytes()).isEqualTo(1);
+ }
+
@TestTemplate
public void compareTo() {
assertThat(fromString(" ").compareTo(blankString(3))).isEqualTo(0);
diff --git
a/paimon-common/src/test/java/org/apache/paimon/predicate/SubstringTransformTest.java
b/paimon-common/src/test/java/org/apache/paimon/predicate/SubstringTransformTest.java
index 7a955e524e..163770b378 100644
---
a/paimon-common/src/test/java/org/apache/paimon/predicate/SubstringTransformTest.java
+++
b/paimon-common/src/test/java/org/apache/paimon/predicate/SubstringTransformTest.java
@@ -84,6 +84,39 @@ class SubstringTransformTest {
assertThat(result).isEqualTo(BinaryString.fromString(""));
}
+ @Test
+ public void testSqlPositionSemantics() {
+ List<Object> inputs = new ArrayList<>();
+ inputs.add(BinaryString.fromString("abcdef"));
+ inputs.add(1);
+ inputs.add(2);
+
+ for (Object[] spec :
+ new Object[][] {
+ {0, 2, "ab"},
+ {1, 2, "ab"},
+ {-2, 2, "ef"},
+ {-2, 9, "ef"},
+ {-9, 2, ""},
+ {2, 0, ""},
+ {2, -1, ""},
+ {9, 2, ""},
+ {-9, 5, "ab"}
+ }) {
+ inputs.set(1, spec[0]);
+ inputs.set(2, spec[1]);
+ assertThat(new
SubstringTransform(inputs).transform(GenericRow.of()))
+ .isEqualTo(BinaryString.fromString((String) spec[2]));
+ }
+
+ inputs.remove(2);
+ for (Object[] spec : new Object[][] {{0, "abcdef"}, {-2, "ef"}, {9,
""}, {-9, "abcdef"}}) {
+ inputs.set(1, spec[0]);
+ assertThat(new
SubstringTransform(inputs).transform(GenericRow.of()))
+ .isEqualTo(BinaryString.fromString((String) spec[1]));
+ }
+ }
+
@Test
public void testSubstringWithSupplementaryCharacter() {
SubstringTransform transform =
diff --git
a/paimon-common/src/test/java/org/apache/paimon/predicate/TrimTransformTest.java
b/paimon-common/src/test/java/org/apache/paimon/predicate/TrimTransformTest.java
index 597a71d280..fdb35fa87d 100644
---
a/paimon-common/src/test/java/org/apache/paimon/predicate/TrimTransformTest.java
+++
b/paimon-common/src/test/java/org/apache/paimon/predicate/TrimTransformTest.java
@@ -101,6 +101,29 @@ class TrimTransformTest {
assertThat(result).isEqualTo(BinaryString.fromString(" aa"));
}
+ @Test
+ public void testTrimsWholeCharactersNotUtf16CodeUnits() {
+ List<Object> inputs = new ArrayList<>();
+ inputs.add(new FieldRef(0, "f0", DataTypes.STRING()));
+ inputs.add(BinaryString.fromString("\uD83D\uDE00"));
+ GenericRow row =
GenericRow.of(BinaryString.fromString("\uD83D\uDE01x\uD83D\uDE01"));
+
+ assertThat(new TrimTransform(inputs,
TrimTransform.Flag.BOTH).transform(row))
+
.isEqualTo(BinaryString.fromString("\uD83D\uDE01x\uD83D\uDE01"));
+
+ inputs.set(1, BinaryString.fromString("\uD83D\uDE01"));
+ assertThat(new TrimTransform(inputs,
TrimTransform.Flag.BOTH).transform(row))
+ .isEqualTo(BinaryString.fromString("x"));
+
+ List<Object> trailing = new ArrayList<>();
+ trailing.add(new FieldRef(0, "f0", DataTypes.STRING()));
+ trailing.add(BinaryString.fromString("\uD83D\uDE00"));
+ assertThat(
+ new TrimTransform(trailing,
TrimTransform.Flag.TRAILING)
+
.transform(GenericRow.of(BinaryString.fromString("x\uD800\uDE00"))))
+ .isEqualTo(BinaryString.fromString("x\uD800\uDE00"));
+ }
+
@Test
public void testNullCharsToTrimYieldsNull() {
List<Object> inputs = new ArrayList<>();
