This is an automated email from the ASF dual-hosted git repository.
morrySnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4a4b8578ec3 [improvement](cast) Add cast conversion failure analysis
(#68367)
4a4b8578ec3 is described below
commit 4a4b8578ec32bb22d081254f3f28c43f4d607296
Author: morrySnow <[email protected]>
AuthorDate: Tue Sep 22 18:06:34 2026 +0800
[improvement](cast) Add cast conversion failure analysis (#68367)
### What problem does this PR solve?
Problem Summary: `Cast.castNullable` describes the outer result column's
nullability, but does not tell an optimizer whether converting a
non-null value can fail. For example, `ARRAY<STRING> -> ARRAY<INT>` can
leave the outer array non-null while an element conversion fails. This
PR adds `Cast.mayFailOnNonNullInput` as a reusable type-pair and
expression-level analysis. It recurses through ARRAY, MAP, and STRUCT,
recognizes BE conversions proven not to report data-dependent failures,
and conservatively treats other conversions as possibly failing. A
strict CAST can report such a failure as an error, while a non-strict
CAST or TRY_CAST may produce NULL. The new API has no production callers
yet.
---
.../doris/nereids/trees/expressions/Cast.java | 72 ++++++++++++++++++++++
.../doris/nereids/trees/expressions/CastTest.java | 65 +++++++++++++++++++
2 files changed, 137 insertions(+)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java
index f31ce955b74..e2316b40a60 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java
@@ -27,6 +27,7 @@ import
org.apache.doris.nereids.trees.expressions.literal.TimeStampNsLiteral;
import org.apache.doris.nereids.trees.expressions.literal.TimestampTzLiteral;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.types.ArrayType;
import org.apache.doris.nereids.types.BigIntType;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.types.DateTimeV2Type;
@@ -34,7 +35,10 @@ import org.apache.doris.nereids.types.DecimalV2Type;
import org.apache.doris.nereids.types.DecimalV3Type;
import org.apache.doris.nereids.types.IntegerType;
import org.apache.doris.nereids.types.LargeIntType;
+import org.apache.doris.nereids.types.MapType;
import org.apache.doris.nereids.types.SmallIntType;
+import org.apache.doris.nereids.types.StructField;
+import org.apache.doris.nereids.types.StructType;
import org.apache.doris.nereids.types.TimeStampNsType;
import org.apache.doris.nereids.types.TimeStampTzType;
import org.apache.doris.nereids.types.TinyIntType;
@@ -269,6 +273,74 @@ public class Cast extends Expression implements
UnaryExpression, Monotonic {
return false;
}
+ /**
+ * Whether converting a non-null value can fail. A strict CAST reports
such a failure as an
+ * error; a non-strict CAST may produce NULL, including inside an ARRAY,
MAP, or STRUCT;
+ * TRY_CAST can turn a reported error into NULL. This property is
independent of the input's
+ * nullability and of the cast mode. It is conservative for conversions
without a proven
+ * failure-free BE implementation. It does not cover unrelated execution
errors.
+ */
+ public boolean mayFailOnNonNullInput() {
+ return mayFailOnNonNullInput(child().getDataType(), targetType);
+ }
+
+ /** Whether the BE conversion from sourceType to targetType can fail on a
non-null input. */
+ public static boolean mayFailOnNonNullInput(DataType sourceType, DataType
targetType) {
+ if (sourceType.equals(targetType)) {
+ return false;
+ }
+ if (sourceType instanceof ArrayType && targetType instanceof
ArrayType) {
+ return mayFailOnNonNullInput(((ArrayType)
sourceType).getItemType(),
+ ((ArrayType) targetType).getItemType());
+ }
+ if (sourceType instanceof MapType && targetType instanceof MapType) {
+ MapType sourceMap = (MapType) sourceType;
+ MapType targetMap = (MapType) targetType;
+ return mayFailOnNonNullInput(sourceMap.getKeyType(),
targetMap.getKeyType())
+ || mayFailOnNonNullInput(sourceMap.getValueType(),
targetMap.getValueType());
+ }
+ if (sourceType instanceof StructType && targetType instanceof
StructType) {
+ List<StructField> sourceFields = ((StructType)
sourceType).getFields();
+ List<StructField> targetFields = ((StructType)
targetType).getFields();
+ if (sourceFields.size() != targetFields.size()) {
+ return true;
+ }
+ for (int i = 0; i < sourceFields.size(); i++) {
+ StructField sourceField = sourceFields.get(i);
+ StructField targetField = targetFields.get(i);
+ if ((sourceField.isNullable() && !targetField.isNullable())
+ || mayFailOnNonNullInput(sourceField.getDataType(),
targetField.getDataType())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ // BE casts to a character type through to_string_batch for these
source types.
+ // JSON and VARIANT take separate paths and are intentionally left
conservative.
+ boolean concreteNumber = (sourceType.isIntegralType() &&
sourceType.width() > 0)
+ || sourceType.isFloatLikeType() ||
sourceType.isDecimalLikeType();
+ if (targetType.isStringLikeType()) {
+ return !(sourceType.isStringLikeType() ||
sourceType.isBooleanType() || concreteNumber
+ || sourceType.isDateLikeType() || sourceType.isTimeType()
+ || sourceType.isArrayType() || sourceType.isMapType() ||
sourceType.isStructType());
+ }
+ // The number-to-boolean and number-to-floating BE kernels cannot
report a conversion
+ // failure. Precision loss (including floating overflow to infinity)
is not a failure.
+ if (targetType.isBooleanType() || targetType.isFloatLikeType()) {
+ return !(sourceType.isBooleanType() || concreteNumber);
+ }
+ // All Doris integral types are signed; a cast to an equal or wider
integral type fits.
+ if (sourceType.isIntegralType() && sourceType.width() > 0
+ && targetType.isIntegralType() && targetType.width() > 0) {
+ return sourceType.width() > targetType.width();
+ }
+ if (sourceType.isBooleanType() && targetType.isIntegralType() &&
targetType.width() > 0) {
+ return false;
+ }
+ return true;
+ }
+
@Override
public Cast withChildren(List<Expression> children) {
Preconditions.checkArgument(children.size() == 1);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/CastTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/CastTest.java
index 7d0ee0f1f26..89f1a9eb610 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/CastTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/CastTest.java
@@ -36,8 +36,11 @@ import org.apache.doris.nereids.types.IPv6Type;
import org.apache.doris.nereids.types.IntegerType;
import org.apache.doris.nereids.types.JsonType;
import org.apache.doris.nereids.types.LargeIntType;
+import org.apache.doris.nereids.types.MapType;
import org.apache.doris.nereids.types.SmallIntType;
import org.apache.doris.nereids.types.StringType;
+import org.apache.doris.nereids.types.StructField;
+import org.apache.doris.nereids.types.StructType;
import org.apache.doris.nereids.types.TimeStampNsType;
import org.apache.doris.nereids.types.TimeStampTzType;
import org.apache.doris.nereids.types.TimeV2Type;
@@ -755,4 +758,66 @@ public class CastTest {
Assertions.assertTrue(cast.nullable());
}
}
+
+ @Test
+ public void testMayFailOnNonNullInputForScalarCasts() {
+
Assertions.assertFalse(Cast.mayFailOnNonNullInput(IntegerType.INSTANCE,
IntegerType.INSTANCE));
+
Assertions.assertFalse(Cast.mayFailOnNonNullInput(IntegerType.INSTANCE,
BigIntType.INSTANCE));
+
Assertions.assertFalse(Cast.mayFailOnNonNullInput(BooleanType.INSTANCE,
IntegerType.INSTANCE));
+
Assertions.assertFalse(Cast.mayFailOnNonNullInput(LargeIntType.INSTANCE,
FloatType.INSTANCE));
+ Assertions.assertFalse(Cast.mayFailOnNonNullInput(DoubleType.INSTANCE,
BooleanType.INSTANCE));
+ Assertions.assertFalse(Cast.mayFailOnNonNullInput(DateType.INSTANCE,
StringType.INSTANCE));
+
Assertions.assertFalse(Cast.mayFailOnNonNullInput(VarcharType.SYSTEM_DEFAULT,
StringType.INSTANCE));
+
+ Assertions.assertTrue(Cast.mayFailOnNonNullInput(BigIntType.INSTANCE,
IntegerType.INSTANCE));
+ Assertions.assertTrue(Cast.mayFailOnNonNullInput(DoubleType.INSTANCE,
IntegerType.INSTANCE));
+ Assertions.assertTrue(Cast.mayFailOnNonNullInput(StringType.INSTANCE,
IntegerType.INSTANCE));
+
Assertions.assertTrue(Cast.mayFailOnNonNullInput(TimeStampNsType.INSTANCE,
DateTimeV2Type.MAX));
+ Assertions.assertTrue(Cast.mayFailOnNonNullInput(JsonType.INSTANCE,
StringType.INSTANCE));
+ Assertions.assertTrue(Cast.mayFailOnNonNullInput(VariantType.INSTANCE,
StringType.INSTANCE));
+
+ Cast safeCast = new Cast(new SlotReference("slot",
IntegerType.INSTANCE, true), BigIntType.INSTANCE);
+ Assertions.assertTrue(safeCast.nullable());
+ Assertions.assertFalse(safeCast.mayFailOnNonNullInput());
+ TryCast failingTryCast = new TryCast(new SlotReference("slot",
StringType.INSTANCE, false),
+ IntegerType.INSTANCE);
+ Assertions.assertTrue(failingTryCast.mayFailOnNonNullInput());
+ }
+
+ @Test
+ public void testMayFailOnNonNullInputForNestedCasts() {
+ ArrayType integers = ArrayType.of(IntegerType.INSTANCE);
+ ArrayType bigInts = ArrayType.of(BigIntType.INSTANCE);
+ ArrayType strings = ArrayType.of(StringType.INSTANCE);
+ Assertions.assertFalse(Cast.mayFailOnNonNullInput(integers, bigInts));
+ Assertions.assertTrue(Cast.mayFailOnNonNullInput(strings, integers));
+ // The outer ARRAY can stay non-null even when an element cast fails.
+ Assertions.assertFalse(Cast.castNullable(false, strings, integers));
+
+ MapType safeSourceMap = MapType.of(IntegerType.INSTANCE, integers);
+ MapType safeTargetMap = MapType.of(BigIntType.INSTANCE, bigInts);
+ Assertions.assertFalse(Cast.mayFailOnNonNullInput(safeSourceMap,
safeTargetMap));
+ Assertions.assertTrue(Cast.mayFailOnNonNullInput(
+ MapType.of(StringType.INSTANCE, integers), safeTargetMap));
+ Assertions.assertTrue(Cast.mayFailOnNonNullInput(
+ MapType.of(IntegerType.INSTANCE, strings), safeTargetMap));
+
+ StructType safeSourceStruct = new StructType(ImmutableList.of(
+ new StructField("metric", IntegerType.INSTANCE, false, ""),
+ new StructField("attributes", safeSourceMap, true, "")));
+ StructType safeTargetStruct = new StructType(ImmutableList.of(
+ new StructField("metric", BigIntType.INSTANCE, false, ""),
+ new StructField("attributes", safeTargetMap, true, "")));
+ Assertions.assertFalse(Cast.mayFailOnNonNullInput(safeSourceStruct,
safeTargetStruct));
+ StructType failingSourceStruct = new StructType(ImmutableList.of(
+ new StructField("metric", StringType.INSTANCE, false, ""),
+ new StructField("attributes", safeSourceMap, true, "")));
+ Assertions.assertTrue(Cast.mayFailOnNonNullInput(failingSourceStruct,
safeTargetStruct));
+ Assertions.assertTrue(Cast.mayFailOnNonNullInput(safeSourceStruct,
+ new StructType(ImmutableList.of(new StructField("metric",
BigIntType.INSTANCE, false, "")))));
+ Assertions.assertTrue(Cast.mayFailOnNonNullInput(safeTargetStruct,
+ new StructType(ImmutableList.of(
+ new StructField("metric", BigIntType.INSTANCE, false,
""),
+ new StructField("attributes", safeTargetMap, false,
"")))));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
