This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new ae320038777 branch-3.0: [fix](nereids) fix generate column need add
cast in stream load #49167 (#50029)
ae320038777 is described below
commit ae32003877783ca921dbb0ecf6b5666414b556e0
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Apr 22 11:07:41 2025 +0800
branch-3.0: [fix](nereids) fix generate column need add cast in stream load
#49167 (#50029)
Cherry-picked from #49167
Co-authored-by: feiniaofeiafei <[email protected]>
---
.../trees/plans/commands/info/CreateTableInfo.java | 25 ++++++++++----
.../objects3.csv | 8 +++++
.../stream_load_and_mysql_load.out | Bin 897 -> 1082 bytes
.../stream_load_and_mysql_load.groovy | 36 +++++++++++++++++++++
4 files changed, 62 insertions(+), 7 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
index 46319a583e8..3de0b7b1cc1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
@@ -58,6 +58,7 @@ import
org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer;
import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
+import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
@@ -71,6 +72,7 @@ import
org.apache.doris.nereids.trees.expressions.functions.scalar.ScalarFunctio
import
org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.coercion.CharacterType;
import org.apache.doris.nereids.util.TypeCoercionUtils;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.qe.ConnectContext;
@@ -940,19 +942,28 @@ public class CreateTableInfo {
Expression boundSlotExpression =
SlotReplacer.INSTANCE.replace(parsedExpression, columnToSlotReference);
Scope scope = new Scope(slots);
ExpressionAnalyzer analyzer = new ExpressionAnalyzer(null, scope,
cascadesContext, false, false);
- Expression expr;
+ Expression expression;
try {
- expr = analyzer.analyze(boundSlotExpression, new
ExpressionRewriteContext(cascadesContext));
+ expression = analyzer.analyze(boundSlotExpression, new
ExpressionRewriteContext(cascadesContext));
} catch (AnalysisException e) {
throw new AnalysisException("In generated column '" +
column.getName() + "', "
+ Utils.convertFirstChar(e.getMessage()));
}
- checkExpressionInGeneratedColumn(expr, column,
nameToColumnDefinition);
- TypeCoercionUtils.checkCanCastTo(expr.getDataType(),
column.getType());
+ checkExpressionInGeneratedColumn(expression, column,
nameToColumnDefinition);
+ TypeCoercionUtils.checkCanCastTo(expression.getDataType(),
column.getType());
ExpressionToExpr translator = new ExpressionToExpr(i,
translateMap);
- Expr e = expr.accept(translator, planTranslatorContext);
- info.get().setExpr(e);
- exprAndnames.add(new GeneratedColumnUtil.ExprAndname(e.clone(),
column.getName()));
+ Expr expr = expression.accept(translator, planTranslatorContext);
+ info.get().setExpr(expr);
+ // Casting slot to its own type is because when loading
data(stream load and other load),
+ // the slots reading from files are string type. So we need to
cast it to its own type to avoid error.
+ Expression expressionForLoad =
expression.rewriteDownShortCircuit(e -> {
+ if (e instanceof SlotReference && !(e.getDataType() instanceof
CharacterType)) {
+ return new Cast(e, e.getDataType());
+ }
+ return e;
+ });
+ Expr exprForLoad = expressionForLoad.accept(translator,
planTranslatorContext);
+ exprAndnames.add(new
GeneratedColumnUtil.ExprAndname(exprForLoad.clone(), column.getName()));
}
// for alter drop column
diff --git
a/regression-test/data/ddl_p0/test_create_table_generated_column/objects3.csv
b/regression-test/data/ddl_p0/test_create_table_generated_column/objects3.csv
new file mode 100644
index 00000000000..b7186df2659
--- /dev/null
+++
b/regression-test/data/ddl_p0/test_create_table_generated_column/objects3.csv
@@ -0,0 +1,8 @@
+5113756983984045899
+5113756983984045898
+3423432435553321
+55545645564565
+5113756983984045895
+4663423423432
+5113756983984045894
+5113756983984045897
\ No newline at end of file
diff --git
a/regression-test/data/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.out
b/regression-test/data/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.out
index 59c5006a2d2..83e2ad75666 100644
Binary files
a/regression-test/data/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.out
and
b/regression-test/data/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.out
differ
diff --git
a/regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy
b/regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy
index a8d4025b9b8..81ba556e0ea 100644
---
a/regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy
+++
b/regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy
@@ -141,4 +141,40 @@ suite("test_generated_column_stream_mysql_load") {
"""
sql "sync"
qt_specify_value_for_gencol "select * from
test_gen_col_common_steam_mysql_load order by 1,2,3"
+
+ sql """
+ drop table if exists objects3;
+ """
+ sql """
+ CREATE TABLE `objects3` (
+ `OBJECTIDHASH` tinyint AS ((`OBJECTID` % 10)) NULL,
+ `OBJECTID` bigint NOT NULL,
+ ) ENGINE=OLAP
+ UNIQUE KEY(`OBJECTIDHASH`, `OBJECTID`)
+ DISTRIBUTED BY HASH( `OBJECTIDHASH`) BUCKETS 128
+ PROPERTIES (
+ "file_cache_ttl_seconds" = "0",
+ "is_being_synced" = "false",
+ "storage_medium" = "hdd",
+ "storage_format" = "V2",
+ "inverted_index_storage_format" = "V2",
+ "compression" = "ZSTD",
+ "enable_unique_key_merge_on_write" = "true",
+ "light_schema_change" = "true",
+ "disable_auto_compaction" = "false",
+ "enable_single_replica_compaction" = "false",
+ "group_commit_interval_ms" = "5000",
+ "group_commit_data_bytes" = "134217728",
+ "enable_mow_light_delete" = "false"
+ );
+ """
+ streamLoad {
+ table 'objects3'
+ set 'column_separator', ','
+ file 'objects3.csv'
+ set 'columns', 'OBJECTID'
+ time 10000 // limit inflight 10s
+ }
+ sql "sync"
+ qt_test_load_cast "select * from objects3 order by OBJECTIDHASH,OBJECTID"
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
