This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0afc483c425 [fix](nereids) fix generate column need add cast in stream
load (#49167)
0afc483c425 is described below
commit 0afc483c4258b7cced2518495f2fa79d08c7d46c
Author: feiniaofeiafei <moail...@selectdb.com>
AuthorDate: Mon Apr 14 17:05:55 2025 +0800
[fix](nereids) fix generate column need add cast in stream load (#49167)
### What problem does this PR solve?
Related PR: #35284
Problem Summary:
Generate column need add cast(casting slot to its own type) in stream
load, because when loading data(stream load and other load), the slots
reading from files are string type. So we need to cast it to its own
type to avoid error.
---
.../trees/plans/commands/info/CreateTableInfo.java | 25 ++++++++++----
.../objects3.csv | 8 +++++
.../stream_load_and_mysql_load.out | Bin 897 -> 1082 bytes
.../stream_load_and_mysql_load.groovy | 36 +++++++++++++++++++++
4 files changed, 62 insertions(+), 7 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
index 8d11d3f9498..48abe07c24a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
@@ -59,6 +59,7 @@ import
org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.rules.analysis.ExpressionAnalyzer;
import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
+import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
@@ -72,6 +73,7 @@ import
org.apache.doris.nereids.trees.expressions.functions.scalar.ScalarFunctio
import
org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.coercion.CharacterType;
import org.apache.doris.nereids.util.TypeCoercionUtils;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.qe.ConnectContext;
@@ -1016,19 +1018,28 @@ public class CreateTableInfo {
Expression boundSlotExpression =
SlotReplacer.INSTANCE.replace(parsedExpression, columnToSlotReference);
Scope scope = new Scope(slots);
ExpressionAnalyzer analyzer = new ExpressionAnalyzer(null, scope,
cascadesContext, false, false);
- Expression expr;
+ Expression expression;
try {
- expr = analyzer.analyze(boundSlotExpression, new
ExpressionRewriteContext(cascadesContext));
+ expression = analyzer.analyze(boundSlotExpression, new
ExpressionRewriteContext(cascadesContext));
} catch (AnalysisException e) {
throw new AnalysisException("In generated column '" +
column.getName() + "', "
+ Utils.convertFirstChar(e.getMessage()));
}
- checkExpressionInGeneratedColumn(expr, column,
nameToColumnDefinition);
- TypeCoercionUtils.checkCanCastTo(expr.getDataType(),
column.getType());
+ checkExpressionInGeneratedColumn(expression, column,
nameToColumnDefinition);
+ TypeCoercionUtils.checkCanCastTo(expression.getDataType(),
column.getType());
ExpressionToExpr translator = new ExpressionToExpr(i,
translateMap);
- Expr e = expr.accept(translator, planTranslatorContext);
- info.get().setExpr(e);
- exprAndnames.add(new GeneratedColumnUtil.ExprAndname(e.clone(),
column.getName()));
+ Expr expr = expression.accept(translator, planTranslatorContext);
+ info.get().setExpr(expr);
+ // Casting slot to its own type is because when loading
data(stream load and other load),
+ // the slots reading from files are string type. So we need to
cast it to its own type to avoid error.
+ Expression expressionForLoad =
expression.rewriteDownShortCircuit(e -> {
+ if (e instanceof SlotReference && !(e.getDataType() instanceof
CharacterType)) {
+ return new Cast(e, e.getDataType());
+ }
+ return e;
+ });
+ Expr exprForLoad = expressionForLoad.accept(translator,
planTranslatorContext);
+ exprAndnames.add(new
GeneratedColumnUtil.ExprAndname(exprForLoad.clone(), column.getName()));
}
// for alter drop column
diff --git
a/regression-test/data/ddl_p0/test_create_table_generated_column/objects3.csv
b/regression-test/data/ddl_p0/test_create_table_generated_column/objects3.csv
new file mode 100644
index 00000000000..b7186df2659
--- /dev/null
+++
b/regression-test/data/ddl_p0/test_create_table_generated_column/objects3.csv
@@ -0,0 +1,8 @@
+5113756983984045899
+5113756983984045898
+3423432435553321
+55545645564565
+5113756983984045895
+4663423423432
+5113756983984045894
+5113756983984045897
\ No newline at end of file
diff --git
a/regression-test/data/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.out
b/regression-test/data/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.out
index 59c5006a2d2..83e2ad75666 100644
Binary files
a/regression-test/data/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.out
and
b/regression-test/data/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.out
differ
diff --git
a/regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy
b/regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy
index a8d4025b9b8..81ba556e0ea 100644
---
a/regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy
+++
b/regression-test/suites/ddl_p0/test_create_table_generated_column/stream_load_and_mysql_load.groovy
@@ -141,4 +141,40 @@ suite("test_generated_column_stream_mysql_load") {
"""
sql "sync"
qt_specify_value_for_gencol "select * from
test_gen_col_common_steam_mysql_load order by 1,2,3"
+
+ sql """
+ drop table if exists objects3;
+ """
+ sql """
+ CREATE TABLE `objects3` (
+ `OBJECTIDHASH` tinyint AS ((`OBJECTID` % 10)) NULL,
+ `OBJECTID` bigint NOT NULL,
+ ) ENGINE=OLAP
+ UNIQUE KEY(`OBJECTIDHASH`, `OBJECTID`)
+ DISTRIBUTED BY HASH( `OBJECTIDHASH`) BUCKETS 128
+ PROPERTIES (
+ "file_cache_ttl_seconds" = "0",
+ "is_being_synced" = "false",
+ "storage_medium" = "hdd",
+ "storage_format" = "V2",
+ "inverted_index_storage_format" = "V2",
+ "compression" = "ZSTD",
+ "enable_unique_key_merge_on_write" = "true",
+ "light_schema_change" = "true",
+ "disable_auto_compaction" = "false",
+ "enable_single_replica_compaction" = "false",
+ "group_commit_interval_ms" = "5000",
+ "group_commit_data_bytes" = "134217728",
+ "enable_mow_light_delete" = "false"
+ );
+ """
+ streamLoad {
+ table 'objects3'
+ set 'column_separator', ','
+ file 'objects3.csv'
+ set 'columns', 'OBJECTID'
+ time 10000 // limit inflight 10s
+ }
+ sql "sync"
+ qt_test_load_cast "select * from objects3 order by OBJECTIDHASH,OBJECTID"
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
