This is an automated email from the ASF dual-hosted git repository.
zhangstar333 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new c0780e60c7c [Fix](auto-partition) fix auto partition expr change
unexpected (#36345)
c0780e60c7c is described below
commit c0780e60c7c31e1bc5211664093e198523fbfd26
Author: zclllyybb <zhaochan...@selectdb.com>
AuthorDate: Tue Jun 18 22:38:18 2024 +0800
[Fix](auto-partition) fix auto partition expr change unexpected (#36345)
## Proposed changes
Issue Number: close #xxx
there's a caller used `getPartitionExprs` and modified the result. the
result is a reference to a part of metadata. If when we create the
table, the number of editlog didn't reach the limit to generate image
file, when it reaches, the image will get the wrong metadata of
`partitionExprs` from memory which was modified.
currently we don't know why, but this root cause has been confirmed by:
```java
private void analyzerPartitionExpr(Analyzer analyzer, PartitionInfo
partitionInfo) throws AnalysisException {
ArrayList<Expr> exprs = partitionInfo.getPartitionExprs();
for (Expr e : exprs) {
LOG.info("beforeB: {}", e.toSql());
e.analyze(analyzer);
LOG.info("afterB: {}", e.toSql());
}
}
```
```log
2024-06-18 21:49:07,449 INFO (mysql-nio-pool-154|695)
[OlapScanNode.analyzerPartitionExpr():1561] beforeB: date_trunc(Carding_Day,
'MONTH')
2024-06-18 21:49:07,449 INFO (mysql-nio-pool-154|695)
[OlapScanNode.analyzerPartitionExpr():1563] afterB: date_trunc(`a`.null,
'MONTH')
```
---
.../org/apache/doris/catalog/PartitionInfo.java | 3 ++-
.../doris/analysis/PartitionPruneTestBase.java | 4 ++-
.../doris/analysis/RangePartitionPruneTest.java | 3 ---
.../test_date_function_prune.groovy | 31 ++++++++++++++++++++++
4 files changed, 36 insertions(+), 5 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
index 6408ba1cb37..5077649de98 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
@@ -253,8 +253,9 @@ public class PartitionInfo implements Writable {
return isAutoCreatePartitions;
}
+ // forbid change metadata.
public ArrayList<Expr> getPartitionExprs() {
- return this.partitionExprs;
+ return Expr.cloneList(this.partitionExprs);
}
public void checkPartitionItemListsMatch(List<PartitionItem> list1,
List<PartitionItem> list2) throws DdlException {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/PartitionPruneTestBase.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/PartitionPruneTestBase.java
index 162a81ccb88..8a9d9787731 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/analysis/PartitionPruneTestBase.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/analysis/PartitionPruneTestBase.java
@@ -34,7 +34,9 @@ public abstract class PartitionPruneTestBase extends
TestWithFeService {
}
private void assertExplainContains(String sql, String subString) throws
Exception {
- Assert.assertTrue(String.format("sql=%s, expectResult=%s", sql,
subString),
+ Assert.assertTrue(
+ String.format("sql=%s, expectResult=%s, but got %s", sql,
subString,
+ getSQLPlanOrErrorMsg("explain " + sql)),
getSQLPlanOrErrorMsg("explain " + sql).contains(subString));
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/RangePartitionPruneTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/RangePartitionPruneTest.java
index 4cd7f8d2049..7bce2526df0 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/analysis/RangePartitionPruneTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/analysis/RangePartitionPruneTest.java
@@ -206,9 +206,6 @@ public class RangePartitionPruneTest extends
PartitionPruneTestBase {
"partitions=6/8");
addCase("select /*+ SET_VAR(enable_nereids_planner=false) */ * from
test.test_to_date_trunc where event_day= \"2023-08-07 11:00:00\" ",
"partitions=1/2");
- addCase("select /*+ SET_VAR(enable_nereids_planner=false) */ * from
test.test_to_date_trunc where date_trunc(event_day, \"day\")= \"2023-08-07
11:00:00\" ",
- "partitions=1/2");
-
}
diff --git
a/regression-test/suites/nereids_rules_p0/partition_prune/test_date_function_prune.groovy
b/regression-test/suites/nereids_rules_p0/partition_prune/test_date_function_prune.groovy
index c126206eba0..c6f122e3c87 100644
---
a/regression-test/suites/nereids_rules_p0/partition_prune/test_date_function_prune.groovy
+++
b/regression-test/suites/nereids_rules_p0/partition_prune/test_date_function_prune.groovy
@@ -91,4 +91,35 @@ suite("test_date_function_prune") {
sql "select * from dp where date_time >
str_to_date('2020-01-02','%Y-%m-%d')"
contains("partitions=2/3 (p2,p3)")
}
+
+ sql "drop table if exists test_to_date_trunc"
+ sql """
+ CREATE TABLE test_to_date_trunc(
+ event_day DATETIME NOT NULL
+ )
+ DUPLICATE KEY(event_day)
+ AUTO PARTITION BY range (date_trunc(event_day, "day")) (
+ PARTITION `p20230807` values [(20230807 ), (20230808 )),
+ PARTITION `p20020106` values [(20020106 ), (20020107 ))
+ )
+ DISTRIBUTED BY HASH(event_day) BUCKETS 4
+ PROPERTIES("replication_num" = "1");
+ """
+ explain {
+ sql """ select /*+ SET_VAR(enable_nereids_planner=false) */ * from
test_to_date_trunc where date_trunc(event_day, "day")= "2023-08-07 11:00:00" """
+ contains("partitions=0/2")
+ }
+ explain {
+ sql """ select * from test_to_date_trunc where date_trunc(event_day,
"day")= "2023-08-07 11:00:00" """
+ contains("VEMPTYSET")
+ }
+ sql """ insert into test_to_date_trunc values ("20230807000000"); """
+ explain {
+ sql """ select /*+ SET_VAR(enable_nereids_planner=false) */ * from
test_to_date_trunc where date_trunc(event_day, "day")= "2023-08-07 11:00:00" """
+ contains("partitions=1/2 (p20230807)")
+ }
+ explain {
+ sql """ select * from test_to_date_trunc where date_trunc(event_day,
"day")= "2023-08-07 11:00:00" """
+ contains("partitions=1/2 (p20230807)")
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
