This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new fe9f4fdd953 [pick](mtmv)Pick some pr to 21 #39998 #40332 #40485
(#40843)
fe9f4fdd953 is described below
commit fe9f4fdd9533d828241adcae19e2d7d8519d1e8b
Author: seawinde <149132972+seawi...@users.noreply.github.com>
AuthorDate: Sun Sep 15 23:48:33 2024 +0800
[pick](mtmv)Pick some pr to 21 #39998 #40332 #40485 (#40843)
## Proposed changes
pr: https://github.com/apache/doris/pull/40485
commitId: 18a374f6
pr: https://github.com/apache/doris/pull/40332
commitId: cd2062ae
pr: https://github.com/apache/doris/pull/39998
commitId: 948fb2ab
---
.../main/java/org/apache/doris/catalog/MTMV.java | 13 +++
.../apache/doris/common/util/PropertyAnalyzer.java | 3 +
.../org/apache/doris/mtmv/MTMVPropertyUtil.java | 15 ++-
.../org/apache/doris/nereids/NereidsPlanner.java | 29 +++---
.../mv/InitMaterializationContextHook.java | 11 ++-
.../exploration/mv/MaterializationContext.java | 2 +-
.../exploration/mv/MaterializedViewUtils.java | 18 +++-
.../trees/plans/commands/info/CreateMTMVInfo.java | 4 +-
.../exploration/mv/MaterializedViewUtilsTest.java | 20 ++++
...ble_date_non_deterministic_function_mtmv.groovy | 6 +-
.../suites/mtmv_p0/test_use_for_rewrite.groovy | 103 +++++++++++++++++++++
11 files changed, 199 insertions(+), 25 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
index b4da3b3ab7e..6287a068857 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
@@ -250,6 +250,19 @@ public class MTMV extends OlapTable {
}
}
+ public boolean isUseForRewrite() {
+ readMvLock();
+ try {
+ if
(!StringUtils.isEmpty(mvProperties.get(PropertyAnalyzer.PROPERTIES_USE_FOR_REWRITE)))
{
+ return
Boolean.valueOf(mvProperties.get(PropertyAnalyzer.PROPERTIES_USE_FOR_REWRITE));
+ }
+ // default is true
+ return true;
+ } finally {
+ readMvUnlock();
+ }
+ }
+
public int getRefreshPartitionNum() {
readMvLock();
try {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index dd39b12a18e..288ee72afd3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -176,6 +176,9 @@ public class PropertyAnalyzer {
public static final String PROPERTIES_ENABLE_NONDETERMINISTIC_FUNCTION =
"enable_nondeterministic_function";
+
+ public static final String PROPERTIES_USE_FOR_REWRITE =
+ "use_for_rewrite";
public static final String PROPERTIES_EXCLUDED_TRIGGER_TABLES =
"excluded_trigger_tables";
public static final String PROPERTIES_REFRESH_PARTITION_NUM =
"refresh_partition_num";
public static final String PROPERTIES_WORKLOAD_GROUP = "workload_group";
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPropertyUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPropertyUtil.java
index 12287183886..dbaccaf9247 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPropertyUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPropertyUtil.java
@@ -38,7 +38,8 @@ public class MTMVPropertyUtil {
PropertyAnalyzer.PROPERTIES_PARTITION_SYNC_LIMIT,
PropertyAnalyzer.PROPERTIES_PARTITION_TIME_UNIT,
PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT,
- PropertyAnalyzer.PROPERTIES_ENABLE_NONDETERMINISTIC_FUNCTION
+ PropertyAnalyzer.PROPERTIES_ENABLE_NONDETERMINISTIC_FUNCTION,
+ PropertyAnalyzer.PROPERTIES_USE_FOR_REWRITE
);
public static void analyzeProperty(String key, String value) {
@@ -65,6 +66,10 @@ public class MTMVPropertyUtil {
analyzePartitionSyncLimit(value);
break;
case PropertyAnalyzer.PROPERTIES_ENABLE_NONDETERMINISTIC_FUNCTION:
+ analyzeBooleanProperty(value,
PropertyAnalyzer.PROPERTIES_ENABLE_NONDETERMINISTIC_FUNCTION);
+ break;
+ case PropertyAnalyzer.PROPERTIES_USE_FOR_REWRITE:
+ analyzeBooleanProperty(value,
PropertyAnalyzer.PROPERTIES_USE_FOR_REWRITE);
break;
default:
throw new AnalysisException("illegal key:" + key);
@@ -138,4 +143,12 @@ public class MTMVPropertyUtil {
}
}
+ private static void analyzeBooleanProperty(String propertyValue, String
propertyName) {
+ if (StringUtils.isEmpty(propertyValue)) {
+ return;
+ }
+ if (!"true".equalsIgnoreCase(propertyValue) &&
!"false".equalsIgnoreCase(propertyValue)) {
+ throw new AnalysisException(String.format("valid property %s
fail", propertyName));
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index 33c1554b1c8..232ec168b3e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -485,6 +485,13 @@ public class NereidsPlanner extends Planner {
public String getExplainString(ExplainOptions explainOptions) {
ExplainLevel explainLevel = getExplainLevel(explainOptions);
String plan = "";
+ String mvSummary = "";
+ if (this.getPhysicalPlan() != null && cascadesContext != null) {
+ mvSummary = cascadesContext.getMaterializationContexts().isEmpty()
? "" :
+ "\n\n========== MATERIALIZATIONS ==========\n"
+ +
MaterializationContext.toSummaryString(cascadesContext.getMaterializationContexts(),
+ this.getPhysicalPlan());
+ }
switch (explainLevel) {
case PARSED_PLAN:
plan = parsedPlan.treeString();
@@ -496,22 +503,16 @@ public class NereidsPlanner extends Planner {
plan = rewrittenPlan.treeString();
break;
case OPTIMIZED_PLAN:
- plan = "cost = " + cost + "\n" + optimizedPlan.treeString();
+ plan = "cost = " + cost + "\n" + optimizedPlan.treeString() +
mvSummary;
break;
case SHAPE_PLAN:
plan = optimizedPlan.shape("");
break;
case MEMO_PLAN:
- StringBuilder materializationStringBuilder = new
StringBuilder();
-
materializationStringBuilder.append("materializationContexts:").append("\n");
- for (MaterializationContext ctx :
cascadesContext.getMaterializationContexts()) {
-
materializationStringBuilder.append("\n").append(ctx).append("\n");
- }
plan = cascadesContext.getMemo().toString()
+ "\n\n========== OPTIMIZED PLAN ==========\n"
+ optimizedPlan.treeString()
- + "\n\n========== MATERIALIZATIONS ==========\n"
- + materializationStringBuilder;
+ + mvSummary;
break;
case ALL_PLAN:
plan = "========== PARSED PLAN "
@@ -525,18 +526,20 @@ public class NereidsPlanner extends Planner {
+ rewrittenPlan.treeString() + "\n\n"
+ "========== OPTIMIZED PLAN "
+
getTimeMetricString(SummaryProfile::getPrettyNereidsOptimizeTime) + "
==========\n"
- + optimizedPlan.treeString();
+ + optimizedPlan.treeString() + "\n\n";
+ plan += mvSummary;
break;
default:
- plan = super.getExplainString(explainOptions)
- +
MaterializationContext.toSummaryString(cascadesContext.getMaterializationContexts(),
- this.getPhysicalPlan());
+ plan = super.getExplainString(explainOptions);
+ plan += mvSummary;
if (statementContext != null) {
if (statementContext.isHasUnknownColStats()) {
- plan += "\n\nStatistics\n planed with unknown column
statistics\n";
+ plan += "\n\n\n========== STATISTICS ==========\n";
+ plan += "planed with unknown column statistics\n";
}
}
}
+
if (statementContext != null) {
if (!statementContext.getHints().isEmpty()) {
String hint =
getHintExplainString(statementContext.getHints());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
index 400e92d4f8c..cacc44a77bf 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
@@ -107,8 +107,8 @@ public class InitMaterializationContextHook implements
PlannerHook {
Set<TableIf> usedTables) {
Set<MTMV> availableMTMVs = getAvailableMTMVs(usedTables,
cascadesContext);
if (availableMTMVs.isEmpty()) {
- LOG.debug(String.format("Enable materialized view rewrite but
availableMTMVs is empty, current queryId "
- + "is %s",
cascadesContext.getConnectContext().getQueryIdentifier()));
+ LOG.debug("Enable materialized view rewrite but availableMTMVs is
empty, current queryId "
+ + "is {}",
cascadesContext.getConnectContext().getQueryIdentifier());
return ImmutableList.of();
}
List<MaterializationContext> asyncMaterializationContext = new
ArrayList<>();
@@ -116,6 +116,13 @@ public class InitMaterializationContextHook implements
PlannerHook {
MTMVCache mtmvCache = null;
try {
mtmvCache =
materializedView.getOrGenerateCache(cascadesContext.getConnectContext());
+ // If mv property use_for_rewrite is set false, should not
partition in
+ // query rewrite by materialized view
+ if (!materializedView.isUseForRewrite()) {
+ LOG.debug("mv doesn't part in query rewrite process
because "
+ + "use_for_rewrite is false, mv is {}",
materializedView.getName());
+ continue;
+ }
if (mtmvCache == null) {
continue;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
index 0f1768e29ce..3c4647e7789 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
@@ -403,7 +403,7 @@ public abstract class MaterializationContext {
}
private static String generateIdentifierName(List<String> qualifiers) {
- return String.join("#", qualifiers);
+ return String.join(".", qualifiers);
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
index 66d4d185ad8..1e6eefbe5b4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
@@ -374,6 +374,20 @@ public class MaterializedViewUtils {
+ "but now is %s",
relation.getClass().getSimpleName()));
return null;
}
+ SlotReference contextPartitionColumn =
getContextPartitionColumn(context);
+ if (contextPartitionColumn == null) {
+ context.addFailReason(String.format("mv partition column is
not from table when relation check, "
+ + "mv partition column is %s",
context.getMvPartitionColumn()));
+ return null;
+ }
+ // Check the table which mv partition column belonged to is same
as the current check relation or not
+ if (!((LogicalCatalogRelation)
relation).getTable().getFullQualifiers().equals(
+
contextPartitionColumn.getTable().map(TableIf::getFullQualifiers).orElse(ImmutableList.of())))
{
+ context.addFailReason(String.format("mv partition column name
is not belonged to current check , "
+ + "table, current table is %s",
+ ((LogicalCatalogRelation)
relation).getTable().getFullQualifiers()));
+ return null;
+ }
LogicalCatalogRelation logicalCatalogRelation =
(LogicalCatalogRelation) relation;
TableIf table = logicalCatalogRelation.getTable();
// if self join, self join can not partition track now, remove the
partition column correspondingly
@@ -402,10 +416,6 @@ public class MaterializedViewUtils {
return null;
}
Set<Column> partitionColumnSet = new
HashSet<>(relatedTable.getPartitionColumns());
- SlotReference contextPartitionColumn =
getContextPartitionColumn(context);
- if (contextPartitionColumn == null) {
- return null;
- }
Column mvReferenceColumn =
contextPartitionColumn.getColumn().get();
Expr definExpr = mvReferenceColumn.getDefineExpr();
if (definExpr instanceof SlotRef) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
index 1ccb8f28b02..dc2f03cc985 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
@@ -359,7 +359,9 @@ public class CreateMTMVInfo {
List<Expression> functionCollectResult =
MaterializedViewUtils.extractNondeterministicFunction(plan);
if (!CollectionUtils.isEmpty(functionCollectResult)) {
throw new AnalysisException(String.format(
- "can not contain invalid expression, the expression is %s",
+ "can not contain nonDeterministic expression, the
expression is %s. "
+ + "Should add 'enable_nondeterministic_function'
= 'true' property "
+ + "when create materialized view if you know the
property real meaning entirely",
functionCollectResult.stream().map(Expression::toString).collect(Collectors.joining(","))));
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java
index 9ac6d7d3a75..2c8b1e1466b 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java
@@ -251,6 +251,26 @@ public class MaterializedViewUtilsTest extends
TestWithFeService {
connectContext.getSessionVariable().setDisableNereidsRules("OLAP_SCAN_PARTITION_PRUNE,PRUNE_EMPTY_PARTITION");
}
+ // Test when join both side are all partition table and partition column
name is same
+ @Test
+ public void joinPartitionNameSameTest() {
+ PlanChecker.from(connectContext)
+ .checkExplain("select t1.upgrade_day, t2.batch_no, count(*) "
+ + "from test2 t2 join test1 t1 on "
+ + "t1.upgrade_day = t2.upgrade_day "
+ + "group by t1.upgrade_day, t2.batch_no;",
+ nereidsPlanner -> {
+ Plan rewrittenPlan =
nereidsPlanner.getRewrittenPlan();
+ RelatedTableInfo relatedTableInfo =
+
MaterializedViewUtils.getRelatedTableInfo("upgrade_day", null,
+ rewrittenPlan,
nereidsPlanner.getCascadesContext());
+ checkRelatedTableInfo(relatedTableInfo,
+ "test1",
+ "upgrade_day",
+ true);
+ });
+ }
+
@Test
public void getRelatedTableInfoWhenAutoPartitionTest() {
PlanChecker.from(connectContext)
diff --git
a/regression-test/suites/mtmv_p0/test_enable_date_non_deterministic_function_mtmv.groovy
b/regression-test/suites/mtmv_p0/test_enable_date_non_deterministic_function_mtmv.groovy
index c085779e707..a8705c6ba9e 100644
---
a/regression-test/suites/mtmv_p0/test_enable_date_non_deterministic_function_mtmv.groovy
+++
b/regression-test/suites/mtmv_p0/test_enable_date_non_deterministic_function_mtmv.groovy
@@ -57,7 +57,7 @@
suite("test_enable_date_non_deterministic_function_mtmv","mtmv") {
Assert.fail();
} catch (Exception e) {
logger.info(e.getMessage())
- assertTrue(e.getMessage().contains("can not contain invalid
expression"));
+ assertTrue(e.getMessage().contains("can not contain nonDeterministic
expression"));
}
sql """drop materialized view if exists ${mvName};"""
@@ -75,7 +75,7 @@
suite("test_enable_date_non_deterministic_function_mtmv","mtmv") {
Assert.fail();
} catch (Exception e) {
logger.info(e.getMessage())
- assertTrue(e.getMessage().contains("can not contain invalid
expression"));
+ assertTrue(e.getMessage().contains("can not contain nonDeterministic
expression"));
}
sql """drop materialized view if exists ${mvName};"""
@@ -128,7 +128,7 @@
suite("test_enable_date_non_deterministic_function_mtmv","mtmv") {
Assert.fail();
} catch (Exception e) {
logger.info(e.getMessage())
- assertTrue(e.getMessage().contains("can not contain invalid
expression"));
+ assertTrue(e.getMessage().contains("can not contain nonDeterministic
expression"));
}
sql """drop table if exists `${tableName}`"""
diff --git a/regression-test/suites/mtmv_p0/test_use_for_rewrite.groovy
b/regression-test/suites/mtmv_p0/test_use_for_rewrite.groovy
new file mode 100644
index 00000000000..d67f6960dfa
--- /dev/null
+++ b/regression-test/suites/mtmv_p0/test_use_for_rewrite.groovy
@@ -0,0 +1,103 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_use_for_rewrite","mtmv") {
+ String suiteName = "test_use_for_rewrite"
+ String tableName = "${suiteName}_table"
+ String mvName = "${suiteName}_mv"
+ String db = context.config.getDbNameByFile(context.file)
+ sql """drop table if exists `${tableName}`"""
+ sql """drop materialized view if exists ${mvName};"""
+
+ sql """
+ CREATE TABLE ${tableName}
+ (
+ k1 TINYINT,
+ k2 INT not null,
+ k3 DATE NOT NULL
+ )
+ COMMENT "my first table"
+ DISTRIBUTED BY HASH(k2) BUCKETS 2
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+ sql """
+ insert into ${tableName} values(1,1, '2024-05-01'),(2,2,
'2024-05-02'),(3,3, '2024-05-03'),
+ (1,1, '2024-05-01'),(2,2, '2024-05-02'), (3,3, '2024-05-03');
+ """
+
+ // when not set use_for_rewrite, should rewrite successfully
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD IMMEDIATE REFRESH AUTO ON MANUAL
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT k1, k2, count(*) from ${tableName} group by k1, k2;
+ """
+
+ waitingMTMVTaskFinished(getJobName(db, mvName))
+ mv_rewrite_success_without_check_chosen("""
+ SELECT k2, count(*) from ${tableName} group by k2;
+ """ ,mvName)
+ sql """drop materialized view if exists ${mvName};"""
+
+
+ // when set use_for_rewrite = true, should rewrite successfully
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD IMMEDIATE REFRESH AUTO ON MANUAL
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1',
+ 'use_for_rewrite' = 'true'
+ )
+ AS
+ SELECT k1, k2, count(*) from ${tableName} group by k1, k2;
+ """
+
+ waitingMTMVTaskFinished(getJobName(db, mvName))
+ mv_rewrite_success_without_check_chosen("""
+ SELECT k2, count(*) from ${tableName} group by k2;
+ """ ,mvName)
+
+
+ // when set use_for_rewrite = false, should not partition in rewritten
+ sql """drop materialized view if exists ${mvName};"""
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD IMMEDIATE REFRESH AUTO ON MANUAL
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1',
+ 'use_for_rewrite' = 'false'
+ )
+ AS
+ SELECT k1, k2, count(*) from ${tableName} group by k1, k2;
+ """
+
+ waitingMTMVTaskFinished(getJobName(db, mvName))
+ mv_not_part_in("""
+ SELECT k2, count(*) from ${tableName} group by k2;
+ """ ,mvName)
+
+ sql """drop materialized view if exists ${mvName};"""
+ sql """drop table if exists `${tableName}`"""
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
