This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 9c6a6893d97 [fix](mtmv) Fix npe when the id of base table in mv is
lager than Integer.MAX_VALUE (#35294) (#35384)
9c6a6893d97 is described below
commit 9c6a6893d9753d905f58faa714c0270c0f7be8ef
Author: seawinde <149132972+seawi...@users.noreply.github.com>
AuthorDate: Fri May 24 23:27:08 2024 +0800
[fix](mtmv) Fix npe when the id of base table in mv is lager than
Integer.MAX_VALUE (#35294) (#35384)
This brought by #34768
---
.../exploration/mv/MaterializationContext.java | 8 +--
.../nereids/rules/exploration/mv/StructInfo.java | 22 +++----
.../doris/nereids/memo/MvTableIdIsLongTest.java | 77 ++++++++++++++++++++++
.../apache/doris/nereids/sqltest/SqlTestBase.java | 7 ++
4 files changed, 99 insertions(+), 15 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
index ebb231c3c68..13e8c4456c0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
@@ -113,12 +113,12 @@ public abstract class MaterializationContext {
viewStructInfos = MaterializedViewUtils.extractStructInfo(mvPlan,
cascadesContext, new BitSet());
if (viewStructInfos.size() > 1) {
// view struct info should only have one, log error and use
the first struct info
- LOG.warn(String.format("view strut info is more than one,
materialization name is %s, mv plan is %s",
- getMaterializationQualifier(),
getMvPlan().treeString()));
+ LOG.warn(String.format("view strut info is more than one, mv
scan plan is %s, mv plan is %s",
+ mvScanPlan.treeString(), mvPlan.treeString()));
}
} catch (Exception exception) {
- LOG.warn(String.format("construct mv struct info fail,
materialization name is %s, mv plan is %s",
- getMaterializationQualifier(), getMvPlan().treeString()),
exception);
+ LOG.warn(String.format("construct mv struct info fail, mv scan
plan is %s, mv plan is %s",
+ mvScanPlan.treeString(), mvPlan.treeString()), exception);
this.available = false;
this.structInfo = null;
return;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
index fec66265880..955431d5cec 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
@@ -119,7 +119,7 @@ public class StructInfo {
@Nullable Predicates predicates,
Map<ExpressionPosition, Map<Expression, Expression>>
shuttledExpressionsToExpressionsMap,
Map<ExprId, Expression> namedExprIdAndExprMapping,
- BitSet talbeIdSet) {
+ BitSet tableIdSet) {
this.originalPlan = originalPlan;
this.originalPlanId = originalPlanId;
this.hyperGraph = hyperGraph;
@@ -128,7 +128,7 @@ public class StructInfo {
this.topPlan = topPlan;
this.bottomPlan = bottomPlan;
this.relations = relations;
- this.tableBitSet = talbeIdSet;
+ this.tableBitSet = tableIdSet;
this.relationIdStructInfoNodeMap = relationIdStructInfoNodeMap;
this.predicates = predicates;
if (predicates == null) {
@@ -159,17 +159,19 @@ public class StructInfo {
Map<ExpressionPosition, Map<Expression, Expression>>
shuttledExpressionsToExpressionsMap,
Map<ExprId, Expression> namedExprIdAndExprMapping,
List<CatalogRelation> relations,
- Map<RelationId, StructInfoNode> relationIdStructInfoNodeMap) {
+ Map<RelationId, StructInfoNode> relationIdStructInfoNodeMap,
+ BitSet hyperTableBitSet,
+ CascadesContext cascadesContext) {
// Collect relations from hyper graph which in the bottom plan firstly
- BitSet hyperTableBitSet = new BitSet();
hyperGraph.getNodes().forEach(node -> {
// plan relation collector and set to map
Plan nodePlan = node.getPlan();
List<CatalogRelation> nodeRelations = new ArrayList<>();
nodePlan.accept(RELATION_COLLECTOR, nodeRelations);
relations.addAll(nodeRelations);
- nodeRelations.forEach(relation -> hyperTableBitSet.set((int)
relation.getTable().getId()));
+ nodeRelations.forEach(relation -> hyperTableBitSet.set(
+
cascadesContext.getStatementContext().getTableId(relation.getTable()).asInt()));
// every node should only have one relation, this is for
LogicalCompatibilityContext
if (!nodeRelations.isEmpty()) {
relationIdStructInfoNodeMap.put(nodeRelations.get(0).getRelationId(),
(StructInfoNode) node);
@@ -308,15 +310,13 @@ public class StructInfo {
Map<ExpressionPosition, Map<Expression, Expression>>
shuttledHashConjunctsToConjunctsMap =
new LinkedHashMap<>();
Map<ExprId, Expression> namedExprIdAndExprMapping = new
LinkedHashMap<>();
+ BitSet tableBitSet = new BitSet();
boolean valid = collectStructInfoFromGraph(hyperGraph, topPlan,
shuttledHashConjunctsToConjunctsMap,
namedExprIdAndExprMapping,
relationList,
- relationIdStructInfoNodeMap);
- // Get mapped table id in relation and set
- BitSet tableBitSet = new BitSet();
- for (CatalogRelation relation : relationList) {
-
tableBitSet.set(cascadesContext.getStatementContext().getTableId(relation.getTable()).asInt());
- }
+ relationIdStructInfoNodeMap,
+ tableBitSet,
+ cascadesContext);
return new StructInfo(originalPlan, originalPlanId, hyperGraph, valid,
topPlan, bottomPlan,
relationList, relationIdStructInfoNodeMap, null,
shuttledHashConjunctsToConjunctsMap,
namedExprIdAndExprMapping, tableBitSet);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MvTableIdIsLongTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MvTableIdIsLongTest.java
new file mode 100644
index 00000000000..75aa7718c00
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MvTableIdIsLongTest.java
@@ -0,0 +1,77 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.memo;
+
+import org.apache.doris.catalog.MTMV;
+import org.apache.doris.mtmv.MTMVRelationManager;
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.sqltest.SqlTestBase;
+import org.apache.doris.nereids.util.PlanChecker;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.BitSet;
+
+/**
+ * Test mv rewrite when base table id is lager then integer
+ */
+public class MvTableIdIsLongTest extends SqlTestBase {
+
+ @Test
+ void testMvRewriteWhenBaseTableIdIsLong() throws Exception {
+
connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION");
+ BitSet disableNereidsRules =
connectContext.getSessionVariable().getDisableNereidsRules();
+ new MockUp<SessionVariable>() {
+ @Mock
+ public BitSet getDisableNereidsRules() {
+ return disableNereidsRules;
+ }
+ };
+ new MockUp<MTMVRelationManager>() {
+ @Mock
+ public boolean isMVPartitionValid(MTMV mtmv, ConnectContext ctx) {
+ return true;
+ }
+ };
+ connectContext.getSessionVariable().enableMaterializedViewRewrite =
true;
+ connectContext.getSessionVariable().enableMaterializedViewNestRewrite
= true;
+ createMvByNereids("create materialized view mv1 BUILD IMMEDIATE
REFRESH COMPLETE ON MANUAL\n"
+ + " DISTRIBUTED BY RANDOM BUCKETS 1\n"
+ + " PROPERTIES ('replication_num' = '1') \n"
+ + " as select T1.id from T1 inner join T2 "
+ + "on T1.id = T2.id;");
+ CascadesContext c1 = createCascadesContext(
+ "select T1.id from T1 inner join T2 "
+ + "on T1.id = T2.id "
+ + "inner join T3 on T1.id = T3.id",
+ connectContext
+ );
+ PlanChecker.from(c1)
+ .analyze()
+ .rewrite()
+ .optimize()
+ .printlnBestPlanTree();
+ Assertions.assertTrue(c1.getMemo().toString().contains("mv1"));
+ dropMvByNereids("drop materialized view mv1");
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java
index e879f461012..887ae3e2921 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java
@@ -17,6 +17,8 @@
package org.apache.doris.nereids.sqltest;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.MetaIdGenerator.IdGeneratorBuffer;
import org.apache.doris.nereids.CascadesContext;
import
org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext;
import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils;
@@ -36,6 +38,11 @@ public abstract class SqlTestBase extends TestWithFeService
implements MemoPatte
createDatabase("test");
connectContext.setDatabase("test");
+ // make table id is larger than Integer.MAX_VALUE
+ IdGeneratorBuffer idGeneratorBuffer =
+ Env.getCurrentEnv().getIdGeneratorBuffer(Integer.MAX_VALUE +
10L);
+ idGeneratorBuffer.getNextId();
+
createTables(
"CREATE TABLE IF NOT EXISTS T0 (\n"
+ " id bigint,\n"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
