This is an automated email from the ASF dual-hosted git repository. huajianlan pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push: new 3eb950119a4 [fix](Nereids) lock table when generate distribute plan (#38950) 3eb950119a4 is described below commit 3eb950119a474d0672be021dfaaaeeb3096cdc93 Author: 924060929 <924060...@qq.com> AuthorDate: Wed Aug 7 14:50:31 2024 +0800 [fix](Nereids) lock table when generate distribute plan (#38950) We should lock table when generate distribute plan, because insert overwrite by async materialized view will drop partitions parallel, and query thread will throw exception: ``` java.lang.RuntimeException: Cannot invoke "org.apache.doris.catalog.Partition.getBaseIndex()" because "partition" is null at org.apache.doris.nereids.util.Utils.execWithUncheckedException(Utils.java:76) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator.translatePlan(PhysicalPlanTranslator.java:278) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.nereids.NereidsPlanner.splitFragments(NereidsPlanner.java:341) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.nereids.NereidsPlanner.distribute(NereidsPlanner.java:400) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.nereids.NereidsPlanner.plan(NereidsPlanner.java:147) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.qe.StmtExecutor.executeByNereids(StmtExecutor.java:796) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:605) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.qe.StmtExecutor.queryRetry(StmtExecutor.java:558) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:548) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.qe.ConnectProcessor.executeQuery(ConnectProcessor.java:385) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.qe.ConnectProcessor.handleQuery(ConnectProcessor.java:237) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.qe.MysqlConnectProcessor.handleQuery(MysqlConnectProcessor.java:260) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.qe.MysqlConnectProcessor.dispatch(MysqlConnectProcessor.java:288) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.qe.MysqlConnectProcessor.processOnce(MysqlConnectProcessor.java:342) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.mysql.ReadListener.lambda$handleEvent$0(ReadListener.java:52) ~[doris-fe.jar:1.2-SNAPSHOT] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:833) ~[?:?] Caused by: java.lang.NullPointerException: Cannot invoke "org.apache.doris.catalog.Partition.getBaseIndex()" because "partition" is null at org.apache.doris.planner.OlapScanNode.mockRowCountInStatistic(OlapScanNode.java:589) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.planner.OlapScanNode.finalizeForNereids(OlapScanNode.java:1733) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.nereids.util.Utils.execWithUncheckedException(Utils.java:74) ~[doris-fe.jar:1.2-SNAPSHOT] ... 17 more 2024-07-29 00:46:17,608 WARN (mysql-nio-pool-114|201) Analyze failed. stmt[210035, 49d3041004ba4b6a-b07fe4491d03c5de] org.apache.doris.common.NereidsException: errCode = 2, detailMessage = Cannot invoke "org.apache.doris.catalog.Partition.getBaseIndex()" because "partition" is null at org.apache.doris.qe.StmtExecutor.executeByNereids(StmtExecutor.java:803) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:605) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.qe.StmtExecutor.queryRetry(StmtExecutor.java:558) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:548) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.qe.ConnectProcessor.executeQuery(ConnectProcessor.java:385) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.qe.ConnectProcessor.handleQuery(ConnectProcessor.java:237) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.qe.MysqlConnectProcessor.handleQuery(MysqlConnectProcessor.java:260) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.qe.MysqlConnectProcessor.dispatch(MysqlConnectProcessor.java:288) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.qe.MysqlConnectProcessor.processOnce(MysqlConnectProcessor.java:342) ~[doris-fe.jar:1.2-SNAPSHOT] at org.apache.doris.mysql.ReadListener.lambda$handleEvent$0(ReadListener.java:52) ~[doris-fe.jar:1.2-SNAPSHOT] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:833) ~[?:?] ``` this exception is too hard to reproduce, so I can not write a test case --- .../main/java/org/apache/doris/mtmv/MTMVCache.java | 2 +- .../java/org/apache/doris/mtmv/MTMVPlanUtil.java | 2 +- .../org/apache/doris/nereids/NereidsPlanner.java | 175 ++++++++++++--------- .../doris/nereids/minidump/MinidumpUtils.java | 2 +- .../exploration/mv/MaterializedViewUtils.java | 2 +- .../trees/plans/commands/AddConstraintCommand.java | 2 +- .../trees/plans/commands/CreateTableCommand.java | 2 +- .../plans/commands/DropConstraintCommand.java | 2 +- .../trees/plans/commands/info/AlterViewInfo.java | 2 +- .../trees/plans/commands/info/CreateMTMVInfo.java | 4 +- .../trees/plans/commands/info/CreateViewInfo.java | 2 +- .../commands/info/MTMVPartitionDefinition.java | 2 +- .../datasource/hive/HiveDDLAndDMLPlanTest.java | 2 +- .../apache/doris/nereids/UnsupportedTypeTest.java | 2 +- .../doris/nereids/preprocess/SelectHintTest.java | 2 +- .../nereids/rules/analysis/AnalyzeCTETest.java | 2 +- .../rules/analysis/AnalyzeSubQueryTest.java | 2 +- .../rules/analysis/AnalyzeWhereSubqueryTest.java | 2 +- .../PushDownExpressionsInHashConditionTest.java | 2 +- .../doris/nereids/trees/expressions/ViewTest.java | 2 +- .../trees/plans/ExplainInsertCommandTest.java | 2 +- .../doris/nereids/trees/plans/OutFileTest.java | 2 +- .../org/apache/doris/nereids/util/PlanChecker.java | 6 +- .../apache/doris/nereids/util/ReadLockTest.java | 10 +- .../org/apache/doris/qe/OlapQueryCacheTest.java | 2 +- 25 files changed, 129 insertions(+), 108 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java index a6403b5892e..aceb453c2c3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java @@ -89,7 +89,7 @@ public class MTMVCache { } // Can not convert to table sink, because use the same column from different table when self join // the out slot is wrong - planner.plan(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN); + planner.planWithLock(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN); Plan originPlan = planner.getCascadesContext().getRewritePlan(); // Eliminate result sink because sink operator is useless in query rewrite by materialized view // and the top sort can also be removed diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java index e1ad35aa758..cf80e58eb7e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java @@ -153,6 +153,6 @@ public class MTMVPlanUtil { StatementBase parsedStmt = statements.get(0); LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan(); NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); - return planner.plan(logicalPlan, PhysicalProperties.ANY, ExplainLevel.NONE); + return planner.planWithLock(logicalPlan, PhysicalProperties.ANY, ExplainLevel.NONE); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index f307f14d78f..78d2025f40d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -78,6 +78,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.function.Consumer; import java.util.function.Function; /** @@ -127,23 +128,22 @@ public class NereidsPlanner extends Planner { PhysicalProperties requireProperties = buildInitRequireProperties(); statementContext.getStopwatch().reset().start(); - Plan resultPlan = null; try { boolean showPlanProcess = showPlanProcess(queryStmt.getExplainOptions()); - resultPlan = plan(parsedPlan, requireProperties, explainLevel, showPlanProcess); + planWithLock(parsedPlan, requireProperties, explainLevel, showPlanProcess, plan -> { + setOptimizedPlan(plan); + if (plan instanceof PhysicalPlan) { + physicalPlan = (PhysicalPlan) plan; + distribute(physicalPlan, explainLevel); + } + }); } finally { statementContext.getStopwatch().stop(); } - setOptimizedPlan(resultPlan); - - if (resultPlan instanceof PhysicalPlan) { - physicalPlan = (PhysicalPlan) resultPlan; - distribute(physicalPlan, explainLevel); - } } @VisibleForTesting - public void plan(StatementBase queryStmt) { + public void planWithLock(StatementBase queryStmt) { try { plan(queryStmt, statementContext.getConnectContext().getSessionVariable().toThrift()); } catch (Exception e) { @@ -151,12 +151,18 @@ public class NereidsPlanner extends Planner { } } - public PhysicalPlan plan(LogicalPlan plan, PhysicalProperties outputProperties) { - return (PhysicalPlan) plan(plan, outputProperties, ExplainLevel.NONE, false); + public PhysicalPlan planWithLock(LogicalPlan plan, PhysicalProperties outputProperties) { + return (PhysicalPlan) planWithLock(plan, outputProperties, ExplainLevel.NONE, false); } - public Plan plan(LogicalPlan plan, PhysicalProperties requireProperties, ExplainLevel explainLevel) { - return plan(plan, requireProperties, explainLevel, false); + public Plan planWithLock(LogicalPlan plan, PhysicalProperties requireProperties, ExplainLevel explainLevel) { + return planWithLock(plan, requireProperties, explainLevel, false); + } + + public Plan planWithLock(LogicalPlan plan, PhysicalProperties requireProperties, + ExplainLevel explainLevel, boolean showPlanProcess) { + Consumer<Plan> noCallback = p -> {}; + return planWithLock(plan, requireProperties, explainLevel, showPlanProcess, noCallback); } /** @@ -164,11 +170,14 @@ public class NereidsPlanner extends Planner { * * @param plan wait for plan * @param requireProperties request physical properties constraints + * @param showPlanProcess is record plan process to CascadesContext + * @param lockCallback this callback function will invoke the table lock * @return plan generated by this planner * @throws AnalysisException throw exception if failed in ant stage */ - public Plan plan(LogicalPlan plan, PhysicalProperties requireProperties, - ExplainLevel explainLevel, boolean showPlanProcess) { + public Plan planWithLock(LogicalPlan plan, PhysicalProperties requireProperties, + ExplainLevel explainLevel, boolean showPlanProcess, + Consumer<Plan> lockCallback) { try { if (plan instanceof LogicalSqlCache) { rewrittenPlan = analyzedPlan = plan; @@ -195,75 +204,83 @@ public class NereidsPlanner extends Planner { initCascadesContext(plan, requireProperties); try (Lock lock = new Lock(plan, cascadesContext)) { - // resolve column, table and function - // analyze this query - analyze(showAnalyzeProcess(explainLevel, showPlanProcess)); - // minidump of input must be serialized first, this process ensure minidump string not null - try { - MinidumpUtils.serializeInputsToDumpFile(plan, cascadesContext.getTables()); - } catch (IOException e) { - throw new RuntimeException(e); - } + Plan resultPlan = planWithoutLock(plan, explainLevel, showPlanProcess, requireProperties); + lockCallback.accept(resultPlan); + return resultPlan; + } + } finally { + statementContext.releasePlannerResources(); + } + } - if (statementContext.getConnectContext().getExecutor() != null) { - statementContext.getConnectContext().getExecutor().getSummaryProfile().setQueryAnalysisFinishTime(); - statementContext.getConnectContext().getExecutor().getSummaryProfile().setNereidsAnalysisTime(); - } + private Plan planWithoutLock( + LogicalPlan plan, ExplainLevel explainLevel, + boolean showPlanProcess, PhysicalProperties requireProperties) { + // resolve column, table and function + // analyze this query + analyze(showAnalyzeProcess(explainLevel, showPlanProcess)); + // minidump of input must be serialized first, this process ensure minidump string not null + try { + MinidumpUtils.serializeInputsToDumpFile(plan, cascadesContext.getTables()); + } catch (IOException e) { + throw new RuntimeException(e); + } - if (explainLevel == ExplainLevel.ANALYZED_PLAN || explainLevel == ExplainLevel.ALL_PLAN) { - analyzedPlan = cascadesContext.getRewritePlan(); - if (explainLevel == ExplainLevel.ANALYZED_PLAN) { - return analyzedPlan; - } - } + if (statementContext.getConnectContext().getExecutor() != null) { + statementContext.getConnectContext().getExecutor().getSummaryProfile().setQueryAnalysisFinishTime(); + statementContext.getConnectContext().getExecutor().getSummaryProfile().setNereidsAnalysisTime(); + } - // rule-based optimize - rewrite(showRewriteProcess(explainLevel, showPlanProcess)); - if (statementContext.getConnectContext().getExecutor() != null) { - statementContext.getConnectContext().getExecutor().getSummaryProfile().setNereidsRewriteTime(); - } + if (explainLevel == ExplainLevel.ANALYZED_PLAN || explainLevel == ExplainLevel.ALL_PLAN) { + analyzedPlan = cascadesContext.getRewritePlan(); + if (explainLevel == ExplainLevel.ANALYZED_PLAN) { + return analyzedPlan; + } + } - if (explainLevel == ExplainLevel.REWRITTEN_PLAN || explainLevel == ExplainLevel.ALL_PLAN) { - rewrittenPlan = cascadesContext.getRewritePlan(); - if (explainLevel == ExplainLevel.REWRITTEN_PLAN) { - return rewrittenPlan; - } - } + // rule-based optimize + rewrite(showRewriteProcess(explainLevel, showPlanProcess)); + if (statementContext.getConnectContext().getExecutor() != null) { + statementContext.getConnectContext().getExecutor().getSummaryProfile().setNereidsRewriteTime(); + } - optimize(); - if (statementContext.getConnectContext().getExecutor() != null) { - statementContext.getConnectContext().getExecutor().getSummaryProfile().setNereidsOptimizeTime(); - } + if (explainLevel == ExplainLevel.REWRITTEN_PLAN || explainLevel == ExplainLevel.ALL_PLAN) { + rewrittenPlan = cascadesContext.getRewritePlan(); + if (explainLevel == ExplainLevel.REWRITTEN_PLAN) { + return rewrittenPlan; + } + } - // print memo before choose plan. - // if chooseNthPlan failed, we could get memo to debug - if (cascadesContext.getConnectContext().getSessionVariable().dumpNereidsMemo) { - String memo = cascadesContext.getMemo().toString(); - LOG.info(ConnectContext.get().getQueryIdentifier() + "\n" + memo); - } + optimize(); + if (statementContext.getConnectContext().getExecutor() != null) { + statementContext.getConnectContext().getExecutor().getSummaryProfile().setNereidsOptimizeTime(); + } - int nth = cascadesContext.getConnectContext().getSessionVariable().getNthOptimizedPlan(); - PhysicalPlan physicalPlan = chooseNthPlan(getRoot(), requireProperties, nth); + // print memo before choose plan. + // if chooseNthPlan failed, we could get memo to debug + if (cascadesContext.getConnectContext().getSessionVariable().dumpNereidsMemo) { + String memo = cascadesContext.getMemo().toString(); + LOG.info(ConnectContext.get().getQueryIdentifier() + "\n" + memo); + } - physicalPlan = postProcess(physicalPlan); - if (cascadesContext.getConnectContext().getSessionVariable().dumpNereidsMemo) { - String tree = physicalPlan.treeString(); - LOG.info(ConnectContext.get().getQueryIdentifier() + "\n" + tree); - } - if (explainLevel == ExplainLevel.OPTIMIZED_PLAN - || explainLevel == ExplainLevel.ALL_PLAN - || explainLevel == ExplainLevel.SHAPE_PLAN) { - optimizedPlan = physicalPlan; - } - // serialize optimized plan to dumpfile, dumpfile do not have this part means optimize failed - MinidumpUtils.serializeOutputToDumpFile(physicalPlan); - NereidsTracer.output(statementContext.getConnectContext()); + int nth = cascadesContext.getConnectContext().getSessionVariable().getNthOptimizedPlan(); + PhysicalPlan physicalPlan = chooseNthPlan(getRoot(), requireProperties, nth); - return physicalPlan; - } - } finally { - statementContext.releasePlannerResources(); + physicalPlan = postProcess(physicalPlan); + if (cascadesContext.getConnectContext().getSessionVariable().dumpNereidsMemo) { + String tree = physicalPlan.treeString(); + LOG.info(ConnectContext.get().getQueryIdentifier() + "\n" + tree); } + if (explainLevel == ExplainLevel.OPTIMIZED_PLAN + || explainLevel == ExplainLevel.ALL_PLAN + || explainLevel == ExplainLevel.SHAPE_PLAN) { + optimizedPlan = physicalPlan; + } + // serialize optimized plan to dumpfile, dumpfile do not have this part means optimize failed + MinidumpUtils.serializeOutputToDumpFile(physicalPlan); + NereidsTracer.output(statementContext.getConnectContext()); + + return physicalPlan; } private LogicalPlan preprocess(LogicalPlan logicalPlan) { @@ -315,7 +332,7 @@ public class NereidsPlanner extends Planner { } } - private void splitFragments(PhysicalPlan resultPlan) throws UserException { + private void splitFragments(PhysicalPlan resultPlan) { if (resultPlan instanceof PhysicalSqlCache) { return; } @@ -379,10 +396,14 @@ public class NereidsPlanner extends Planner { cascadesContext.releaseMemo(); // update scan nodes visible version at the end of plan phase. - ScanNode.setVisibleVersionForOlapScanNodes(getScanNodes()); + try { + ScanNode.setVisibleVersionForOlapScanNodes(getScanNodes()); + } catch (UserException ue) { + throw new NereidsException(ue.getMessage(), ue); + } } - private void distribute(PhysicalPlan physicalPlan, ExplainLevel explainLevel) throws UserException { + private void distribute(PhysicalPlan physicalPlan, ExplainLevel explainLevel) { boolean canUseNereidsDistributePlanner = SessionVariable.canUseNereidsDistributePlanner(); if ((!canUseNereidsDistributePlanner && explainLevel.isPlanLevel)) { return; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java index bd64da6d470..8d6d9b0e794 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/minidump/MinidumpUtils.java @@ -214,7 +214,7 @@ public class MinidumpUtils { } NereidsPlanner nereidsPlanner = new NereidsPlanner( new StatementContext(ConnectContext.get(), new OriginStatement(sql, 0))); - nereidsPlanner.plan(LogicalPlanAdapter.of(parsed)); + nereidsPlanner.planWithLock(LogicalPlanAdapter.of(parsed)); return ((AbstractPlan) nereidsPlanner.getOptimizedPlan()).toJson(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index f92d8c6fbb2..429afcc0419 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -304,7 +304,7 @@ public class MaterializedViewUtils { } // Can not convert to table sink, because use the same column from different table when self join // the out slot is wrong - planner.plan(unboundMvPlan, PhysicalProperties.ANY, ExplainCommand.ExplainLevel.ALL_PLAN); + planner.planWithLock(unboundMvPlan, PhysicalProperties.ANY, ExplainCommand.ExplainLevel.ALL_PLAN); Plan originPlan = planner.getRewrittenPlan(); // Eliminate result sink because sink operator is useless in query rewrite by materialized view // and the top sort can also be removed diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AddConstraintCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AddConstraintCommand.java index 38bf71133c0..08954741c80 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AddConstraintCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AddConstraintCommand.java @@ -75,7 +75,7 @@ public class AddConstraintCommand extends Command implements ForwardWithSync { private Pair<ImmutableList<String>, TableIf> extractColumnsAndTable(ConnectContext ctx, LogicalPlan plan) { NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); - Plan analyzedPlan = planner.plan(plan, PhysicalProperties.ANY, ExplainLevel.ANALYZED_PLAN); + Plan analyzedPlan = planner.planWithLock(plan, PhysicalProperties.ANY, ExplainLevel.ANALYZED_PLAN); Set<LogicalCatalogRelation> logicalCatalogRelationSet = analyzedPlan .collect(LogicalCatalogRelation.class::isInstance); if (logicalCatalogRelationSet.size() != 1) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java index 72e72cbe092..9f54b858d68 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java @@ -110,7 +110,7 @@ public class CreateTableCommand extends Command implements ForwardWithSync { NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); // must disable constant folding by be, because be constant folding may return wrong type ctx.getSessionVariable().disableConstantFoldingByBEOnce(); - Plan plan = planner.plan(new UnboundResultSink<>(query), PhysicalProperties.ANY, ExplainLevel.NONE); + Plan plan = planner.planWithLock(new UnboundResultSink<>(query), PhysicalProperties.ANY, ExplainLevel.NONE); if (ctasCols == null) { // we should analyze the plan firstly to get the columns' name. ctasCols = plan.getOutput().stream().map(NamedExpression::getName).collect(Collectors.toList()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropConstraintCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropConstraintCommand.java index 9878b0e9263..fe777ea8297 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropConstraintCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropConstraintCommand.java @@ -61,7 +61,7 @@ public class DropConstraintCommand extends Command implements ForwardWithSync { private TableIf extractTable(ConnectContext ctx, LogicalPlan plan) { NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); - Plan analyzedPlan = planner.plan(plan, PhysicalProperties.ANY, ExplainLevel.ANALYZED_PLAN); + Plan analyzedPlan = planner.planWithLock(plan, PhysicalProperties.ANY, ExplainLevel.ANALYZED_PLAN); Set<LogicalCatalogRelation> logicalCatalogRelationSet = analyzedPlan .collect(LogicalCatalogRelation.class::isInstance); if (logicalCatalogRelationSet.size() != 1) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AlterViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AlterViewInfo.java index 60ae767fedc..73231ab461c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AlterViewInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AlterViewInfo.java @@ -86,7 +86,7 @@ public class AlterViewInfo extends BaseViewInfo { /**validate*/ public void validate(ConnectContext ctx) throws UserException { NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); - planner.plan(new UnboundResultSink<>(logicalQuery), PhysicalProperties.ANY, ExplainLevel.NONE); + planner.planWithLock(new UnboundResultSink<>(logicalQuery), PhysicalProperties.ANY, ExplainLevel.NONE); Set<String> colSets = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); for (Column col : finalCols) { if (!colSets.add(col.getName())) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java index 53d6fa1353e..6ccc6a08fb9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java @@ -209,7 +209,7 @@ public class CreateMTMVInfo { LogicalSink<Plan> logicalSink = new UnboundResultSink<>(logicalQuery); // must disable constant folding by be, because be constant folding may return wrong type ctx.getSessionVariable().disableConstantFoldingByBEOnce(); - Plan plan = planner.plan(logicalSink, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN); + Plan plan = planner.planWithLock(logicalSink, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN); if (plan.anyMatch(node -> node instanceof OneRowRelation)) { throw new AnalysisException("at least contain one table"); } @@ -280,7 +280,7 @@ public class CreateMTMVInfo { } Plan plan; try { - plan = planner.plan(logicalQuery, PhysicalProperties.ANY, ExplainLevel.NONE); + plan = planner.planWithLock(logicalQuery, PhysicalProperties.ANY, ExplainLevel.NONE); } finally { sessionVariable.setDisableNereidsRules(String.join(",", tempDisableRules)); ctx.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java index bf3eabcc4c1..65eb3453ec3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java @@ -79,7 +79,7 @@ public class CreateViewInfo extends BaseViewInfo { /**validate*/ public void validate(ConnectContext ctx) throws UserException { NereidsPlanner planner = new NereidsPlanner(ctx.getStatementContext()); - planner.plan(new UnboundResultSink<>(logicalQuery), PhysicalProperties.ANY, ExplainLevel.NONE); + planner.planWithLock(new UnboundResultSink<>(logicalQuery), PhysicalProperties.ANY, ExplainLevel.NONE); Set<String> colSets = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); for (Column col : finalCols) { if (!colSets.add(col.getName())) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java index 2c5cbb4649e..427e2368e7a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java @@ -137,7 +137,7 @@ public class MTMVPartitionDefinition { cascadesContext.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); try { Plan mvRewrittenPlan = - planner.plan(logicalQuery, PhysicalProperties.ANY, ExplainLevel.REWRITTEN_PLAN); + planner.planWithLock(logicalQuery, PhysicalProperties.ANY, ExplainLevel.REWRITTEN_PLAN); RelatedTableInfo relatedTableInfo = MaterializedViewUtils .getRelatedTableInfo(partitionColName, timeUnit, mvRewrittenPlan, cascadesContext); if (!relatedTableInfo.isPctPossible()) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java index fcf647b2d03..a40a4fed385 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java @@ -610,7 +610,7 @@ public class HiveDDLAndDMLPlanTest extends TestWithFeService { } Assertions.assertTrue(exPlan instanceof UnboundLogicalSink); NereidsPlanner planner = new NereidsPlanner(statementContext); - return planner.plan((UnboundLogicalSink<?>) exPlan, physicalProperties); + return planner.planWithLock((UnboundLogicalSink<?>) exPlan, physicalProperties); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/UnsupportedTypeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/UnsupportedTypeTest.java index 8031516ff5e..c202230ce82 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/UnsupportedTypeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/UnsupportedTypeTest.java @@ -94,7 +94,7 @@ public class UnsupportedTypeTest extends TestWithFeService { } private void runPlanner(String sql) { - new NereidsPlanner(MemoTestUtils.createStatementContext(connectContext, sql)).plan( + new NereidsPlanner(MemoTestUtils.createStatementContext(connectContext, sql)).planWithLock( new NereidsParser().parseSingle(sql), PhysicalProperties.ANY ); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/preprocess/SelectHintTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/preprocess/SelectHintTest.java index 94dab1eef7b..f548a499cad 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/preprocess/SelectHintTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/preprocess/SelectHintTest.java @@ -61,7 +61,7 @@ public class SelectHintTest { sv.setEnableNereidsPlanner(true); sv.enableFallbackToOriginalPlanner = false; Assertions.assertThrows(AnalysisException.class, () -> new NereidsPlanner(statementContext) - .plan(new NereidsParser().parseSingle(sql), PhysicalProperties.ANY)); + .planWithLock(new NereidsParser().parseSingle(sql), PhysicalProperties.ANY)); // manually recover sv sv.setEnableNereidsPlanner(true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTETest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTETest.java index 522f198e3ff..abe82c858d4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTETest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeCTETest.java @@ -119,7 +119,7 @@ public class AnalyzeCTETest extends TestWithFeService implements MemoPatternMatc for (String sql : testSqls) { StatementScopeIdGenerator.clear(); StatementContext statementContext = MemoTestUtils.createStatementContext(connectContext, sql); - PhysicalPlan plan = new NereidsPlanner(statementContext).plan( + PhysicalPlan plan = new NereidsPlanner(statementContext).planWithLock( parser.parseSingle(sql), PhysicalProperties.ANY ); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeSubQueryTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeSubQueryTest.java index db56b16e6f4..138db44863f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeSubQueryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeSubQueryTest.java @@ -91,7 +91,7 @@ public class AnalyzeSubQueryTest extends TestWithFeService implements MemoPatter StatementScopeIdGenerator.clear(); StatementContext statementContext = MemoTestUtils.createStatementContext(connectContext, sql); NereidsPlanner planner = new NereidsPlanner(statementContext); - PhysicalPlan plan = planner.plan( + PhysicalPlan plan = planner.planWithLock( parser.parseSingle(sql), PhysicalProperties.ANY ); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeWhereSubqueryTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeWhereSubqueryTest.java index a8f9543f182..258f9ace453 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeWhereSubqueryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/AnalyzeWhereSubqueryTest.java @@ -137,7 +137,7 @@ public class AnalyzeWhereSubqueryTest extends TestWithFeService implements MemoP try { StatementScopeIdGenerator.clear(); StatementContext statementContext = MemoTestUtils.createStatementContext(connectContext, sql); - PhysicalPlan plan = new NereidsPlanner(statementContext).plan( + PhysicalPlan plan = new NereidsPlanner(statementContext).planWithLock( parser.parseSingle(sql), PhysicalProperties.ANY ); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownExpressionsInHashConditionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownExpressionsInHashConditionTest.java index 4370298d6c9..f9907e2088a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownExpressionsInHashConditionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownExpressionsInHashConditionTest.java @@ -72,7 +72,7 @@ public class PushDownExpressionsInHashConditionTest extends TestWithFeService im "SELECT * FROM T1 JOIN (SELECT ID, SUM(SCORE) SCORE FROM T2 GROUP BY ID) T ON T1.ID + 1 = T.ID AND T.SCORE < 10", "SELECT * FROM T1 JOIN (SELECT ID, SUM(SCORE) SCORE FROM T2 GROUP BY ID ORDER BY ID) T ON T1.ID + 1 = T.ID AND T.SCORE < 10" ); - testSql.forEach(sql -> new NereidsPlanner(createStatementCtx(sql)).plan( + testSql.forEach(sql -> new NereidsPlanner(createStatementCtx(sql)).planWithLock( new NereidsParser().parseSingle(sql), PhysicalProperties.ANY )); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ViewTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ViewTest.java index d55a034710a..7d2cb54d33e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ViewTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ViewTest.java @@ -101,7 +101,7 @@ public class ViewTest extends TestWithFeService implements MemoPatternMatchSuppo System.out.println("\n\n***** " + sql + " *****\n\n"); StatementContext statementContext = MemoTestUtils.createStatementContext(connectContext, sql); NereidsPlanner planner = new NereidsPlanner(statementContext); - PhysicalPlan plan = planner.plan( + PhysicalPlan plan = planner.planWithLock( new NereidsParser().parseSingle(sql), PhysicalProperties.ANY ); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/ExplainInsertCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/ExplainInsertCommandTest.java index de73470d59b..d4756746e98 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/ExplainInsertCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/ExplainInsertCommandTest.java @@ -140,7 +140,7 @@ public class ExplainInsertCommandTest extends TestWithFeService { NereidsPlanner planner = new NereidsPlanner(statementContext); LogicalPlan logicalPlan = (LogicalPlan) ((Explainable) (((ExplainCommand) parser.parseSingle(sql)) .getLogicalPlan())).getExplainPlan(connectContext); - PhysicalPlan plan = planner.plan(logicalPlan, PhysicalProperties.ANY); + PhysicalPlan plan = planner.planWithLock(logicalPlan, PhysicalProperties.ANY); return new PhysicalPlanTranslator(new PlanTranslatorContext(planner.getCascadesContext())).translatePlan(plan); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/OutFileTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/OutFileTest.java index f8f59fa4e62..6a10110cf90 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/OutFileTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/OutFileTest.java @@ -82,7 +82,7 @@ public class OutFileTest extends TestWithFeService implements PlanPatternMatchSu StatementScopeIdGenerator.clear(); StatementContext statementContext = MemoTestUtils.createStatementContext(connectContext, sql); NereidsPlanner planner = new NereidsPlanner(statementContext); - PhysicalPlan plan = planner.plan( + PhysicalPlan plan = planner.planWithLock( parser.parseSingle(sql), PhysicalProperties.ANY ); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java index fdce1c0b9af..3d908214062 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java @@ -181,7 +181,7 @@ public class PlanChecker { LogicalPlan command = parser.parseSingle(sql); NereidsPlanner planner = new NereidsPlanner( new StatementContext(connectContext, new OriginStatement(sql, 0))); - planner.plan(command, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN, true); + planner.planWithLock(command, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN, true); this.cascadesContext = planner.getCascadesContext(); return cascadesContext.getPlanProcesses(); } @@ -560,7 +560,7 @@ public class PlanChecker { new StatementContext(connectContext, new OriginStatement(sql, 0))); LogicalPlanAdapter adapter = LogicalPlanAdapter.of(parsed); adapter.setIsExplain(new ExplainOptions(ExplainLevel.ALL_PLAN, false)); - nereidsPlanner.plan(adapter); + nereidsPlanner.planWithLock(adapter); consumer.accept(nereidsPlanner); return this; } @@ -569,7 +569,7 @@ public class PlanChecker { LogicalPlan parsed = new NereidsParser().parseSingle(sql); NereidsPlanner nereidsPlanner = new NereidsPlanner( new StatementContext(connectContext, new OriginStatement(sql, 0))); - nereidsPlanner.plan(LogicalPlanAdapter.of(parsed)); + nereidsPlanner.planWithLock(LogicalPlanAdapter.of(parsed)); consumer.accept(nereidsPlanner); return this; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ReadLockTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ReadLockTest.java index 7bcbecafb02..9283b286f41 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ReadLockTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/ReadLockTest.java @@ -43,7 +43,7 @@ public class ReadLockTest extends SSBTestBase { String sql = "SELECT s_suppkey FROM supplier"; StatementContext statementContext = MemoTestUtils.createStatementContext(connectContext, sql); NereidsPlanner planner = new NereidsPlanner(statementContext); - planner.plan( + planner.planWithLock( parser.parseSingle(sql), PhysicalProperties.ANY ); @@ -65,7 +65,7 @@ public class ReadLockTest extends SSBTestBase { + " FROM cte1 as t1, cte1 as t2"; StatementContext statementContext = MemoTestUtils.createStatementContext(connectContext, sql); NereidsPlanner planner = new NereidsPlanner(statementContext); - planner.plan( + planner.planWithLock( parser.parseSingle(sql), PhysicalProperties.ANY ); @@ -80,7 +80,7 @@ public class ReadLockTest extends SSBTestBase { String sql = "SELECT s_suppkey FROM (SELECT * FROM supplier) t"; StatementContext statementContext = MemoTestUtils.createStatementContext(connectContext, sql); NereidsPlanner planner = new NereidsPlanner(statementContext); - planner.plan( + planner.planWithLock( parser.parseSingle(sql), PhysicalProperties.ANY ); @@ -95,7 +95,7 @@ public class ReadLockTest extends SSBTestBase { String sql = "SELECT s_suppkey FROM supplier WHERE s_suppkey > (SELECT MAX(lo_orderkey) FROM lineorder)"; StatementContext statementContext = MemoTestUtils.createStatementContext(connectContext, sql); NereidsPlanner planner = new NereidsPlanner(statementContext); - planner.plan( + planner.planWithLock( parser.parseSingle(sql), PhysicalProperties.ANY ); @@ -113,7 +113,7 @@ public class ReadLockTest extends SSBTestBase { StatementContext statementContext = MemoTestUtils.createStatementContext(connectContext, sql); InsertIntoTableCommand insertIntoTableCommand = (InsertIntoTableCommand) parser.parseSingle(sql); NereidsPlanner planner = new NereidsPlanner(statementContext); - planner.plan( + planner.planWithLock( (LogicalPlan) insertIntoTableCommand.getExplainPlan(connectContext), PhysicalProperties.ANY ); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java index b29a9a4119c..cbc12f4ecce 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/OlapQueryCacheTest.java @@ -502,7 +502,7 @@ public class OlapQueryCacheTest { ctx.setStatementContext(statementContext); NereidsPlanner nereidsPlanner = new NereidsPlanner(statementContext); LogicalPlanAdapter adapter = new LogicalPlanAdapter(plan, statementContext); - nereidsPlanner.plan(adapter); + nereidsPlanner.planWithLock(adapter); statementContext.setParsedStatement(adapter); stmt = adapter; } catch (Throwable throwable) { --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org