This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 2d7acd01bea [improvement](mv) Catch exception when init sync
materialized view context #37701 (#37872)
2d7acd01bea is described below
commit 2d7acd01beab07e8b9720d7cb7e40e205de4c927
Author: seawinde <149132972+seawi...@users.noreply.github.com>
AuthorDate: Tue Jul 16 11:46:40 2024 +0800
[improvement](mv) Catch exception when init sync materialized view context
#37701 (#37872)
## Proposed changes
cherry-pick 3.0
pr: https://github.com/apache/doris/pull/37701
commit: e478570e
pr: https://github.com/apache/doris/pull/37154
commit: 841e39a2
---------
Co-authored-by: starocean999
<40539150+starocean...@users.noreply.github.com>
---
.../java/org/apache/doris/catalog/OlapTable.java | 9 +++
.../mv/InitMaterializationContextHook.java | 64 ++++++++++++----------
2 files changed, 45 insertions(+), 28 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 7d3b9baf341..471cbdf0ec6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -2990,4 +2990,13 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
public long getReplicaCount() {
return statistics.getReplicaCount();
}
+
+ public boolean isShadowIndex(long indexId) {
+ String indexName = getIndexNameById(indexId);
+ if (indexName != null &&
indexName.startsWith(org.apache.doris.alter.SchemaChangeHandler.SHADOW_NAME_PREFIX))
{
+ return true;
+ } else {
+ return false;
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
index 072765fb2a4..53a352e2ef8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java
@@ -145,38 +145,46 @@ public class InitMaterializationContextHook implements
PlannerHook {
keyCount += column.isKey() ? 1 : 0;
}
for (Map.Entry<String, Long> entry :
olapTable.getIndexNameToId().entrySet()) {
- if (entry.getValue() != baseIndexId) {
- MaterializedIndexMeta meta =
olapTable.getIndexMetaByIndexId(entry.getValue());
- String createMvSql;
- if (meta.getDefineStmt() != null) {
- // get the original create mv sql
- createMvSql = meta.getDefineStmt().originStmt;
- } else {
- // it's rollup, need assemble create mv sql manually
- if (olapTable.getKeysType() == KeysType.AGG_KEYS) {
- createMvSql =
assembleCreateMvSqlForAggTable(olapTable.getQualifiedName(),
- entry.getKey(), meta.getSchema(false),
keyCount);
+ long indexId = entry.getValue();
+ try {
+ // when doing schema change, a shadow index would be created
and put together with mv indexes
+ // we must roll out these unexpected shadow indexes here
+ if (indexId != baseIndexId &&
!olapTable.isShadowIndex(indexId)) {
+ MaterializedIndexMeta meta =
olapTable.getIndexMetaByIndexId(entry.getValue());
+ String createMvSql;
+ if (meta.getDefineStmt() != null) {
+ // get the original create mv sql
+ createMvSql = meta.getDefineStmt().originStmt;
} else {
- createMvSql =
-
assembleCreateMvSqlForDupOrUniqueTable(olapTable.getQualifiedName(),
- entry.getKey(), meta.getSchema(false));
+ // it's rollup, need assemble create mv sql manually
+ if (olapTable.getKeysType() == KeysType.AGG_KEYS) {
+ createMvSql =
assembleCreateMvSqlForAggTable(olapTable.getQualifiedName(),
+ entry.getKey(), meta.getSchema(false),
keyCount);
+ } else {
+ createMvSql =
+
assembleCreateMvSqlForDupOrUniqueTable(olapTable.getQualifiedName(),
+ entry.getKey(),
meta.getSchema(false));
+ }
}
- }
- if (createMvSql != null) {
- Optional<String> querySql =
- new NereidsParser().parseForSyncMv(createMvSql);
- if (!querySql.isPresent()) {
- LOG.warn(String.format("can't parse %s ",
createMvSql));
- continue;
+ if (createMvSql != null) {
+ Optional<String> querySql =
+ new
NereidsParser().parseForSyncMv(createMvSql);
+ if (!querySql.isPresent()) {
+ LOG.warn(String.format("can't parse %s ",
createMvSql));
+ continue;
+ }
+ MTMVCache mtmvCache =
MaterializedViewUtils.createMTMVCache(querySql.get(),
+ cascadesContext.getConnectContext());
+ contexts.add(new
SyncMaterializationContext(mtmvCache.getLogicalPlan(),
+ mtmvCache.getOriginalPlan(), olapTable,
meta.getIndexId(), entry.getKey(),
+ cascadesContext, mtmvCache.getStatistics()));
+ } else {
+ LOG.warn(String.format("can't assemble create mv sql
for index ", entry.getKey()));
}
- MTMVCache mtmvCache =
MaterializedViewUtils.createMTMVCache(querySql.get(),
- cascadesContext.getConnectContext());
- contexts.add(new
SyncMaterializationContext(mtmvCache.getLogicalPlan(),
- mtmvCache.getOriginalPlan(), olapTable,
meta.getIndexId(), entry.getKey(),
- cascadesContext, mtmvCache.getStatistics()));
- } else {
- LOG.warn(String.format("can't assemble create mv sql for
index ", entry.getKey()));
}
+ } catch (Exception exception) {
+ LOG.warn(String.format("createSyncMvContexts exception, index
id is %s, index name is %s",
+ entry.getValue(), entry.getValue()), exception);
}
}
return contexts;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
