This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 913cda6a963 [feat](mtmv)Unified external table interface supporting
partition refresh and partition pruning (#44673)
913cda6a963 is described below
commit 913cda6a96351ac58e22e628d16100574c49b0b8
Author: zhangdong <zhangd...@selectdb.com>
AuthorDate: Thu Nov 28 10:22:20 2024 +0800
[feat](mtmv)Unified external table interface supporting partition refresh
and partition pruning (#44673)
### What problem does this PR solve?
- Add `MvccTable` to represent a table that supports querying specified
version data
- Add the `MvccSnapshot` interface to store snapshot information of mvcc
at a certain moment in time
- Add the `MvccSnapshot` parameter to the method of the
`MTMVRelatedTableIf `interface to retrieve data of a specified version
- Partition pruning related methods combined with the `MvccSnapshot`
parameter are used to obtain partition information for a specified
version
- Load the snapshot information of mvccTable at the beginning of the
query plan and store it in StatementContext
### Release note
Unified external table interface supporting partition refresh and
partition pruning
---
.../main/java/org/apache/doris/catalog/MTMV.java | 7 +-
.../java/org/apache/doris/catalog/OlapTable.java | 25 +++++--
.../org/apache/doris/datasource/ExternalTable.java | 18 ++---
.../doris/datasource/hive/HMSExternalTable.java | 33 ++++-----
.../mvcc/MvccSnapshot.java} | 18 ++---
.../mvcc/MvccTable.java} | 24 +++----
.../doris/datasource/mvcc/MvccTableInfo.java | 84 ++++++++++++++++++++++
.../datasource/paimon/PaimonExternalTable.java | 14 ++--
.../doris/mtmv/MTMVPartitionExprDateTrunc.java | 2 +-
.../org/apache/doris/mtmv/MTMVPartitionInfo.java | 3 +-
.../org/apache/doris/mtmv/MTMVPartitionUtil.java | 12 ++--
.../MTMVRelatedPartitionDescInitGenerator.java | 4 +-
.../MTMVRelatedPartitionDescRollUpGenerator.java | 3 +-
.../org/apache/doris/mtmv/MTMVRelatedTableIf.java | 27 ++++---
.../org/apache/doris/nereids/CascadesContext.java | 8 +++
.../org/apache/doris/nereids/NereidsPlanner.java | 2 +-
.../org/apache/doris/nereids/StatementContext.java | 31 ++++++++
.../exploration/mv/MaterializedViewUtils.java | 4 +-
.../rules/rewrite/PruneFileScanPartition.java | 6 +-
.../trees/plans/commands/info/CreateMTMVInfo.java | 2 +-
.../commands/info/MTMVPartitionDefinition.java | 3 +-
.../trees/plans/logical/LogicalFileScan.java | 3 +-
.../apache/doris/mtmv/MTMVPartitionUtilTest.java | 6 +-
23 files changed, 235 insertions(+), 104 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
index 825bdef9f09..955bfd4279f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java
@@ -59,7 +59,6 @@ import java.io.IOException;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
-import java.util.OptionalLong;
import java.util.Set;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -364,7 +363,7 @@ public class MTMV extends OlapTable {
* @return mvPartitionName ==> mvPartitionKeyDesc
*/
public Map<String, PartitionKeyDesc> generateMvPartitionDescs() throws
AnalysisException {
- Map<String, PartitionItem> mtmvItems =
getAndCopyPartitionItems(OptionalLong.empty());
+ Map<String, PartitionItem> mtmvItems = getAndCopyPartitionItems();
Map<String, PartitionKeyDesc> result = Maps.newHashMap();
for (Entry<String, PartitionItem> entry : mtmvItems.entrySet()) {
result.put(entry.getKey(), entry.getValue().toPartitionKeyDesc());
@@ -393,7 +392,7 @@ public class MTMV extends OlapTable {
Map<String, String> baseToMv = Maps.newHashMap();
Map<PartitionKeyDesc, Set<String>> relatedPartitionDescs =
MTMVPartitionUtil
.generateRelatedPartitionDescs(mvPartitionInfo, mvProperties);
- Map<String, PartitionItem> mvPartitionItems =
getAndCopyPartitionItems(OptionalLong.empty());
+ Map<String, PartitionItem> mvPartitionItems =
getAndCopyPartitionItems();
for (Entry<String, PartitionItem> entry : mvPartitionItems.entrySet())
{
Set<String> basePartitionNames =
relatedPartitionDescs.getOrDefault(entry.getValue().toPartitionKeyDesc(),
Sets.newHashSet());
@@ -426,7 +425,7 @@ public class MTMV extends OlapTable {
Map<String, Set<String>> res = Maps.newHashMap();
Map<PartitionKeyDesc, Set<String>> relatedPartitionDescs =
MTMVPartitionUtil
.generateRelatedPartitionDescs(mvPartitionInfo, mvProperties);
- Map<String, PartitionItem> mvPartitionItems =
getAndCopyPartitionItems(OptionalLong.empty());
+ Map<String, PartitionItem> mvPartitionItems =
getAndCopyPartitionItems();
for (Entry<String, PartitionItem> entry : mvPartitionItems.entrySet())
{
res.put(entry.getKey(),
relatedPartitionDescs.getOrDefault(entry.getValue().toPartitionKeyDesc(),
Sets.newHashSet()));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index ec3bd2acbc5..5d57540017f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -50,6 +50,7 @@ import org.apache.doris.common.io.Text;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.datasource.mvcc.MvccSnapshot;
import org.apache.doris.mtmv.MTMVRefreshContext;
import org.apache.doris.mtmv.MTMVRelatedTableIf;
import org.apache.doris.mtmv.MTMVSnapshotIf;
@@ -110,7 +111,6 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Optional;
-import java.util.OptionalLong;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
@@ -1049,6 +1049,10 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
}
@Override
+ public Set<String> getPartitionColumnNames(Optional<MvccSnapshot>
snapshot) throws DdlException {
+ return getPartitionColumnNames();
+ }
+
public Set<String> getPartitionColumnNames() throws DdlException {
Set<String> partitionColumnNames = Sets.newHashSet();
if (partitionInfo instanceof SinglePartitionInfo) {
@@ -3251,12 +3255,21 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
}
@Override
+ public PartitionType getPartitionType(Optional<MvccSnapshot> snapshot) {
+ return getPartitionType();
+ }
+
public PartitionType getPartitionType() {
return partitionInfo.getType();
}
@Override
- public Map<String, PartitionItem> getAndCopyPartitionItems(OptionalLong
snapshotId) throws AnalysisException {
+ public Map<String, PartitionItem>
getAndCopyPartitionItems(Optional<MvccSnapshot> snapshot)
+ throws AnalysisException {
+ return getAndCopyPartitionItems();
+ }
+
+ public Map<String, PartitionItem> getAndCopyPartitionItems() throws
AnalysisException {
if (!tryReadLock(1, TimeUnit.MINUTES)) {
throw new AnalysisException("get table read lock timeout,
database=" + getDBName() + ",table=" + getName());
}
@@ -3275,13 +3288,17 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
}
@Override
+ public List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot) {
+ return getPartitionColumns();
+ }
+
public List<Column> getPartitionColumns() {
return getPartitionInfo().getPartitionColumns();
}
@Override
public MTMVSnapshotIf getPartitionSnapshot(String partitionName,
MTMVRefreshContext context,
- OptionalLong snapshotId)
+ Optional<MvccSnapshot> snapshot)
throws AnalysisException {
Map<String, Long> partitionVersions =
context.getBaseVersions().getPartitionVersions();
long partitionId =
getPartitionOrAnalysisException(partitionName).getId();
@@ -3291,7 +3308,7 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
}
@Override
- public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context,
OptionalLong snapshotId) {
+ public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context,
Optional<MvccSnapshot> snapshot) {
Map<Long, Long> tableVersions =
context.getBaseVersions().getTableVersions();
long visibleVersion = tableVersions.containsKey(id) ?
tableVersions.get(id) : getVisibleVersion();
return new MTMVVersionSnapshot(visibleVersion, id);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
index 3aee5550acf..d82959954f2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
@@ -31,6 +31,7 @@ import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.common.util.Util;
+import org.apache.doris.datasource.mvcc.MvccSnapshot;
import
org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions;
import org.apache.doris.persist.gson.GsonPostProcessable;
import org.apache.doris.persist.gson.GsonUtils;
@@ -55,7 +56,6 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.OptionalLong;
import java.util.Set;
/**
@@ -373,17 +373,17 @@ public class ExternalTable implements TableIf, Writable,
GsonPostProcessable {
/**
* Retrieve all partitions and initialize SelectedPartitions
*
- * @param snapshotId if not support mvcc, ignore this
+ * @param snapshot if not support mvcc, ignore this
* @return
*/
- public SelectedPartitions initSelectedPartitions(OptionalLong snapshotId) {
+ public SelectedPartitions initSelectedPartitions(Optional<MvccSnapshot>
snapshot) {
if (!supportPartitionPruned()) {
return SelectedPartitions.NOT_PRUNED;
}
- if (CollectionUtils.isEmpty(this.getPartitionColumns(snapshotId))) {
+ if (CollectionUtils.isEmpty(this.getPartitionColumns(snapshot))) {
return SelectedPartitions.NOT_PRUNED;
}
- Map<String, PartitionItem> nameToPartitionItems =
getNameToPartitionItems(snapshotId);
+ Map<String, PartitionItem> nameToPartitionItems =
getNameToPartitionItems(snapshot);
return new SelectedPartitions(nameToPartitionItems.size(),
nameToPartitionItems, false);
}
@@ -391,10 +391,10 @@ public class ExternalTable implements TableIf, Writable,
GsonPostProcessable {
* get partition map
* If partition related operations are supported, this method needs to be
implemented in the subclass
*
- * @param snapshotId if not support mvcc, ignore this
+ * @param snapshot if not support mvcc, ignore this
* @return partitionName ==> PartitionItem
*/
- public Map<String, PartitionItem> getNameToPartitionItems(OptionalLong
snapshotId) {
+ public Map<String, PartitionItem>
getNameToPartitionItems(Optional<MvccSnapshot> snapshot) {
return Collections.emptyMap();
}
@@ -402,10 +402,10 @@ public class ExternalTable implements TableIf, Writable,
GsonPostProcessable {
* get partition column list
* If partition related operations are supported, this method needs to be
implemented in the subclass
*
- * @param snapshotId if not support mvcc, ignore this
+ * @param snapshot if not support mvcc, ignore this
* @return
*/
- public List<Column> getPartitionColumns(OptionalLong snapshotId) {
+ public List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot) {
return Collections.emptyList();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
index 1c30fa24cfb..6d65f8bcdbc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
@@ -33,6 +33,7 @@ import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.SchemaCacheValue;
import org.apache.doris.datasource.hudi.HudiUtils;
import org.apache.doris.datasource.iceberg.IcebergUtils;
+import org.apache.doris.datasource.mvcc.MvccSnapshot;
import org.apache.doris.mtmv.MTMVBaseTableIf;
import org.apache.doris.mtmv.MTMVMaxTimestampSnapshot;
import org.apache.doris.mtmv.MTMVRefreshContext;
@@ -83,7 +84,6 @@ import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
-import java.util.OptionalLong;
import java.util.Set;
import java.util.stream.Collectors;
@@ -288,7 +288,6 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
.orElse(Collections.emptyList());
}
- @Override
public List<Column> getPartitionColumns() {
makeSureInitialized();
Optional<SchemaCacheValue> schemaCacheValue = getSchemaCacheValue();
@@ -297,7 +296,7 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
}
@Override
- public List<Column> getPartitionColumns(OptionalLong snapshotId) {
+ public List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot) {
return getPartitionColumns();
}
@@ -307,7 +306,7 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
}
@Override
- public Map<String, PartitionItem> getNameToPartitionItems(OptionalLong
snapshotId) {
+ public Map<String, PartitionItem>
getNameToPartitionItems(Optional<MvccSnapshot> snapshot) {
return getNameToPartitionItems();
}
@@ -756,34 +755,32 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
}
@Override
+ public PartitionType getPartitionType(Optional<MvccSnapshot> snapshot) {
+ return getPartitionType();
+ }
+
public PartitionType getPartitionType() {
return getPartitionColumns().size() > 0 ? PartitionType.LIST :
PartitionType.UNPARTITIONED;
}
@Override
+ public Set<String> getPartitionColumnNames(Optional<MvccSnapshot>
snapshot) {
+ return getPartitionColumnNames();
+ }
+
public Set<String> getPartitionColumnNames() {
return getPartitionColumns().stream()
.map(c ->
c.getName().toLowerCase()).collect(Collectors.toSet());
}
@Override
- public Map<String, PartitionItem> getAndCopyPartitionItems(OptionalLong
snapshotId) {
- HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
- .getMetaStoreCache((HMSExternalCatalog) getCatalog());
- HiveMetaStoreCache.HivePartitionValues hivePartitionValues =
cache.getPartitionValues(
- getDbName(), getName(), getPartitionColumnTypes());
- Map<String, PartitionItem> res = Maps.newHashMap();
- Map<Long, PartitionItem> idToPartitionItem =
hivePartitionValues.getIdToPartitionItem();
- BiMap<Long, String> idToName =
hivePartitionValues.getPartitionNameToIdMap().inverse();
- for (Entry<Long, PartitionItem> entry : idToPartitionItem.entrySet()) {
- res.put(idToName.get(entry.getKey()), entry.getValue());
- }
- return res;
+ public Map<String, PartitionItem>
getAndCopyPartitionItems(Optional<MvccSnapshot> snapshot) {
+ return getNameToPartitionItems();
}
@Override
public MTMVSnapshotIf getPartitionSnapshot(String partitionName,
MTMVRefreshContext context,
- OptionalLong snapshotId) throws AnalysisException {
+ Optional<MvccSnapshot> snapshot) throws AnalysisException {
HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
.getMetaStoreCache((HMSExternalCatalog) getCatalog());
HiveMetaStoreCache.HivePartitionValues hivePartitionValues =
cache.getPartitionValues(
@@ -795,7 +792,7 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
}
@Override
- public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context,
OptionalLong snapshotId)
+ public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context,
Optional<MvccSnapshot> snapshot)
throws AnalysisException {
if (getPartitionType() == PartitionType.UNPARTITIONED) {
return new MTMVMaxTimestampSnapshot(getName(), getLastDdlTime());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescInitGenerator.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccSnapshot.java
similarity index 58%
copy from
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescInitGenerator.java
copy to
fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccSnapshot.java
index ef3100dec4c..d7826b0a5de 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescInitGenerator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccSnapshot.java
@@ -15,21 +15,11 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.mtmv;
-
-import org.apache.doris.common.AnalysisException;
-
-import java.util.Map;
-import java.util.OptionalLong;
+package org.apache.doris.datasource.mvcc;
/**
- * get all related partition descs
+ * The snapshot information of mvcc is defined by each table,
+ * but it should be ensured that the table information queried through this
snapshot remains unchanged
*/
-public class MTMVRelatedPartitionDescInitGenerator implements
MTMVRelatedPartitionDescGeneratorService {
-
- @Override
- public void apply(MTMVPartitionInfo mvPartitionInfo, Map<String, String>
mvProperties,
- RelatedPartitionDescResult lastResult) throws AnalysisException {
-
lastResult.setItems(mvPartitionInfo.getRelatedTable().getAndCopyPartitionItems(OptionalLong.empty()));
- }
+public interface MvccSnapshot {
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescInitGenerator.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccTable.java
similarity index 58%
copy from
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescInitGenerator.java
copy to fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccTable.java
index ef3100dec4c..d69e0f3114d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescInitGenerator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccTable.java
@@ -15,21 +15,19 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.mtmv;
+package org.apache.doris.datasource.mvcc;
-import org.apache.doris.common.AnalysisException;
-
-import java.util.Map;
-import java.util.OptionalLong;
+import org.apache.doris.catalog.TableIf;
/**
- * get all related partition descs
+ * The table that needs to query data based on the version needs to implement
this interface.
*/
-public class MTMVRelatedPartitionDescInitGenerator implements
MTMVRelatedPartitionDescGeneratorService {
-
- @Override
- public void apply(MTMVPartitionInfo mvPartitionInfo, Map<String, String>
mvProperties,
- RelatedPartitionDescResult lastResult) throws AnalysisException {
-
lastResult.setItems(mvPartitionInfo.getRelatedTable().getAndCopyPartitionItems(OptionalLong.empty()));
- }
+public interface MvccTable extends TableIf {
+ /**
+ * Retrieve the current snapshot information of the table,
+ * and the returned result will be used for the entire process of this
query
+ *
+ * @return MvccSnapshot
+ */
+ MvccSnapshot loadSnapshot();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccTableInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccTableInfo.java
new file mode 100644
index 00000000000..0d865f837c8
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/mvcc/MvccTableInfo.java
@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.mvcc;
+
+import org.apache.doris.catalog.DatabaseIf;
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.datasource.CatalogIf;
+
+import com.google.common.base.Objects;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class MvccTableInfo {
+ private static final Logger LOG =
LogManager.getLogger(MvccTableInfo.class);
+
+ private String tableName;
+ private String dbName;
+ private String ctlName;
+
+ public MvccTableInfo(TableIf table) {
+ java.util.Objects.requireNonNull(table, "table is null");
+ DatabaseIf database = table.getDatabase();
+ java.util.Objects.requireNonNull(database, "database is null");
+ CatalogIf catalog = database.getCatalog();
+ java.util.Objects.requireNonNull(database, "catalog is null");
+ this.tableName = table.getName();
+ this.dbName = database.getFullName();
+ this.ctlName = catalog.getName();
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public String getDbName() {
+ return dbName;
+ }
+
+ public String getCtlName() {
+ return ctlName;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ MvccTableInfo that = (MvccTableInfo) o;
+ return Objects.equal(tableName, that.tableName) && Objects.equal(
+ dbName, that.dbName) && Objects.equal(ctlName, that.ctlName);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(tableName, dbName, ctlName);
+ }
+
+ @Override
+ public String toString() {
+ return "MvccTableInfo{"
+ + "tableName='" + tableName + '\''
+ + ", dbName='" + dbName + '\''
+ + ", ctlName='" + ctlName + '\''
+ + '}';
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
index 632a0da0ebd..7fe3c858448 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java
@@ -28,6 +28,7 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.SchemaCacheValue;
+import org.apache.doris.datasource.mvcc.MvccSnapshot;
import org.apache.doris.mtmv.MTMVBaseTableIf;
import org.apache.doris.mtmv.MTMVRefreshContext;
import org.apache.doris.mtmv.MTMVRelatedTableIf;
@@ -67,7 +68,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.OptionalLong;
import java.util.Set;
import java.util.stream.Collectors;
@@ -313,29 +313,29 @@ public class PaimonExternalTable extends ExternalTable
implements MTMVRelatedTab
}
@Override
- public Map<String, PartitionItem> getAndCopyPartitionItems(OptionalLong
snapshotId) {
+ public Map<String, PartitionItem>
getAndCopyPartitionItems(Optional<MvccSnapshot> snapshot) {
return
Maps.newHashMap(getPartitionInfoFromCache().getNameToPartitionItem());
}
@Override
- public PartitionType getPartitionType() {
+ public PartitionType getPartitionType(Optional<MvccSnapshot> snapshot) {
return getPartitionColumnsFromCache().size() > 0 ? PartitionType.LIST
: PartitionType.UNPARTITIONED;
}
@Override
- public Set<String> getPartitionColumnNames() {
+ public Set<String> getPartitionColumnNames(Optional<MvccSnapshot>
snapshot) {
return getPartitionColumnsFromCache().stream()
.map(c ->
c.getName().toLowerCase()).collect(Collectors.toSet());
}
@Override
- public List<Column> getPartitionColumns() {
+ public List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot) {
return getPartitionColumnsFromCache();
}
@Override
public MTMVSnapshotIf getPartitionSnapshot(String partitionName,
MTMVRefreshContext context,
- OptionalLong snapshotId)
+ Optional<MvccSnapshot> snapshot)
throws AnalysisException {
PaimonPartition paimonPartition =
getPartitionInfoFromCache().getNameToPartition().get(partitionName);
if (paimonPartition == null) {
@@ -345,7 +345,7 @@ public class PaimonExternalTable extends ExternalTable
implements MTMVRelatedTab
}
@Override
- public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context,
OptionalLong snapshotId)
+ public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context,
Optional<MvccSnapshot> snapshot)
throws AnalysisException {
return new MTMVVersionSnapshot(getLatestSnapshotIdFromCache());
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java
index ea15c84d1b9..95a8717e01c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java
@@ -69,7 +69,7 @@ public class MTMVPartitionExprDateTrunc implements
MTMVPartitionExprService {
String.format("timeUnit not support: %s, only support:
%s", this.timeUnit, timeUnits));
}
MTMVRelatedTableIf relatedTable = mvPartitionInfo.getRelatedTable();
- PartitionType partitionType = relatedTable.getPartitionType();
+ PartitionType partitionType =
relatedTable.getPartitionType(Optional.empty());
if (partitionType == PartitionType.RANGE) {
Type partitionColumnType = MTMVPartitionUtil
.getPartitionColumnType(mvPartitionInfo.getRelatedTable(),
mvPartitionInfo.getRelatedCol());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
index b3cd239269a..7eae44db0af 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java
@@ -25,6 +25,7 @@ import org.apache.doris.datasource.CatalogMgr;
import com.google.gson.annotations.SerializedName;
import java.util.List;
+import java.util.Optional;
/**
* MTMVPartitionInfo
@@ -115,7 +116,7 @@ public class MTMVPartitionInfo {
if (partitionType == MTMVPartitionType.SELF_MANAGE) {
throw new AnalysisException("partitionType is: " + partitionType);
}
- List<Column> partitionColumns =
getRelatedTable().getPartitionColumns();
+ List<Column> partitionColumns =
getRelatedTable().getPartitionColumns(Optional.empty());
for (int i = 0; i < partitionColumns.size(); i++) {
if
(partitionColumns.get(i).getName().equalsIgnoreCase(relatedCol)) {
return i;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
index 1bbc51fb004..8ba022de415 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
@@ -50,7 +50,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
-import java.util.OptionalLong;
+import java.util.Optional;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -330,7 +330,7 @@ public class MTMVPartitionUtil {
}
for (String relatedPartitionName : relatedPartitionNames) {
MTMVSnapshotIf relatedPartitionCurrentSnapshot = relatedTable
- .getPartitionSnapshot(relatedPartitionName, context,
OptionalLong.empty());
+ .getPartitionSnapshot(relatedPartitionName, context,
Optional.empty());
if (!mtmv.getRefreshSnapshot()
.equalsWithRelatedPartition(mtmvPartitionName,
relatedPartitionName,
relatedPartitionCurrentSnapshot)) {
@@ -447,7 +447,7 @@ public class MTMVPartitionUtil {
if (!baseTable.needAutoRefresh()) {
return true;
}
- MTMVSnapshotIf baseTableCurrentSnapshot =
baseTable.getTableSnapshot(context, OptionalLong.empty());
+ MTMVSnapshotIf baseTableCurrentSnapshot =
baseTable.getTableSnapshot(context, Optional.empty());
return mtmv.getRefreshSnapshot()
.equalsWithBaseTable(mtmvPartitionName, new
BaseTableInfo(baseTable), baseTableCurrentSnapshot);
}
@@ -483,7 +483,7 @@ public class MTMVPartitionUtil {
MTMVRelatedTableIf relatedTable =
mtmv.getMvPartitionInfo().getRelatedTable();
for (String relatedPartitionName : relatedPartitionNames) {
MTMVSnapshotIf partitionSnapshot = relatedTable
- .getPartitionSnapshot(relatedPartitionName, context,
OptionalLong.empty());
+ .getPartitionSnapshot(relatedPartitionName, context,
Optional.empty());
refreshPartitionSnapshot.getPartitions()
.put(relatedPartitionName, partitionSnapshot);
}
@@ -498,13 +498,13 @@ public class MTMVPartitionUtil {
continue;
}
refreshPartitionSnapshot.addTableSnapshot(baseTableInfo,
- ((MTMVRelatedTableIf) table).getTableSnapshot(context,
OptionalLong.empty()));
+ ((MTMVRelatedTableIf) table).getTableSnapshot(context,
Optional.empty()));
}
return refreshPartitionSnapshot;
}
public static Type getPartitionColumnType(MTMVRelatedTableIf relatedTable,
String col) throws AnalysisException {
- List<Column> partitionColumns = relatedTable.getPartitionColumns();
+ List<Column> partitionColumns =
relatedTable.getPartitionColumns(Optional.empty());
for (Column column : partitionColumns) {
if (column.getName().equals(col)) {
return column.getType();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescInitGenerator.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescInitGenerator.java
index ef3100dec4c..c6b4e331184 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescInitGenerator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescInitGenerator.java
@@ -20,7 +20,7 @@ package org.apache.doris.mtmv;
import org.apache.doris.common.AnalysisException;
import java.util.Map;
-import java.util.OptionalLong;
+import java.util.Optional;
/**
* get all related partition descs
@@ -30,6 +30,6 @@ public class MTMVRelatedPartitionDescInitGenerator implements
MTMVRelatedPartiti
@Override
public void apply(MTMVPartitionInfo mvPartitionInfo, Map<String, String>
mvProperties,
RelatedPartitionDescResult lastResult) throws AnalysisException {
-
lastResult.setItems(mvPartitionInfo.getRelatedTable().getAndCopyPartitionItems(OptionalLong.empty()));
+
lastResult.setItems(mvPartitionInfo.getRelatedTable().getAndCopyPartitionItems(Optional.empty()));
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGenerator.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGenerator.java
index 76e20ef70f5..325fab819d9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGenerator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGenerator.java
@@ -31,6 +31,7 @@ import com.google.common.collect.Sets;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Optional;
import java.util.Set;
/**
@@ -45,7 +46,7 @@ public class MTMVRelatedPartitionDescRollUpGenerator
implements MTMVRelatedParti
return;
}
MTMVRelatedTableIf relatedTable = mvPartitionInfo.getRelatedTable();
- PartitionType partitionType = relatedTable.getPartitionType();
+ PartitionType partitionType =
relatedTable.getPartitionType(Optional.empty());
if (partitionType == PartitionType.RANGE) {
lastResult.setDescs(rollUpRange(lastResult.getDescs(),
mvPartitionInfo));
} else if (partitionType == PartitionType.LIST) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java
index e18784ae253..c4261aa78f1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java
@@ -23,10 +23,11 @@ import org.apache.doris.catalog.PartitionType;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
+import org.apache.doris.datasource.mvcc.MvccSnapshot;
import java.util.List;
import java.util.Map;
-import java.util.OptionalLong;
+import java.util.Optional;
import java.util.Set;
/**
@@ -39,32 +40,35 @@ public interface MTMVRelatedTableIf extends TableIf {
* Note: This method is called every time there is a refresh and
transparent rewrite,
* so if this method is slow, it will significantly reduce query
performance
*
- * @param snapshotId
+ * @param snapshot
* @return partitionName->PartitionItem
*/
- Map<String, PartitionItem> getAndCopyPartitionItems(OptionalLong
snapshotId) throws AnalysisException;
+ Map<String, PartitionItem> getAndCopyPartitionItems(Optional<MvccSnapshot>
snapshot) throws AnalysisException;
/**
* getPartitionType LIST/RANGE/UNPARTITIONED
*
+ * @param snapshot
* @return
*/
- PartitionType getPartitionType();
+ PartitionType getPartitionType(Optional<MvccSnapshot> snapshot);
/**
* getPartitionColumnNames
*
+ * @param snapshot
* @return
* @throws DdlException
*/
- Set<String> getPartitionColumnNames() throws DdlException;
+ Set<String> getPartitionColumnNames(Optional<MvccSnapshot> snapshot)
throws DdlException;
/**
* getPartitionColumns
*
+ * @param snapshot
* @return
*/
- List<Column> getPartitionColumns();
+ List<Column> getPartitionColumns(Optional<MvccSnapshot> snapshot);
/**
* getPartitionSnapshot
@@ -72,14 +76,14 @@ public interface MTMVRelatedTableIf extends TableIf {
* If snapshots have already been obtained in bulk in the context,
* the results should be obtained directly from the context
*
- * @param snapshotId
+ * @param snapshot
* @param partitionName
* @param context
* @return partition snapshot at current time
* @throws AnalysisException
*/
- MTMVSnapshotIf getPartitionSnapshot(String partitionName,
MTMVRefreshContext context, OptionalLong snapshotId)
- throws AnalysisException;
+ MTMVSnapshotIf getPartitionSnapshot(String partitionName,
MTMVRefreshContext context,
+ Optional<MvccSnapshot> snapshot) throws AnalysisException;
/**
* getTableSnapshot
@@ -87,12 +91,13 @@ public interface MTMVRelatedTableIf extends TableIf {
* If snapshots have already been obtained in bulk in the context,
* the results should be obtained directly from the context
*
- * @param snapshotId
+ * @param snapshot
* @param context
* @return table snapshot at current time
* @throws AnalysisException
*/
- MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context, OptionalLong
snapshotId) throws AnalysisException;
+ MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context,
Optional<MvccSnapshot> snapshot)
+ throws AnalysisException;
/**
* Does the current type of table allow timed triggering
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
index a5c966370f0..17ae5883063 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
@@ -70,6 +70,7 @@ import org.apache.doris.statistics.StatisticsBuilder;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
+import org.apache.commons.collections.MapUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -462,6 +463,13 @@ public class CascadesContext implements ScheduleContext {
return tableNames;
}
+ public Map<List<String>, TableIf> getOrExtractTables(LogicalPlan
logicalPlan) {
+ if (MapUtils.isEmpty(tables)) {
+ extractTables(logicalPlan);
+ }
+ return tables;
+ }
+
private Set<List<String>> extractTableNamesFromHaving(LogicalHaving<?>
having) {
Set<SubqueryExpr> subqueryExprs = having.getPredicate()
.collect(SubqueryExpr.class::isInstance);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index 8fb0f6e7797..c7478411a5d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -216,7 +216,7 @@ public class NereidsPlanner extends Planner {
plan = preprocess(plan);
initCascadesContext(plan, requireProperties);
-
+
statementContext.loadSnapshots(cascadesContext.getOrExtractTables(plan));
try (Lock lock = new Lock(plan, cascadesContext)) {
Plan resultPlan = planWithoutLock(plan, explainLevel,
showPlanProcess, requireProperties);
lockCallback.accept(resultPlan);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
index 6b6e335b74a..b172f9dc591 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
@@ -24,6 +24,9 @@ import org.apache.doris.common.FormatOptions;
import org.apache.doris.common.Id;
import org.apache.doris.common.IdGenerator;
import org.apache.doris.common.Pair;
+import org.apache.doris.datasource.mvcc.MvccSnapshot;
+import org.apache.doris.datasource.mvcc.MvccTable;
+import org.apache.doris.datasource.mvcc.MvccTableInfo;
import org.apache.doris.nereids.hint.Hint;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.rules.analysis.ColumnAliasGenerator;
@@ -174,6 +177,8 @@ public class StatementContext implements Closeable {
private Backend groupCommitMergeBackend;
+ private final Map<MvccTableInfo, MvccSnapshot> snapshots =
Maps.newHashMap();
+
private boolean privChecked;
public StatementContext() {
@@ -510,6 +515,32 @@ public class StatementContext implements Closeable {
this.plannerHooks.add(plannerHook);
}
+ /**
+ * Load snapshot information of mvcc
+ *
+ * @param tables Tables used in queries
+ */
+ public void loadSnapshots(Map<List<String>, TableIf> tables) {
+ if (tables == null) {
+ return;
+ }
+ for (TableIf tableIf : tables.values()) {
+ if (tableIf instanceof MvccTable) {
+ snapshots.put(new MvccTableInfo(tableIf), ((MvccTable)
tableIf).loadSnapshot());
+ }
+ }
+ }
+
+ /**
+ * Obtain snapshot information of mvcc
+ *
+ * @param mvccTable mvccTable
+ * @return MvccSnapshot
+ */
+ public MvccSnapshot getSnapshot(MvccTable mvccTable) {
+ return snapshots.get(new MvccTableInfo(mvccTable));
+ }
+
private static class CloseableResource implements Closeable {
public final String resourceName;
public final String threadName;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
index a659c2f9990..484abd11f01 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
@@ -481,13 +481,13 @@ public class MaterializedViewUtils {
return null;
}
MTMVRelatedTableIf relatedTable = (MTMVRelatedTableIf) table;
- PartitionType type = relatedTable.getPartitionType();
+ PartitionType type =
relatedTable.getPartitionType(Optional.empty());
if (PartitionType.UNPARTITIONED.equals(type)) {
context.addFailReason(String.format("related base table is not
partition table, the table is %s",
table.getName()));
return null;
}
- Set<Column> partitionColumnSet = new
HashSet<>(relatedTable.getPartitionColumns());
+ Set<Column> partitionColumnSet = new
HashSet<>(relatedTable.getPartitionColumns(Optional.empty()));
Column mvReferenceColumn =
contextPartitionColumn.getColumn().get();
Expr definExpr = mvReferenceColumn.getDefineExpr();
if (definExpr instanceof SlotRef) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java
index d5021938307..4bbb0a8aa76 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java
@@ -36,7 +36,7 @@ import org.apache.commons.collections.CollectionUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import java.util.OptionalLong;
+import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -75,7 +75,7 @@ public class PruneFileScanPartition extends
OneRewriteRuleFactory {
LogicalFilter<LogicalFileScan> filter, LogicalFileScan scan,
CascadesContext ctx) {
Map<String, PartitionItem> selectedPartitionItems = Maps.newHashMap();
// todo: real snapshotId
- if
(CollectionUtils.isEmpty(externalTable.getPartitionColumns(OptionalLong.empty())))
{
+ if
(CollectionUtils.isEmpty(externalTable.getPartitionColumns(Optional.empty()))) {
// non partitioned table, return NOT_PRUNED.
// non partition table will be handled in HiveScanNode.
return SelectedPartitions.NOT_PRUNED;
@@ -84,7 +84,7 @@ public class PruneFileScanPartition extends
OneRewriteRuleFactory {
.stream()
.collect(Collectors.toMap(slot ->
slot.getName().toLowerCase(), Function.identity()));
// todo: real snapshotId
- List<Slot> partitionSlots =
externalTable.getPartitionColumns(OptionalLong.empty())
+ List<Slot> partitionSlots =
externalTable.getPartitionColumns(Optional.empty())
.stream()
.map(column -> scanOutput.get(column.getName().toLowerCase()))
.collect(Collectors.toList());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
index de5e188d5a6..8c44b42a5cc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
@@ -351,7 +351,7 @@ public class CreateMTMVInfo {
allPartitionDescs.size(),
ctx.getSessionVariable().getCreateTablePartitionMaxNum()));
}
try {
- PartitionType type = relatedTable.getPartitionType();
+ PartitionType type =
relatedTable.getPartitionType(Optional.empty());
if (type == PartitionType.RANGE) {
return new
RangePartitionDesc(Lists.newArrayList(mvPartitionInfo.getPartitionCol()),
allPartitionDescs);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java
index 427e2368e7a..c4117e8608e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java
@@ -54,6 +54,7 @@ import org.apache.doris.qe.SessionVariable;
import com.google.common.collect.Sets;
import java.util.List;
+import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@@ -147,7 +148,7 @@ public class MTMVPartitionDefinition {
MTMVRelatedTableIf mtmvBaseRealtedTable =
MTMVUtil.getRelatedTable(relatedTableInfo.getTableInfo());
Set<String> partitionColumnNames =
Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
try {
-
partitionColumnNames.addAll(mtmvBaseRealtedTable.getPartitionColumnNames());
+
partitionColumnNames.addAll(mtmvBaseRealtedTable.getPartitionColumnNames(Optional.empty()));
} catch (DdlException e) {
throw new AnalysisException(e.getMessage(), e);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
index 010c30d915d..96b8e032d11 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
@@ -36,7 +36,6 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
-import java.util.OptionalLong;
/**
* Logical file scan for external catalog.
@@ -64,7 +63,7 @@ public class LogicalFileScan extends LogicalCatalogRelation {
Optional<TableSample> tableSample,
Optional<TableSnapshot> tableSnapshot) {
// todo: real snapshotId
this(id, table, qualifier, Optional.empty(), Optional.empty(),
- table.initSelectedPartitions(OptionalLong.empty()),
tableSample, tableSnapshot);
+ table.initSelectedPartitions(Optional.empty()), tableSample,
tableSnapshot);
}
public SelectedPartitions getSelectedPartitions() {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
index 96ac59b8121..e5d2e21a8db 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
@@ -35,7 +35,7 @@ import org.junit.Before;
import org.junit.Test;
import java.util.List;
-import java.util.OptionalLong;
+import java.util.Optional;
import java.util.Set;
public class MTMVPartitionUtilTest {
@@ -113,7 +113,7 @@ public class MTMVPartitionUtilTest {
minTimes = 0;
result = true;
- baseOlapTable.getTableSnapshot((MTMVRefreshContext) any,
(OptionalLong) any);
+ baseOlapTable.getTableSnapshot((MTMVRefreshContext) any,
(Optional) any);
minTimes = 0;
result = baseSnapshotIf;
@@ -133,7 +133,7 @@ public class MTMVPartitionUtilTest {
minTimes = 0;
result = true;
- baseOlapTable.getPartitionSnapshot(anyString,
(MTMVRefreshContext) any, (OptionalLong) any);
+ baseOlapTable.getPartitionSnapshot(anyString,
(MTMVRefreshContext) any, (Optional) any);
minTimes = 0;
result = baseSnapshotIf;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
