morrySnow commented on code in PR #44911:
URL: https://github.com/apache/doris/pull/44911#discussion_r1870605077
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java:
##########
@@ -74,17 +74,17 @@ public Rule build() {
private SelectedPartitions pruneExternalPartitions(ExternalTable
externalTable,
LogicalFilter<LogicalFileScan> filter, LogicalFileScan scan,
CascadesContext ctx) {
Map<String, PartitionItem> selectedPartitionItems = Maps.newHashMap();
- // todo: real snapshotId
- if
(CollectionUtils.isEmpty(externalTable.getPartitionColumns(Optional.empty()))) {
+ if (CollectionUtils.isEmpty(externalTable.getPartitionColumns(
+
ConnectContext.get().getStatementContext().getSnapshot(externalTable)))) {
Review Comment:
get statementContext from cascadescontext
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonSource.java:
##########
@@ -36,7 +37,8 @@ public class PaimonSource {
public PaimonSource(TupleDescriptor desc) {
this.desc = desc;
this.paimonExtTable = (PaimonExternalTable) desc.getTable();
- this.originTable = paimonExtTable.getPaimonTable();
+ this.originTable = paimonExtTable.getPaimonTable(
+
ConnectContext.get().getStatementContext().getSnapshot(paimonExtTable));
Review Comment:
do we need check `ConnectContext.get() == null` and
`ConnectContext.get().getStatementContext()`. sometimes they do not exist
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java:
##########
@@ -74,17 +74,17 @@ public Rule build() {
private SelectedPartitions pruneExternalPartitions(ExternalTable
externalTable,
LogicalFilter<LogicalFileScan> filter, LogicalFileScan scan,
CascadesContext ctx) {
Map<String, PartitionItem> selectedPartitionItems = Maps.newHashMap();
- // todo: real snapshotId
- if
(CollectionUtils.isEmpty(externalTable.getPartitionColumns(Optional.empty()))) {
+ if (CollectionUtils.isEmpty(externalTable.getPartitionColumns(
+
ConnectContext.get().getStatementContext().getSnapshot(externalTable)))) {
// non partitioned table, return NOT_PRUNED.
// non partition table will be handled in HiveScanNode.
return SelectedPartitions.NOT_PRUNED;
}
Map<String, Slot> scanOutput = scan.getOutput()
.stream()
.collect(Collectors.toMap(slot ->
slot.getName().toLowerCase(), Function.identity()));
- // todo: real snapshotId
- List<Slot> partitionSlots =
externalTable.getPartitionColumns(Optional.empty())
+ List<Slot> partitionSlots = externalTable.getPartitionColumns(
+
ConnectContext.get().getStatementContext().getSnapshot(externalTable))
Review Comment:
ditto
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java:
##########
@@ -526,19 +526,37 @@ public void loadSnapshots(Map<List<String>, TableIf>
tables) {
}
for (TableIf tableIf : tables.values()) {
if (tableIf instanceof MvccTable) {
- snapshots.put(new MvccTableInfo(tableIf), ((MvccTable)
tableIf).loadSnapshot());
+ MvccTableInfo mvccTableInfo = new MvccTableInfo(tableIf);
+ // may be set by MTMV, we can not load again
+ if (!snapshots.containsKey(mvccTableInfo)) {
+ snapshots.put(mvccTableInfo, ((MvccTable)
tableIf).loadSnapshot());
+ }
}
}
}
/**
* Obtain snapshot information of mvcc
*
- * @param mvccTable mvccTable
+ * @param tableIf tableIf
* @return MvccSnapshot
*/
- public MvccSnapshot getSnapshot(MvccTable mvccTable) {
- return snapshots.get(new MvccTableInfo(mvccTable));
+ public Optional<MvccSnapshot> getSnapshot(TableIf tableIf) {
+ if (!(tableIf instanceof MvccTable)) {
+ return Optional.empty();
+ }
+ MvccTableInfo mvccTableInfo = new MvccTableInfo(tableIf);
+ return snapshots.containsKey(mvccTableInfo) ?
Optional.of(snapshots.get(mvccTableInfo)) : Optional.empty();
Review Comment:
could simplify by `Optional.ofNullable(snapshots.get(mvccTableInfo))`
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataCacheMgr.java:
##########
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.paimon;
+
+import java.util.concurrent.ExecutorService;
+
+public class PaimonMetadataCacheMgr {
+
+ private PaimonMetadataCache paimonMetadataCache;
+
+ public PaimonMetadataCacheMgr(ExecutorService executor) {
+ this.paimonMetadataCache = new PaimonMetadataCache(executor);
+ }
+
+ public PaimonMetadataCache getPaimonMetadataCache() {
+ return paimonMetadataCache;
+ }
+
+ public void removeCache(long catalogId) {
+ paimonMetadataCache.invalidateCatalogCache(catalogId);
+ }
+
+ public void invalidateCatalogCache(long catalogId) {
+ paimonMetadataCache.invalidateCatalogCache(catalogId);
+ }
Review Comment:
what's different between these two interface
##########
fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java:
##########
@@ -152,4 +169,107 @@ public static ListPartitionItem
toListPartitionItem(String partitionName, List<C
ListPartitionItem listPartitionItem = new
ListPartitionItem(Lists.newArrayList(key));
return listPartitionItem;
}
+
+ private static Type
paimonPrimitiveTypeToDorisType(org.apache.paimon.types.DataType dataType) {
+ int tsScale = 3; // default
+ switch (dataType.getTypeRoot()) {
+ case BOOLEAN:
+ return Type.BOOLEAN;
+ case INTEGER:
+ return Type.INT;
+ case BIGINT:
+ return Type.BIGINT;
+ case FLOAT:
+ return Type.FLOAT;
+ case DOUBLE:
+ return Type.DOUBLE;
+ case SMALLINT:
+ return Type.SMALLINT;
+ case TINYINT:
+ return Type.TINYINT;
+ case VARCHAR:
+ case BINARY:
+ case CHAR:
+ case VARBINARY:
+ return Type.STRING;
+ case DECIMAL:
+ DecimalType decimal = (DecimalType) dataType;
+ return ScalarType.createDecimalV3Type(decimal.getPrecision(),
decimal.getScale());
+ case DATE:
+ return ScalarType.createDateV2Type();
+ case TIMESTAMP_WITHOUT_TIME_ZONE:
+ if (dataType instanceof org.apache.paimon.types.TimestampType)
{
+ tsScale = ((org.apache.paimon.types.TimestampType)
dataType).getPrecision();
+ if (tsScale > 6) {
+ tsScale = 6;
+ }
+ } else if (dataType instanceof
org.apache.paimon.types.LocalZonedTimestampType) {
+ tsScale =
((org.apache.paimon.types.LocalZonedTimestampType) dataType).getPrecision();
+ if (tsScale > 6) {
+ tsScale = 6;
+ }
+ }
+ return ScalarType.createDatetimeV2Type(tsScale);
+ case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
Review Comment:
so data load into doris will lead to wrong time zone?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
