This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8278b16b934 [enhance](mtmv)Provide a method to determine whether
incremental updates of multiple tables are allowed (#41691)
8278b16b934 is described below
commit 8278b16b934e4b2a60835aa994dfeffffb458856
Author: zhangdong <493738...@qq.com>
AuthorDate: Fri Oct 11 16:58:11 2024 +0800
[enhance](mtmv)Provide a method to determine whether incremental updates of
multiple tables are allowed (#41691)
Currently, only range partitioning is supported, and it must be a
dynamic partition or auto partition.
At the same time, it is required that the partitioning rules of
different tables be consistent
Think about the list partition carefully before making any decisions
---
.../apache/doris/analysis/PartitionExprUtil.java | 25 +-
.../doris/common/util/DynamicPartitionUtil.java | 18 ++
.../apache/doris/mtmv/MTMVPartitionCheckUtil.java | 144 +++++++++++
.../doris/mtmv/MTMVPartitionCheckUtilTest.java | 269 +++++++++++++++++++++
4 files changed, 454 insertions(+), 2 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java
index 420bee53e18..9302b6485aa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java
@@ -25,6 +25,7 @@ import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.thrift.TNullableStringLiteral;
+import com.google.common.base.Objects;
import com.google.common.collect.Maps;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -87,7 +88,7 @@ public class PartitionExprUtil {
} else {
throw new AnalysisException("now range partition only support
date_trunc/date_floor/date_ceil.");
}
- return partitionExprUtil.new FunctionIntervalInfo(timeUnit, interval);
+ return partitionExprUtil.new FunctionIntervalInfo(fnName, timeUnit,
interval);
}
public static DateLiteral getRangeEnd(DateLiteral beginTime,
FunctionIntervalInfo intervalInfo)
@@ -250,12 +251,32 @@ public class PartitionExprUtil {
}
public class FunctionIntervalInfo {
+ public String fnName;
public String timeUnit;
public long interval;
- public FunctionIntervalInfo(String timeUnit, long interval) {
+ public FunctionIntervalInfo(String fnName, String timeUnit, long
interval) {
+ this.fnName = fnName;
this.timeUnit = timeUnit;
this.interval = interval;
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ FunctionIntervalInfo that = (FunctionIntervalInfo) o;
+ return interval == that.interval && Objects.equal(fnName,
that.fnName)
+ && Objects.equal(timeUnit, that.timeUnit);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(fnName, timeUnit, interval);
+ }
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
index a713df0427f..ba716bcb6f7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
@@ -46,6 +46,7 @@ import org.apache.doris.policy.StoragePolicy;
import org.apache.doris.resource.Tag;
import org.apache.doris.thrift.TStorageMedium;
+import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.Maps;
@@ -1001,5 +1002,22 @@ public class DynamicPartitionUtil {
// TODO Auto-generated method stub
return super.toString();
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ StartOfDate that = (StartOfDate) o;
+ return month == that.month && day == that.day && dayOfWeek ==
that.dayOfWeek;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(month, day, dayOfWeek);
+ }
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionCheckUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionCheckUtil.java
new file mode 100644
index 00000000000..1f65880583c
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionCheckUtil.java
@@ -0,0 +1,144 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.mtmv;
+
+import org.apache.doris.analysis.PartitionExprUtil;
+import org.apache.doris.analysis.PartitionExprUtil.FunctionIntervalInfo;
+import org.apache.doris.catalog.DynamicPartitionProperty;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.PartitionType;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Pair;
+import org.apache.doris.common.util.DynamicPartitionUtil;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+
+import java.util.List;
+import java.util.Objects;
+
+public class MTMVPartitionCheckUtil {
+ /**
+ * Check if the partitioning method of the table meets the requirements
for multi table partitioning updates
+ *
+ * @param relatedTable base table of materialized view
+ * @return Inspection results and reasons
+ */
+ public static Pair<Boolean, String>
checkIfAllowMultiTablePartitionRefresh(MTMVRelatedTableIf relatedTable) {
+ if (!(relatedTable instanceof OlapTable)) {
+ return Pair.of(false, "only support OlapTable");
+ }
+ OlapTable olapTable = (OlapTable) relatedTable;
+ if (olapTable.getPartitionType() != PartitionType.RANGE) {
+ return Pair.of(false, "only support range partition");
+ }
+ boolean isDynamicOrAuto = isDynamicPartition(olapTable) ||
isAutoPartition(olapTable);
+ if (!isDynamicOrAuto) {
+ return Pair.of(false, "only support dynamic/auto partition");
+ }
+ return Pair.of(true, "");
+ }
+
+ /**
+ * Compare whether the partitioning rules of two tables are consistent
+ *
+ * @param originalTable partition table of materialized view
+ * @param relatedTable Partition refresh table for materialized views
+ * @return Inspection results and reasons
+ * @throws AnalysisException The preconditions are not met
+ */
+ public static Pair<Boolean, String>
compareOriginalTableAndRelatedTable(OlapTable originalTable,
+ OlapTable relatedTable) throws AnalysisException {
+ if (isDynamicPartition(originalTable)) {
+ return compareDynamicPartition(originalTable, relatedTable);
+ } else if (isAutoPartition(originalTable)) {
+ return compareAutoPartition(originalTable, relatedTable);
+ } else {
+ throw new AnalysisException("only support dynamic/auto partition");
+ }
+ }
+
+ /**
+ * Determine which related table partitioning rules are consistent with
the original table
+ *
+ * @param originalTable partition table of materialized view
+ * @param relatedTables Partition refresh table for materialized views
+ * @return Inspection results and reasons
+ * @throws AnalysisException The preconditions are not met
+ */
+ public static List<Pair<Boolean, String>>
compareOriginalTableAndRelatedTables(OlapTable originalTable,
+ List<OlapTable> relatedTables) throws AnalysisException {
+ List<Pair<Boolean, String>> res =
Lists.newArrayListWithCapacity(relatedTables.size());
+ for (OlapTable relatedTable : relatedTables) {
+ res.add(compareOriginalTableAndRelatedTable(originalTable,
relatedTable));
+ }
+ return res;
+ }
+
+ @VisibleForTesting
+ public static Pair<Boolean, String> compareDynamicPartition(OlapTable
originalTable,
+ OlapTable relatedTable) throws AnalysisException {
+ if (!isDynamicPartition(relatedTable)) {
+ return Pair.of(false, "relatedTable is not dynamic partition.");
+ }
+ DynamicPartitionProperty originalDynamicProperty =
originalTable.getTableProperty()
+ .getDynamicPartitionProperty();
+ DynamicPartitionProperty relatedDynamicProperty =
relatedTable.getTableProperty().getDynamicPartitionProperty();
+ if (originalDynamicProperty == null || relatedDynamicProperty == null)
{
+ throw new AnalysisException("dynamicProperty is null");
+ }
+ if (originalDynamicProperty.getTimeZone() !=
relatedDynamicProperty.getTimeZone()) {
+ return Pair.of(false, "timeZone not equal.");
+ }
+ if (originalDynamicProperty.getTimeUnit() !=
relatedDynamicProperty.getTimeUnit()) {
+ return Pair.of(false, "timeUnit not equal.");
+ }
+ if
(!originalDynamicProperty.getStartOfMonth().equals(relatedDynamicProperty.getStartOfMonth()))
{
+ return Pair.of(false, "startOfMonth not equal.");
+ }
+ if
(!originalDynamicProperty.getStartOfWeek().equals(relatedDynamicProperty.getStartOfWeek()))
{
+ return Pair.of(false, "startOfWeek not equal.");
+ }
+ return Pair.of(true, "");
+ }
+
+ @VisibleForTesting
+ public static Pair<Boolean, String> compareAutoPartition(OlapTable
originalTable,
+ OlapTable relatedTable) throws AnalysisException {
+ if (!isDynamicPartition(relatedTable)) {
+ return Pair.of(false, "relatedTable is not dynamic partition.");
+ }
+ FunctionIntervalInfo originalFunctionIntervalInfo =
PartitionExprUtil.getFunctionIntervalInfo(
+ originalTable.getPartitionInfo().getPartitionExprs(),
originalTable.getPartitionType());
+ FunctionIntervalInfo relatedFunctionIntervalInfo =
PartitionExprUtil.getFunctionIntervalInfo(
+ relatedTable.getPartitionInfo().getPartitionExprs(),
relatedTable.getPartitionType());
+ boolean equals = Objects.equals(originalFunctionIntervalInfo,
relatedFunctionIntervalInfo);
+ if (!equals) {
+ return Pair.of(false, "functionIntervalInfo not equal.");
+ }
+ return Pair.of(true, "");
+ }
+
+ private static boolean isDynamicPartition(OlapTable olapTable) {
+ return DynamicPartitionUtil.isDynamicPartitionTable(olapTable);
+ }
+
+ private static boolean isAutoPartition(OlapTable olapTable) {
+ return olapTable.getPartitionInfo().enableAutomaticPartition();
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionCheckUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionCheckUtilTest.java
new file mode 100644
index 00000000000..224816de4ae
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionCheckUtilTest.java
@@ -0,0 +1,269 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.mtmv;
+
+import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.PartitionExprUtil;
+import org.apache.doris.catalog.DynamicPartitionProperty;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.PartitionInfo;
+import org.apache.doris.catalog.PartitionType;
+import org.apache.doris.catalog.TableProperty;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.MetaNotFoundException;
+import org.apache.doris.common.Pair;
+import org.apache.doris.common.util.DynamicPartitionUtil;
+import org.apache.doris.common.util.DynamicPartitionUtil.StartOfDate;
+import org.apache.doris.datasource.hive.HMSExternalTable;
+
+import com.google.common.collect.Lists;
+import mockit.Expectations;
+import mockit.Mocked;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+
+public class MTMVPartitionCheckUtilTest {
+ @Mocked
+ private HMSExternalTable hmsExternalTable;
+ @Mocked
+ private OlapTable originalTable;
+ @Mocked
+ private OlapTable relatedTable;
+ @Mocked
+ private DynamicPartitionUtil dynamicPartitionUtil;
+ @Mocked
+ private PartitionExprUtil partitionExprUtil;
+ @Mocked
+ private PartitionInfo originalPartitionInfo;
+ @Mocked
+ private PartitionInfo relatedPartitionInfo;
+ @Mocked
+ private TableProperty originalTableProperty;
+ @Mocked
+ private TableProperty relatedTableProperty;
+ @Mocked
+ private DynamicPartitionProperty originalDynamicPartitionProperty;
+ @Mocked
+ private DynamicPartitionProperty relatedDynamicPartitionProperty;
+ @Mocked
+ private Expr expr1;
+ private ArrayList<Expr> originalExprs = Lists.newArrayList();
+ private ArrayList<Expr> relatedExprs = Lists.newArrayList(expr1);
+
+
+ @Before
+ public void setUp()
+ throws NoSuchMethodException, SecurityException,
AnalysisException, DdlException, MetaNotFoundException {
+
+ new Expectations() {
+ {
+ originalTable.getPartitionInfo();
+ minTimes = 0;
+ result = originalPartitionInfo;
+
+ originalTable.getPartitionType();
+ minTimes = 0;
+ result = PartitionType.RANGE;
+
+ originalTable.getTableProperty();
+ minTimes = 0;
+ result = originalTableProperty;
+
+ originalTableProperty.getDynamicPartitionProperty();
+ minTimes = 0;
+ result = originalDynamicPartitionProperty;
+
+ relatedTable.getPartitionInfo();
+ minTimes = 0;
+ result = relatedPartitionInfo;
+
+ relatedTable.getTableProperty();
+ minTimes = 0;
+ result = relatedTableProperty;
+
+ relatedTableProperty.getDynamicPartitionProperty();
+ minTimes = 0;
+ result = relatedDynamicPartitionProperty;
+
+ dynamicPartitionUtil.isDynamicPartitionTable(relatedTable);
+ minTimes = 0;
+ result = true;
+
+ originalDynamicPartitionProperty.getStartOfMonth();
+ minTimes = 0;
+ result = new StartOfDate(1, 1, 1);
+
+ relatedDynamicPartitionProperty.getStartOfMonth();
+ minTimes = 0;
+ result = new StartOfDate(1, 1, 1);
+
+ relatedDynamicPartitionProperty.getStartOfWeek();
+ minTimes = 0;
+ result = new StartOfDate(1, 1, 1);
+
+ originalDynamicPartitionProperty.getStartOfWeek();
+ minTimes = 0;
+ result = new StartOfDate(1, 1, 1);
+
+ originalPartitionInfo.getPartitionExprs();
+ minTimes = 0;
+ result = originalExprs;
+
+ relatedPartitionInfo.getPartitionExprs();
+ minTimes = 0;
+ result = relatedExprs;
+ }
+ };
+ }
+
+ @Test
+ public void testCheckIfAllowMultiTablePartitionRefreshNotOlapTable() {
+ Pair<Boolean, String> res =
MTMVPartitionCheckUtil.checkIfAllowMultiTablePartitionRefresh(
+ hmsExternalTable);
+ Assert.assertFalse(res.first);
+ }
+
+ @Test
+ public void testCheckIfAllowMultiTablePartitionRefreshNotRangePartition() {
+ new Expectations() {
+ {
+ originalTable.getPartitionType();
+ minTimes = 0;
+ result = PartitionType.LIST;
+ }
+ };
+ Pair<Boolean, String> res =
MTMVPartitionCheckUtil.checkIfAllowMultiTablePartitionRefresh(
+ originalTable);
+ Assert.assertFalse(res.first);
+ }
+
+ @Test
+ public void testCheckIfAllowMultiTablePartitionRefreshNotDynamicAndAuto() {
+ new Expectations() {
+ {
+ originalPartitionInfo.enableAutomaticPartition();
+ minTimes = 0;
+ result = false;
+
+ dynamicPartitionUtil.isDynamicPartitionTable(originalTable);
+ minTimes = 0;
+ result = false;
+ }
+ };
+ Pair<Boolean, String> res =
MTMVPartitionCheckUtil.checkIfAllowMultiTablePartitionRefresh(
+ originalTable);
+ Assert.assertFalse(res.first);
+ }
+
+ @Test
+ public void testCheckIfAllowMultiTablePartitionRefreshDynamic() {
+ new Expectations() {
+ {
+ originalPartitionInfo.enableAutomaticPartition();
+ minTimes = 0;
+ result = true;
+
+ dynamicPartitionUtil.isDynamicPartitionTable(originalTable);
+ minTimes = 0;
+ result = false;
+ }
+ };
+ Pair<Boolean, String> res =
MTMVPartitionCheckUtil.checkIfAllowMultiTablePartitionRefresh(
+ originalTable);
+ Assert.assertTrue(res.first);
+ }
+
+ @Test
+ public void testCheckIfAllowMultiTablePartitionRefreshAuto() {
+ new Expectations() {
+ {
+ originalPartitionInfo.enableAutomaticPartition();
+ minTimes = 0;
+ result = false;
+
+ dynamicPartitionUtil.isDynamicPartitionTable(originalTable);
+ minTimes = 0;
+ result = true;
+ }
+ };
+ Pair<Boolean, String> res =
MTMVPartitionCheckUtil.checkIfAllowMultiTablePartitionRefresh(
+ originalTable);
+ Assert.assertTrue(res.first);
+ }
+
+ @Test
+ public void testCompareDynamicPartition() throws AnalysisException {
+ Pair<Boolean, String> res =
MTMVPartitionCheckUtil.compareDynamicPartition(originalTable, relatedTable);
+ Assert.assertTrue(res.first);
+ }
+
+ @Test
+ public void testCompareDynamicPartitionNotEqual() throws AnalysisException
{
+ new Expectations() {
+ {
+ relatedDynamicPartitionProperty.getStartOfWeek();
+ minTimes = 0;
+ result = new StartOfDate(1, 1, 1);
+
+ originalDynamicPartitionProperty.getStartOfWeek();
+ minTimes = 0;
+ result = new StartOfDate(1, 1, 2);
+ }
+ };
+ Pair<Boolean, String> res =
MTMVPartitionCheckUtil.compareDynamicPartition(originalTable, relatedTable);
+ Assert.assertFalse(res.first);
+ }
+
+ @Test
+ public void testCompareAutpPartition() throws AnalysisException {
+ new Expectations() {
+ {
+ partitionExprUtil.getFunctionIntervalInfo(originalExprs,
(PartitionType) any);
+ minTimes = 0;
+ result = partitionExprUtil.new
FunctionIntervalInfo("datetrunc", "week", 1);
+
+ partitionExprUtil.getFunctionIntervalInfo(relatedExprs,
(PartitionType) any);
+ minTimes = 0;
+ result = partitionExprUtil.new
FunctionIntervalInfo("datetrunc", "week", 1);
+ }
+ };
+ Pair<Boolean, String> res =
MTMVPartitionCheckUtil.compareAutoPartition(originalTable, relatedTable);
+ Assert.assertTrue(res.first);
+ }
+
+ @Test
+ public void testCompareAutpPartitionNotEqual() throws AnalysisException {
+ new Expectations() {
+ {
+ partitionExprUtil.getFunctionIntervalInfo(originalExprs,
(PartitionType) any);
+ minTimes = 0;
+ result = partitionExprUtil.new
FunctionIntervalInfo("datetrunc", "week", 1);
+
+ partitionExprUtil.getFunctionIntervalInfo(relatedExprs,
(PartitionType) any);
+ minTimes = 0;
+ result = partitionExprUtil.new
FunctionIntervalInfo("datetrunc", "week", 2);
+ }
+ };
+ Pair<Boolean, String> res =
MTMVPartitionCheckUtil.compareAutoPartition(originalTable, relatedTable);
+ Assert.assertFalse(res.first);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
