This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 63353dbcd4a02b0bfc8b82490ec3b54cff3f9e5e
Author: zhangdong <493738...@qq.com>
AuthorDate: Wed Feb 7 17:18:24 2024 +0800
[case](mtmv) MTMV hive case (#30930)
---
.../org/apache/doris/mtmv/MTMVPartitionUtil.java | 4 +-
.../data/mtmv_p0/test_hive_olap_mtmv.out | 17 ++
.../data/mtmv_p0/test_hive_refresh_mtmv.out | 17 ++
.../suites/mtmv_p0/test_hive_mtmv.groovy | 2 +-
.../suites/mtmv_p0/test_hive_olap_mtmv.groovy | 90 +++++++++++
.../suites/mtmv_p0/test_hive_refresh_mtmv.groovy | 180 +++++++++++++++++++++
6 files changed, 307 insertions(+), 3 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
index bcf182e4b1f..a4f3f4cb53e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
@@ -90,8 +90,8 @@ public class MTMVPartitionUtil {
*/
public static void alignMvPartition(MTMV mtmv, MTMVRelatedTableIf
relatedTable)
throws DdlException, AnalysisException {
- Map<Long, PartitionItem> relatedTableItems =
relatedTable.getPartitionItems();
- Map<Long, PartitionItem> mtmvItems = mtmv.getPartitionItems();
+ Map<Long, PartitionItem> relatedTableItems =
Maps.newHashMap(relatedTable.getPartitionItems());
+ Map<Long, PartitionItem> mtmvItems =
Maps.newHashMap(mtmv.getPartitionItems());
// drop partition of mtmv
for (Entry<Long, PartitionItem> entry : mtmvItems.entrySet()) {
long partitionId = getExistPartitionId(entry.getValue(),
relatedTableItems);
diff --git a/regression-test/data/mtmv_p0/test_hive_olap_mtmv.out
b/regression-test/data/mtmv_p0/test_hive_olap_mtmv.out
new file mode 100644
index 00000000000..d62b4f4edc7
--- /dev/null
+++ b/regression-test/data/mtmv_p0/test_hive_olap_mtmv.out
@@ -0,0 +1,17 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !refresh_complete --
+1 20230101 clz
+2 20230101 zhangsang
+3 20230101 \N
+4 20230102 \N
+5 20230102 \N
+6 20230102 \N
+
+-- !refresh_2 --
+1 20230101 clz
+2 20230101 zhangsang
+3 20230101 ff
+4 20230102 gg
+5 20230102 \N
+6 20230102 \N
+
diff --git a/regression-test/data/mtmv_p0/test_hive_refresh_mtmv.out
b/regression-test/data/mtmv_p0/test_hive_refresh_mtmv.out
new file mode 100644
index 00000000000..ec05013d615
--- /dev/null
+++ b/regression-test/data/mtmv_p0/test_hive_refresh_mtmv.out
@@ -0,0 +1,17 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !test --
+1 1 2020
+
+-- !mtmv_1 --
+1 1 2020
+
+-- !mtmv_2 --
+1 1 2020
+1 1 2020
+
+-- !task_error --
+FAILED
+
+-- !task_recover --
+SUCCESS
+
diff --git a/regression-test/suites/mtmv_p0/test_hive_mtmv.groovy
b/regression-test/suites/mtmv_p0/test_hive_mtmv.groovy
index cf34cfb616a..695cf0270b4 100644
--- a/regression-test/suites/mtmv_p0/test_hive_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_hive_mtmv.groovy
@@ -51,7 +51,7 @@ suite("test_hive_mtmv",
"p0,external,hive,external_docker,external_docker_hive")
sql """
REFRESH MATERIALIZED VIEW ${mvName} partitions(p_20230101);
"""
- jobName = getJobName(dbName, mvName);
+ def jobName = getJobName(dbName, mvName);
log.info(jobName)
waitingMTMVTaskFinished(jobName)
order_qt_refresh_one_partition "SELECT * FROM ${mvName} order by
id"
diff --git a/regression-test/suites/mtmv_p0/test_hive_olap_mtmv.groovy
b/regression-test/suites/mtmv_p0/test_hive_olap_mtmv.groovy
new file mode 100644
index 00000000000..de605f1a2b0
--- /dev/null
+++ b/regression-test/suites/mtmv_p0/test_hive_olap_mtmv.groovy
@@ -0,0 +1,90 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_hive_olap_mtmv",
"p0,external,hive,external_docker,external_docker_hive") {
+ String enabled = context.config.otherConfigs.get("enableHiveTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("diable Hive test.")
+ return;
+ }
+ String hms_port = context.config.otherConfigs.get("hms_port")
+ String catalog_name = "hive_olap_test_mtmv"
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """create catalog if not exists ${catalog_name} properties (
+ "type"="hms",
+ 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}'
+ );"""
+
+ def tableName = "t_test_base_mtmv_user"
+ sql """drop table if exists `${tableName}`"""
+ sql """
+ CREATE TABLE IF NOT EXISTS `${tableName}` (
+ id INT,
+ username VARCHAR(20)
+ )
+ DISTRIBUTED BY HASH(id) BUCKETS 10
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+ sql """
+ INSERT INTO ${tableName} VALUES(1,"clz"),(2,"zhangsang");
+ """
+
+ def mvName = "test_hive_olap_mtmv"
+ def dbName = "regression_test_mtmv_p0"
+ sql """drop materialized view if exists ${mvName};"""
+
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD DEFERRED REFRESH AUTO ON MANUAL
+ partition by(`part_col`)
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES ('replication_num' = '1')
+ AS
+ SELECT t1.`id`,t1.`part_col`,t2.`username` FROM
${catalog_name}.`default`.mtmv_base1 t1 left join ${tableName} t2 on t1.id =
t2.id;
+ """
+ def showPartitionsResult = sql """show partitions from ${mvName}"""
+ logger.info("showPartitionsResult: " + showPartitionsResult.toString())
+ assertTrue(showPartitionsResult.toString().contains("p_20230101"))
+ assertTrue(showPartitionsResult.toString().contains("p_20230102"))
+
+ def jobName = getJobName(dbName, mvName);
+
+ //refresh complete
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} complete
+ """
+ waitingMTMVTaskFinished(jobName)
+ order_qt_refresh_complete "SELECT * FROM ${mvName} order by id"
+
+ sql """
+ INSERT INTO ${tableName} VALUES(3,"ff"),(4,"gg");
+ """
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName}
+ """
+ waitingMTMVTaskFinished(jobName)
+ order_qt_refresh_2 "SELECT * FROM ${mvName} order by id"
+ sql """drop materialized view if exists ${mvName};"""
+
+ sql """drop catalog if exists ${catalog_name}"""
+
+}
+
diff --git a/regression-test/suites/mtmv_p0/test_hive_refresh_mtmv.groovy
b/regression-test/suites/mtmv_p0/test_hive_refresh_mtmv.groovy
new file mode 100644
index 00000000000..ce75e20a6de
--- /dev/null
+++ b/regression-test/suites/mtmv_p0/test_hive_refresh_mtmv.groovy
@@ -0,0 +1,180 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_hive_refresh_mtmv",
"p0,external,hive,external_docker,external_docker_hive") {
+ String enabled = context.config.otherConfigs.get("enableHiveTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("diable Hive test.")
+ return;
+ }
+ // prepare data in hive
+ def hive_database = "mtmv_test_db"
+ def hive_table = "test_hive_refresh_mtmv_t1"
+
+ def drop_table_str = """ drop table if exists
${hive_database}.${hive_table} """
+ def drop_database_str = """ drop database if exists ${hive_database}"""
+ def create_database_str = """ create database ${hive_database}"""
+ def create_table_str = """ CREATE TABLE ${hive_database}.${hive_table} (
+ user_id INT,
+ num INT
+ )
+ partitioned by(year int )
+ STORED AS ORC;
+ """
+ def add_partition_str = """
+ alter table ${hive_database}.${hive_table} add
if not exists
+ partition(year=2020);
+ """
+ def insert_str = """ insert into ${hive_database}.${hive_table}
PARTITION(year=2020) values(1,1)"""
+ logger.info("hive sql: " + drop_table_str)
+ hive_docker """ ${drop_table_str} """
+ logger.info("hive sql: " + drop_database_str)
+ hive_docker """ ${drop_database_str} """
+ logger.info("hive sql: " + create_database_str)
+ hive_docker """ ${create_database_str}"""
+ logger.info("hive sql: " + create_table_str)
+ hive_docker """ ${create_table_str} """
+ logger.info("hive sql: " + add_partition_str)
+ hive_docker """ ${add_partition_str} """
+ logger.info("hive sql: " + insert_str)
+ hive_docker """ ${insert_str} """
+
+
+ // prepare catalog
+ String hms_port = context.config.otherConfigs.get("hms_port")
+ String catalog_name = "hive_test_mtmv"
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """create catalog if not exists ${catalog_name} properties (
+ "type"="hms",
+ 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}'
+ );"""
+
+ order_qt_test "SELECT * FROM
${catalog_name}.${hive_database}.${hive_table}"
+
+
+ // prepare mtmv
+ def mvName = "test_hive_refresh_mtmv"
+ def dbName = "regression_test_mtmv_p0"
+ sql """drop materialized view if exists ${mvName};"""
+
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD DEFERRED REFRESH AUTO ON MANUAL
+ partition by(`year`)
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES ('replication_num' = '1')
+ AS
+ SELECT user_id,num,year FROM
${catalog_name}.${hive_database}.${hive_table};
+ """
+ def showPartitionsResult = sql """show partitions from ${mvName}"""
+ logger.info("showPartitionsResult: " + showPartitionsResult.toString())
+ assertTrue(showPartitionsResult.toString().contains("p_2020"))
+
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} complete
+ """
+ def jobName = getJobName(dbName, mvName);
+ waitingMTMVTaskFinished(jobName)
+ order_qt_mtmv_1 "SELECT * FROM ${mvName} order by user_id"
+
+ // hive data change
+ logger.info("hive sql: " + insert_str)
+ hive_docker """ ${insert_str} """
+ sql """
+ REFRESH catalog ${catalog_name}
+ """
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} complete
+ """
+ waitingMTMVTaskFinished(jobName)
+ order_qt_mtmv_2 "SELECT * FROM ${mvName} order by user_id"
+
+ // hive add partition
+ def add_partition2021_str = """
+ alter table
${hive_database}.${hive_table} add if not exists
+ partition(year=2021);
+ """
+ logger.info("hive sql: " + add_partition2021_str)
+ hive_docker """ ${add_partition2021_str} """
+ sql """
+ REFRESH catalog ${catalog_name}
+ """
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} complete
+ """
+ waitingMTMVTaskFinished(jobName)
+ showPartitionsResult = sql """show partitions from ${mvName}"""
+ logger.info("showPartitionsResult: " + showPartitionsResult.toString())
+ assertTrue(showPartitionsResult.toString().contains("p_2020"))
+ assertTrue(showPartitionsResult.toString().contains("p_2021"))
+
+ // hive drop partition
+ def drop_partition2021_str = """
+ alter table
${hive_database}.${hive_table} drop if exists
+ partition(year=2021);
+ """
+ logger.info("hive sql: " + drop_partition2021_str)
+ hive_docker """ ${drop_partition2021_str} """
+ sql """
+ REFRESH catalog ${catalog_name}
+ """
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} complete
+ """
+ waitingMTMVTaskFinished(jobName)
+ showPartitionsResult = sql """show partitions from ${mvName}"""
+ logger.info("showPartitionsResult: " + showPartitionsResult.toString())
+ assertTrue(showPartitionsResult.toString().contains("p_2020"))
+ assertFalse(showPartitionsResult.toString().contains("p_2021"))
+
+ // hive rename column name
+ def rename_column_str = """
+ alter table
${hive_database}.${hive_table} CHANGE COLUMN num num1 int;
+ """
+ logger.info("hive sql: " + rename_column_str)
+ hive_docker """ ${rename_column_str} """
+ sql """
+ REFRESH catalog ${catalog_name}
+ """
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} complete
+ """
+ waitingMTMVTaskFinishedNotNeedSuccess(jobName)
+ order_qt_task_error "select Status from tasks('type'='mv') where
JobName = '${jobName}' order by CreateTime DESC limit 1"
+
+ // hive recover column name
+ def recover_column_str = """
+ alter table ${hive_database}.${hive_table}
CHANGE COLUMN num1 num int;
+ """
+ logger.info("hive sql: " + recover_column_str)
+ hive_docker """ ${recover_column_str} """
+ sql """
+ REFRESH catalog ${catalog_name}
+ """
+ sql """
+ REFRESH MATERIALIZED VIEW ${mvName} complete
+ """
+ waitingMTMVTaskFinishedNotNeedSuccess(jobName)
+ order_qt_task_recover "select Status from tasks('type'='mv') where
JobName = '${jobName}' order by CreateTime DESC limit 1"
+
+ sql """drop materialized view if exists ${mvName};"""
+
+ sql """drop catalog if exists ${catalog_name}"""
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
