This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 63f6aec730b [enhance](mtmv)Disable mtmv list rollup (#38124)
63f6aec730b is described below
commit 63f6aec730b02242da6ee37a5013b437dcbc8f46
Author: zhangdong <493738...@qq.com>
AuthorDate: Fri Aug 9 17:09:41 2024 +0800
[enhance](mtmv)Disable mtmv list rollup (#38124)
Prohibit creating materialized views on partitions where the base table
is a list partition
---
.../doris/mtmv/MTMVPartitionExprDateTrunc.java | 2 +
.../mtmv_p0/test_hive_limit_partition_mtmv.out | 5 -
.../data/mtmv_p0/test_rollup_partition_mtmv.out | 37 ---
.../suites/mtmv_p0/test_build_mtmv.groovy | 12 +-
.../mtmv_p0/test_hive_limit_partition_mtmv.groovy | 45 ++-
.../mtmv_p0/test_rollup_partition_mtmv.groovy | 334 +--------------------
6 files changed, 39 insertions(+), 396 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java
index f16252cede0..0fcfaa909e6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java
@@ -78,6 +78,8 @@ public class MTMVPartitionExprDateTrunc implements
MTMVPartitionExprService {
"partitionColumnType should be date/datetime "
+ "when PartitionType is range and expr is
date_trunc");
}
+ } else {
+ throw new AnalysisException("date_trunc only support range
partition");
}
}
diff --git a/regression-test/data/mtmv_p0/test_hive_limit_partition_mtmv.out
b/regression-test/data/mtmv_p0/test_hive_limit_partition_mtmv.out
index 1a8731d6a1c..48e60bb46cc 100644
--- a/regression-test/data/mtmv_p0/test_hive_limit_partition_mtmv.out
+++ b/regression-test/data/mtmv_p0/test_hive_limit_partition_mtmv.out
@@ -11,11 +11,6 @@
2 20380101 sh
5 20380102 bj
--- !mtmv_datetrunc --
-1 20380101 bj
-2 20380101 sh
-5 20380102 bj
-
-- !select_base_table --
1 bj 20380101
2 sh 20380101
diff --git a/regression-test/data/mtmv_p0/test_rollup_partition_mtmv.out
b/regression-test/data/mtmv_p0/test_rollup_partition_mtmv.out
index 5f01fb0ee29..f828e65cc70 100644
--- a/regression-test/data/mtmv_p0/test_rollup_partition_mtmv.out
+++ b/regression-test/data/mtmv_p0/test_rollup_partition_mtmv.out
@@ -1,41 +1,4 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
--- !date_list_month --
-1 2020-01-01 2020-01-01
-2 2020-01-02 2020-01-02
-3 2020-02-01 2020-02-01
-
--- !date_list_month_partition_by_column --
-2020-01-01 1 2020-01-01 2020-01-01
-2020-01-01 2 2020-01-02 2020-01-02
-2020-02-01 3 2020-02-01 2020-02-01
-
--- !date_list_month_level --
-2020-01-01 1 2020-01-01 2020-01-01
-2020-01-02 2 2020-01-02 2020-01-02
-2020-02-01 3 2020-02-01 2020-02-01
-
--- !date_list_month_level_agg --
-2020-01-01 1 1
-2020-01-02 2 1
-2020-02-01 3 1
-
--- !date_list_month_level_agg_multi --
-2020-01-01 2020-01-01 1
-2020-01-02 2020-01-02 1
-2020-02-01 2020-02-01 1
-
--- !date_list_month_level_agg --
-2020-01-01 1
-2020-01-02 1
-2020-02-01 1
-
--- !date_list_year_partition_by_column --
-
--- !string_list_month --
-1 2020==01==01
-2 2020==01==02
-3 2020==02==01
-
-- !date_range_month --
1 2020-01-01 2020-01-01
2 2020-01-02 2020-01-02
diff --git a/regression-test/suites/mtmv_p0/test_build_mtmv.groovy
b/regression-test/suites/mtmv_p0/test_build_mtmv.groovy
index 580b9f9bbfc..ae1af18fdc5 100644
--- a/regression-test/suites/mtmv_p0/test_build_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_build_mtmv.groovy
@@ -500,12 +500,12 @@ suite("test_build_mtmv") {
sql """
DROP MATERIALIZED VIEW ${mvName}
"""
- def jobs = sql """select count(1) from jobs("type"="mv") where name=
'${jobName}'"""
- println jobs
- assertEquals(jobs.get(0).get(0), 0);
- def tasks = sql """select count(1) from tasks("type"="mv") where jobname =
'${jobName}'"""
- println tasks
- assertEquals(tasks.get(0).get(0), 0);
+ def jobs = sql """select * from jobs("type"="mv") where MvName=
'${mvName}'"""
+ log.info(jobs.toString())
+ assertEquals(0, jobs.size());
+ def tasks = sql """select * from tasks("type"="mv") where MvName =
'${mvName}'"""
+ log.info(tasks.toString())
+ assertEquals(0, tasks.size());
// test bitmap
sql """drop table if exists `${tableName}`"""
diff --git
a/regression-test/suites/mtmv_p0/test_hive_limit_partition_mtmv.groovy
b/regression-test/suites/mtmv_p0/test_hive_limit_partition_mtmv.groovy
index 1e6c49bb50a..ea5c2ff2844 100644
--- a/regression-test/suites/mtmv_p0/test_hive_limit_partition_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_hive_limit_partition_mtmv.groovy
@@ -115,34 +115,23 @@ suite("test_hive_limit_partition_mtmv",
"p0,external,hive,external_docker,extern
// date trunc
sql """drop materialized view if exists ${mvName};"""
- sql """
- CREATE MATERIALIZED VIEW ${mvName}
- BUILD DEFERRED REFRESH AUTO ON MANUAL
- partition by (date_trunc(`day`,'month'))
- DISTRIBUTED BY RANDOM BUCKETS 2
- PROPERTIES (
- 'replication_num' = '1',
- 'partition_sync_limit'='2',
- 'partition_sync_time_unit'='MONTH',
- 'partition_date_format'='%Y%m%d'
- )
- AS
- SELECT k1,day,region FROM
${catalog_name}.${hive_database}.${hive_table};
- """
- showPartitionsResult = sql """show partitions from ${mvName}"""
- logger.info("showPartitionsResult: " + showPartitionsResult.toString())
- assertEquals(1, showPartitionsResult.size())
- assertTrue(showPartitionsResult.toString().contains("_20380101"))
- assertTrue(showPartitionsResult.toString().contains("_20380102"))
-
- // refresh complete
- sql """
- REFRESH MATERIALIZED VIEW ${mvName} complete
- """
- jobName = getJobName(dbName, mvName);
- waitingMTMVTaskFinished(jobName)
- order_qt_mtmv_datetrunc "SELECT * FROM ${mvName} order by k1,day,region"
-
+ test {
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD DEFERRED REFRESH AUTO ON MANUAL
+ partition by (date_trunc(`day`,'month'))
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1',
+ 'partition_sync_limit'='2',
+ 'partition_sync_time_unit'='MONTH',
+ 'partition_date_format'='%Y%m%d'
+ )
+ AS
+ SELECT k1,day,region FROM
${catalog_name}.${hive_database}.${hive_table};
+ """
+ exception "only support"
+ }
// date type
sql """drop materialized view if exists ${mvName};"""
diff --git a/regression-test/suites/mtmv_p0/test_rollup_partition_mtmv.groovy
b/regression-test/suites/mtmv_p0/test_rollup_partition_mtmv.groovy
index 21c77bc1e24..514c784534c 100644
--- a/regression-test/suites/mtmv_p0/test_rollup_partition_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_rollup_partition_mtmv.groovy
@@ -46,326 +46,20 @@ suite("test_rollup_partition_mtmv") {
insert into ${tableName} values(1,"2020-01-01",
"2020-01-01"),(2,"2020-01-02", "2020-01-02"),(3,"2020-02-01", "2020-02-01");
"""
- // list date month
- sql """
- CREATE MATERIALIZED VIEW ${mvName}
- BUILD DEFERRED REFRESH AUTO ON MANUAL
- partition by (date_trunc(`k2`,'month'))
- DISTRIBUTED BY RANDOM BUCKETS 2
- PROPERTIES (
- 'replication_num' = '1'
- )
- AS
- SELECT * FROM ${tableName};
- """
- showPartitionsResult = sql """show partitions from ${mvName}"""
- logger.info("showPartitionsResult: " + showPartitionsResult.toString())
- assertEquals(2, showPartitionsResult.size())
-
- sql """
- REFRESH MATERIALIZED VIEW ${mvName} AUTO
- """
- def jobName = getJobName(dbName, mvName);
- log.info(jobName)
- waitingMTMVTaskFinished(jobName)
- order_qt_date_list_month "SELECT * FROM ${mvName} order by k1,k2"
-
- sql """drop materialized view if exists ${mvName};"""
- sql """
- CREATE MATERIALIZED VIEW ${mvName}
- BUILD IMMEDIATE REFRESH AUTO ON MANUAL
- partition by (month_alias)
- DISTRIBUTED BY RANDOM BUCKETS 2
- PROPERTIES (
- 'replication_num' = '1'
- )
- AS
- SELECT date_trunc(`k2`,'month') as month_alias, * FROM
${tableName};
- """
- def date_list_month_partitions = sql """show partitions from ${mvName}"""
- logger.info("showPartitionsResult: " +
date_list_month_partitions.toString())
- assertEquals(2, date_list_month_partitions.size())
- waitingMTMVTaskFinished(getJobName(dbName, mvName))
- order_qt_date_list_month_partition_by_column "SELECT * FROM ${mvName}"
-
- sql """drop materialized view if exists ${mvName};"""
- sql """
- CREATE MATERIALIZED VIEW ${mvName}
- BUILD IMMEDIATE REFRESH AUTO ON MANUAL
- partition by (date_trunc(month_alias, 'month'))
- DISTRIBUTED BY RANDOM BUCKETS 2
- PROPERTIES (
- 'replication_num' = '1'
- )
- AS
- SELECT date_trunc(`k2`,'day') as month_alias, * FROM ${tableName};
- """
- def date_list_month_partitions_level = sql """show partitions from
${mvName}"""
- logger.info("showPartitionsResult: " +
date_list_month_partitions_level.toString())
- assertEquals(2, date_list_month_partitions_level.size())
- waitingMTMVTaskFinished(getJobName(dbName, mvName))
- order_qt_date_list_month_level "SELECT * FROM ${mvName}"
-
-
- sql """drop materialized view if exists ${mvName};"""
- sql """
- CREATE MATERIALIZED VIEW ${mvName}
- BUILD IMMEDIATE REFRESH AUTO ON MANUAL
- partition by (date_trunc(month_alias, 'month'))
- DISTRIBUTED BY RANDOM BUCKETS 2
- PROPERTIES (
- 'replication_num' = '1'
- )
- AS
- SELECT date_trunc(`k2`,'day') as month_alias, k1, count(*) FROM
${tableName} group by month_alias, k1;
- """
- def date_list_month_partitions_level_agg = sql """show partitions from
${mvName}"""
- logger.info("showPartitionsResult: " +
date_list_month_partitions_level_agg.toString())
- assertEquals(2, date_list_month_partitions_level_agg.size())
- waitingMTMVTaskFinished(getJobName(dbName, mvName))
- order_qt_date_list_month_level_agg "SELECT * FROM ${mvName}"
-
-
- sql """drop materialized view if exists ${mvName};"""
- sql """
- CREATE MATERIALIZED VIEW ${mvName}
- BUILD IMMEDIATE REFRESH AUTO ON MANUAL
- partition by (date_trunc(month_alias, 'month'))
- DISTRIBUTED BY RANDOM BUCKETS 2
- PROPERTIES (
- 'replication_num' = '1'
- )
- AS
- SELECT date_trunc(`k2`,'day') as month_alias, k3, count(*) FROM
${tableName} group by date_trunc(`k2`,'day'), k3;
- """
- def date_list_month_partitions_level_agg_multi = sql """show partitions
from ${mvName}"""
- logger.info("showPartitionsResult: " +
date_list_month_partitions_level_agg_multi.toString())
- assertEquals(2, date_list_month_partitions_level_agg_multi.size())
- waitingMTMVTaskFinished(getJobName(dbName, mvName))
- order_qt_date_list_month_level_agg_multi "SELECT * FROM ${mvName}"
-
-
- sql """drop materialized view if exists ${mvName};"""
- sql """
- CREATE MATERIALIZED VIEW ${mvName}
- BUILD IMMEDIATE REFRESH AUTO ON MANUAL
- partition by (date_trunc(month_alias, 'month'))
- DISTRIBUTED BY RANDOM BUCKETS 2
- PROPERTIES (
- 'replication_num' = '1'
- )
- AS
- SELECT date_trunc(`k2`,'day') as month_alias, count(*) FROM
${tableName} group by k2;
- """
- def date_list_month_partitions_level_agg_direct = sql """show partitions
from ${mvName}"""
- logger.info("showPartitionsResult: " +
date_list_month_partitions_level_agg_direct.toString())
- assertEquals(2, date_list_month_partitions_level_agg_direct.size())
- waitingMTMVTaskFinished(getJobName(dbName, mvName))
- order_qt_date_list_month_level_agg "SELECT * FROM ${mvName}"
-
-
-
- // mv partition level should be higher or equal then query, should fail
- sql """drop materialized view if exists ${mvName};"""
- try {
- sql """
- CREATE MATERIALIZED VIEW ${mvName}
- BUILD IMMEDIATE REFRESH AUTO ON MANUAL
- partition by (date_trunc(month_alias, 'day'))
- DISTRIBUTED BY RANDOM BUCKETS 2
- PROPERTIES (
- 'replication_num' = '1'
- )
- AS
- SELECT date_trunc(`k2`,'month') as month_alias, * FROM
${tableName};
- """
- Assert.fail();
- } catch (Exception e) {
- log.info(e.getMessage())
- assertTrue(e.getMessage().contains("partition column time unit level
should be greater than sql select column"))
- }
-
- // mv partition use a column not in mv sql select, should fail
- sql """drop materialized view if exists ${mvName};"""
- try {
- sql """
- CREATE MATERIALIZED VIEW ${mvName}
- BUILD IMMEDIATE REFRESH AUTO ON MANUAL
- partition by (date_trunc(`k2`, 'month'))
- DISTRIBUTED BY RANDOM BUCKETS 2
- PROPERTIES (
- 'replication_num' = '1'
- )
- AS
- SELECT date_trunc(`k2`,'day') as month_alias FROM ${tableName};
- """
- Assert.fail();
- } catch (Exception e) {
- log.info(e.getMessage())
- assertTrue(e.getMessage().contains("partition column can not find from
sql select column"))
- }
-
- sql """drop materialized view if exists ${mvName};"""
- // list date year
- sql """
- CREATE MATERIALIZED VIEW ${mvName}
- BUILD DEFERRED REFRESH AUTO ON MANUAL
- partition by (date_trunc(`k2`,'year'))
- DISTRIBUTED BY RANDOM BUCKETS 2
- PROPERTIES (
- 'replication_num' = '1'
- )
- AS
- SELECT * FROM ${tableName};
- """
- showPartitionsResult = sql """show partitions from ${mvName}"""
- logger.info("showPartitionsResult: " + showPartitionsResult.toString())
- assertEquals(1, showPartitionsResult.size())
-
- sql """drop materialized view if exists ${mvName};"""
- // list date year
- sql """
- CREATE MATERIALIZED VIEW ${mvName}
- BUILD IMMEDIATE REFRESH AUTO ON MANUAL
- partition by (year_alias)
- DISTRIBUTED BY RANDOM BUCKETS 2
- PROPERTIES (
- 'replication_num' = '1'
- )
- AS
- SELECT date_trunc(`k2`,'year') as year_alias, * FROM ${tableName};
- """
- def date_list_year_partitions = sql """show partitions from ${mvName}"""
- assertEquals(1, date_list_year_partitions.size())
- order_qt_date_list_year_partition_by_column "SELECT * FROM ${mvName}"
-
- // list string month
- sql """drop table if exists `${tableName}`"""
- sql """drop materialized view if exists ${mvName};"""
- sql """
- CREATE TABLE `${tableName}` (
- `k1` LARGEINT NOT NULL COMMENT '\"用户id\"',
- `k2` varchar(200) NOT NULL COMMENT '\"数据灌入日期时间\"'
- ) ENGINE=OLAP
- DUPLICATE KEY(`k1`)
- COMMENT 'OLAP'
- PARTITION BY list(`k2`)
- (
- PARTITION p_20200101 VALUES IN ("2020==01==01"),
- PARTITION p_20200102 VALUES IN ("2020==01==02"),
- PARTITION p_20200201 VALUES IN ("2020==02==01")
- )
- DISTRIBUTED BY HASH(`k1`) BUCKETS 2
- PROPERTIES ('replication_num' = '1') ;
- """
- sql """
- insert into ${tableName}
values(1,"2020==01==01"),(2,"2020==01==02"),(3,"2020==02==01");
- """
-
- sql """
- CREATE MATERIALIZED VIEW ${mvName}
- BUILD DEFERRED REFRESH AUTO ON MANUAL
- partition by (date_trunc(`k2`,'month'))
- DISTRIBUTED BY RANDOM BUCKETS 2
- PROPERTIES (
- 'replication_num' = '1',
- 'partition_date_format'='%Y==%m==%d'
- )
- AS
- SELECT * FROM ${tableName};
- """
- showPartitionsResult = sql """show partitions from ${mvName}"""
- logger.info("showPartitionsResult: " + showPartitionsResult.toString())
- assertEquals(2, showPartitionsResult.size())
-
- sql """
- REFRESH MATERIALIZED VIEW ${mvName} AUTO
- """
- jobName = getJobName(dbName, mvName);
- log.info(jobName)
- waitingMTMVTaskFinished(jobName)
- order_qt_string_list_month "SELECT * FROM ${mvName} order by k1,k2"
-
-
- sql """drop materialized view if exists ${mvName};"""
- try {
- sql """
- CREATE MATERIALIZED VIEW ${mvName}
- BUILD IMMEDIATE REFRESH AUTO ON MANUAL
- partition by (date_trunc(month_alias, 'month'))
- DISTRIBUTED BY RANDOM BUCKETS 2
- PROPERTIES (
- 'replication_num' = '1'
- )
- AS
- SELECT date_trunc(`k2`,'day') as month_alias, * FROM ${tableName};
- """
- Assert.fail();
- } catch (Exception e) {
- log.info(e.getMessage())
- assertTrue(e.getMessage().contains("use invalid implicit expression"))
- }
-
- // mv partition level should be higher or equal then query, should fail
- sql """drop materialized view if exists ${mvName};"""
- try {
- sql """
- CREATE MATERIALIZED VIEW ${mvName}
- BUILD IMMEDIATE REFRESH AUTO ON MANUAL
- partition by (date_trunc(month_alias, 'day'))
- DISTRIBUTED BY RANDOM BUCKETS 2
- PROPERTIES (
- 'replication_num' = '1'
- )
- AS
- SELECT date_trunc(`k2`,'month') as month_alias, * FROM
${tableName};
- """
- Assert.fail();
- } catch (Exception e) {
- log.info(e.getMessage())
- assertTrue(e.getMessage().contains("use invalid implicit expression"))
- }
-
- // mv partition use a column not in mv sql select, should fail
- sql """drop materialized view if exists ${mvName};"""
- try {
- sql """
- CREATE MATERIALIZED VIEW ${mvName}
- BUILD IMMEDIATE REFRESH AUTO ON MANUAL
- partition by (date_trunc(`k2`, 'month'))
- DISTRIBUTED BY RANDOM BUCKETS 2
- PROPERTIES (
- 'replication_num' = '1'
- )
- AS
- SELECT date_trunc(`k2`,'day') as month_alias FROM ${tableName};
- """
- Assert.fail();
- } catch (Exception e) {
- log.info(e.getMessage())
- assertTrue(e.getMessage().contains("partition column can not find from
sql select column"))
- }
-
- // mv partition column type is date, base table is string, partition
mapping fail
- // support later
- sql """drop materialized view if exists ${mvName};"""
- try {
- sql """
- CREATE MATERIALIZED VIEW ${mvName}
- BUILD IMMEDIATE REFRESH AUTO ON MANUAL
- partition by (month_alias)
- DISTRIBUTED BY RANDOM BUCKETS 2
- PROPERTIES (
- 'replication_num' = '1',
- 'partition_date_format'='%Y==%m==%d'
- )
- AS
- SELECT date_trunc(`k2`,'month') as month_alias, * FROM
${tableName};
- """
- Assert.fail();
- } catch (Exception e) {
- log.info(e.getMessage())
- }
+ test {
+ sql """
+ CREATE MATERIALIZED VIEW ${mvName}
+ BUILD DEFERRED REFRESH AUTO ON MANUAL
+ partition by (date_trunc(`k2`,'month'))
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES (
+ 'replication_num' = '1'
+ )
+ AS
+ SELECT * FROM ${tableName};
+ """
+ exception "only support"
+ }
// range date month
sql """drop table if exists `${tableName}`"""
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
