This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 067643df6b9 [fix](mtmv) Fix mv is deleted in nested mv causing query
err and fix some test (#45744)
067643df6b9 is described below
commit 067643df6b9ee30c509846687e200814b763dae9
Author: seawinde <w...@selectdb.com>
AuthorDate: Tue Dec 24 14:10:01 2024 +0800
[fix](mtmv) Fix mv is deleted in nested mv causing query err and fix some
test (#45744)
### What problem does this PR solve?
Related PR: #45045
Problem Summary:
if mv2 sql contains mv1 as following, if drop mv_level_1, query mv2
directly would cause err, this fix this
CREATE MATERIALIZED VIEW mv2
BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL
DISTRIBUTED BY RANDOM BUCKETS 2
PROPERTIES ('replication_num' = '1')
AS
SELECT * FROM mv_level_1;
---
.../nereids/rules/analysis/CollectRelation.java | 9 ++-
.../mv/nested/nested_mv_delete.out | 11 ++++
.../org/apache/doris/regression/suite/Suite.groovy | 39 ++++++++++-
.../suites/auth_p0/test_select_column_auth.groovy | 4 ++
.../suites/mv_p0/unique/unique_rewrite.groovy | 9 +++
.../range_date_part_up_rewrite.groovy | 14 ++--
.../mv/is_in_debug_mode/is_in_debug_mode.groovy | 20 ++++--
.../mv/nested/nested_mv_delete.groovy | 77 ++++++++++++++++++++++
8 files changed, 169 insertions(+), 14 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
index 9c6e3adbe74..01adc549e36 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CollectRelation.java
@@ -29,6 +29,7 @@ import org.apache.doris.nereids.StatementContext.TableFrom;
import org.apache.doris.nereids.analyzer.UnboundRelation;
import org.apache.doris.nereids.analyzer.UnboundResultSink;
import org.apache.doris.nereids.analyzer.UnboundTableSink;
+import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.parser.NereidsParser;
import org.apache.doris.nereids.pattern.MatchingContext;
import org.apache.doris.nereids.properties.PhysicalProperties;
@@ -197,7 +198,13 @@ public class CollectRelation implements
AnalysisRuleFactory {
try {
for (BaseTableInfo baseTableInfo :
mtmv.getRelation().getBaseTables()) {
LOG.info("mtmv {} related base table include {}", new
BaseTableInfo(mtmv), baseTableInfo);
-
cascadesContext.getStatementContext().getAndCacheTable(baseTableInfo.toList(),
TableFrom.MTMV);
+ try {
+
cascadesContext.getStatementContext().getAndCacheTable(baseTableInfo.toList(),
+ TableFrom.MTMV);
+ } catch (AnalysisException exception) {
+ LOG.warn("mtmv related base table get err, related
table is "
+ + baseTableInfo.toList(),
exception);
+ }
}
} finally {
mtmv.readMvUnlock();
diff --git
a/regression-test/data/nereids_rules_p0/mv/nested/nested_mv_delete.out
b/regression-test/data/nereids_rules_p0/mv/nested/nested_mv_delete.out
new file mode 100644
index 00000000000..65b48e2b8c1
--- /dev/null
+++ b/regression-test/data/nereids_rules_p0/mv/nested/nested_mv_delete.out
@@ -0,0 +1,11 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !query_after_delete --
+1 1 o 10.50 2023-12-08 a b 1 yy
\N
+1 1 o 9.50 2023-12-08 a b 1 yy
1
+2 1 o 11.50 2023-12-09 a b 1 yy
2
+3 1 o 12.50 2023-12-10 a b 1 yy
\N
+3 1 o 33.50 2023-12-10 a b 1 yy
3
+4 2 o 43.20 2023-12-11 c d 2 mm
\N
+5 2 o 1.20 2023-12-12 c d 2 mi
\N
+5 2 o 56.20 2023-12-12 c d 2 mi
4
+
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index 16cc6bbd21b..7ed3f1cb605 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -1974,6 +1974,9 @@ class Suite implements GroovyInterceptable {
for (String mv_name : mv_names) {
success = success && result.contains("(${mv_name})")
}
+ if (!success) {
+ logger.info("mv_rewrite_all_success fail =" + result)
+ }
Assert.assertEquals(true, success)
}
}
@@ -1984,7 +1987,11 @@ class Suite implements GroovyInterceptable {
check { result ->
boolean success = true;
for (String mv_name : mv_names) {
- Assert.assertEquals(true, result.contains("${mv_name}
chose"))
+ def contains = result.contains("${mv_name} chose")
+ if (!contains) {
+ logger.info("mv_rewrite_all_success fail =" + result)
+ }
+ Assert.assertEquals(true, contains)
}
}
}
@@ -2011,6 +2018,9 @@ class Suite implements GroovyInterceptable {
for (String mv_name : mv_names) {
success = success || result.contains("(${mv_name})")
}
+ if (!success) {
+ logger.info("mv_rewrite_any_success fail =" + result)
+ }
Assert.assertEquals(true, success)
}
}
@@ -2023,6 +2033,9 @@ class Suite implements GroovyInterceptable {
for (String mv_name : mv_names) {
success = success || result.contains("${mv_name} chose")
}
+ if (!success) {
+ logger.info("mv_rewrite_any_success fail =" + result)
+ }
Assert.assertEquals(true, success)
}
}
@@ -2043,6 +2056,9 @@ class Suite implements GroovyInterceptable {
def each_result = splitResult.length == 2 ?
splitResult[0].contains(mv_name) : false
success = success && (result.contains("(${mv_name})")
|| each_result)
}
+ if (!success) {
+
logger.info("mv_rewrite_all_success_without_check_chosen fail =" + result)
+ }
Assert.assertEquals(true, success)
}
}
@@ -2056,6 +2072,9 @@ class Suite implements GroovyInterceptable {
boolean stepSuccess = result.contains("${mv_name} chose")
|| result.contains("${mv_name} not chose")
success = success && stepSuccess
}
+ if (!success) {
+ logger.info("mv_rewrite_all_success_without_check_chosen
fail =" + result)
+ }
Assert.assertEquals(true, success)
}
}
@@ -2076,6 +2095,9 @@ class Suite implements GroovyInterceptable {
def each_result = splitResult.length == 2 ?
splitResult[0].contains(mv_name) : false
success = success || (result.contains("(${mv_name})")
|| each_result)
}
+ if (!success) {
+
logger.info("mv_rewrite_any_success_without_check_chosen fail =" + result)
+ }
Assert.assertEquals(true, success)
}
}
@@ -2088,6 +2110,9 @@ class Suite implements GroovyInterceptable {
for (String mv_name : mv_names) {
success = success || result.contains("${mv_name} chose")
|| result.contains("${mv_name} not chose")
}
+ if (!success) {
+ logger.info("mv_rewrite_any_success_without_check_chosen
fail =" + result)
+ }
Assert.assertEquals(true, success)
}
}
@@ -2146,6 +2171,9 @@ class Suite implements GroovyInterceptable {
boolean stepFail = !result.contains("(${mv_name})")
fail = fail && stepFail
}
+ if (!fail) {
+ logger.info("mv_rewrite_all_fail =" + result)
+ }
Assert.assertEquals(true, fail)
}
}
@@ -2159,6 +2187,9 @@ class Suite implements GroovyInterceptable {
boolean stepFail = result.contains("${mv_name} fail")
fail = fail && stepFail
}
+ if (!fail) {
+ logger.info("mv_rewrite_all_fail =" + result)
+ }
Assert.assertEquals(true, fail)
}
}
@@ -2176,6 +2207,9 @@ class Suite implements GroovyInterceptable {
for (String mv_name : mv_names) {
fail = fail || !result.contains("(${mv_name})")
}
+ if (!fail) {
+ logger.info("mv_rewrite_any_fail =" + result)
+ }
Assert.assertEquals(true, fail)
}
}
@@ -2188,6 +2222,9 @@ class Suite implements GroovyInterceptable {
for (String mv_name : mv_names) {
fail = fail || result.contains("${mv_name} fail")
}
+ if (!fail) {
+ logger.info("mv_rewrite_any_fail =" + result)
+ }
Assert.assertEquals(true, fail)
}
}
diff --git a/regression-test/suites/auth_p0/test_select_column_auth.groovy
b/regression-test/suites/auth_p0/test_select_column_auth.groovy
index 52f1dc02697..36cc2a0a09c 100644
--- a/regression-test/suites/auth_p0/test_select_column_auth.groovy
+++ b/regression-test/suites/auth_p0/test_select_column_auth.groovy
@@ -130,6 +130,10 @@ suite("test_select_column_auth","p0,auth") {
sql """grant select_priv(sum_id) on ${dbName}.${mtmv_name} to ${user}"""
sql """grant select_priv(id) on ${dbName}.${tableName} to ${user}"""
connect(user, "${pwd}", context.config.jdbcUrl) {
+ def show_grants = sql """show grants;"""
+ logger.info("show grants:" + show_grants.toString())
+ // If exec on fe follower, wait meta data is ready on follower
+ Thread.sleep(2000)
sql "SET enable_materialized_view_rewrite=true"
explain {
sql("""select username, sum(id) from ${dbName}.${tableName} group
by username""")
diff --git a/regression-test/suites/mv_p0/unique/unique_rewrite.groovy
b/regression-test/suites/mv_p0/unique/unique_rewrite.groovy
index e8c3dd05f80..1e8a37c7091 100644
--- a/regression-test/suites/mv_p0/unique/unique_rewrite.groovy
+++ b/regression-test/suites/mv_p0/unique/unique_rewrite.groovy
@@ -96,6 +96,10 @@ suite("mv_on_unique_table") {
AS
${mv1}
""")
+
+ def desc_all_mv1 = sql """desc lineitem_2_uniq all;"""
+ logger.info("desc mv1 is: " + desc_all_mv1.toString())
+
explain {
sql("""${query1}""")
check {result ->
@@ -124,6 +128,11 @@ suite("mv_on_unique_table") {
AS
${mv2}
""")
+
+ def desc_all_mv2 = sql """desc lineitem_2_uniq all;"""
+ logger.info("desc mv2 is" + desc_all_mv2)
+ // If exec on fe follower, wait meta data is ready on follower
+ Thread.sleep(2000)
explain {
sql("""${query2}""")
check {result ->
diff --git
a/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_part_up_rewrite.groovy
b/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_part_up_rewrite.groovy
index 88d8ad6ea2d..35d4a60e625 100644
---
a/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_part_up_rewrite.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/create_part_and_up/range_date_part_up_rewrite.groovy
@@ -169,7 +169,7 @@ suite("mtmv_range_date_part_up_rewrite") {
for (int i = 0; i < mv_name_list.size(); i++) {
def job_name = getJobName(db, mv_name_list[i])
waitingMTMVTaskFinished(job_name)
- mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+ mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
compare_res(query_stmt_list[i] + " order by 1,2,3")
}
@@ -178,38 +178,38 @@ suite("mtmv_range_date_part_up_rewrite") {
sql """insert into lineitem_range_date_union values
(1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18',
'2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-11-01')"""
for (int i = 0; i < mv_name_list.size(); i++) {
- mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+ mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
compare_res(query_stmt_list[i] + " order by 1,2,3")
}
for (int i = 0; i < mv_name_list.size(); i++) {
sql """refresh MATERIALIZED VIEW ${mv_name_list[i]} auto;"""
- mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+ mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
compare_res(query_stmt_list[i] + " order by 1,2,3")
}
sql """insert into lineitem_range_date_union values
(2, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18',
'2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-11-01');"""
for (int i = 0; i < mv_name_list.size(); i++) {
- mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+ mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
compare_res(query_stmt_list[i] + " order by 1,2,3")
}
for (int i = 0; i < mv_name_list.size(); i++) {
sql """refresh MATERIALIZED VIEW ${mv_name_list[i]} auto;"""
- mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+ mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
compare_res(query_stmt_list[i] + " order by 1,2,3")
}
sql """ALTER TABLE lineitem_range_date_union DROP PARTITION IF EXISTS p4
FORCE"""
for (int i = 0; i < mv_name_list.size(); i++) {
- mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+ mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
compare_res(query_stmt_list[i] + " order by 1,2,3")
}
for (int i = 0; i < mv_name_list.size(); i++) {
sql """refresh MATERIALIZED VIEW ${mv_name_list[i]} auto;"""
- mv_rewrite_success(query_stmt_list[i], mv_name_list[i])
+ mv_rewrite_any_success(query_stmt_list[i], mv_name_list)
compare_res(query_stmt_list[i] + " order by 1,2,3")
}
diff --git
a/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy
b/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy
index 15d93e32f65..f973d031ade 100644
---
a/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/is_in_debug_mode/is_in_debug_mode.groovy
@@ -83,7 +83,9 @@ suite("is_in_debug_mode") {
AS select * from orders where o_orderkey > 2;
"""
} catch (Exception e) {
- Assert.assertTrue(e.getMessage().contains("because is in debug mode"))
+ def message = e.getMessage()
+ logger.info("test_create_mv1" + message)
+ Assert.assertTrue(message.contains("because is in debug mode"))
}
sql """set skip_delete_sign = false;"""
@@ -99,7 +101,9 @@ suite("is_in_debug_mode") {
AS select * from orders where o_orderkey > 2;
"""
} catch (Exception e) {
- Assert.assertTrue(e.getMessage().contains("because is in debug mode"))
+ def message = e.getMessage()
+ logger.info("test_create_mv2" + message)
+ Assert.assertTrue(message.contains("because is in debug mode"))
}
sql """set skip_storage_engine_merge = false;"""
@@ -115,7 +119,9 @@ suite("is_in_debug_mode") {
AS select * from orders where o_orderkey > 2;
"""
} catch (Exception e) {
- Assert.assertTrue(e.getMessage().contains("because is in debug mode"))
+ def message = e.getMessage()
+ logger.info("test_create_mv3: " + message)
+ Assert.assertTrue(message.contains("because is in debug mode"))
}
sql """set skip_delete_bitmap = false;"""
@@ -131,7 +137,9 @@ suite("is_in_debug_mode") {
AS select * from orders where o_orderkey > 2;
"""
} catch (Exception e) {
- Assert.assertTrue(e.getMessage().contains("because is in debug mode"))
+ def message = e.getMessage()
+ logger.info("test_create_mv4" + message)
+ Assert.assertTrue(message.contains("because is in debug mode"))
}
sql """set skip_delete_predicate = false;"""
@@ -147,7 +155,9 @@ suite("is_in_debug_mode") {
AS select * from orders where o_orderkey > 2;
"""
} catch (Exception e) {
- Assert.assertTrue(e.getMessage().contains("because is in debug mode"))
+ def message = e.getMessage()
+ logger.info("test_create_mv5" + message)
+ Assert.assertTrue(message.contains("because is in debug mode"))
}
sql """set show_hidden_columns = false;"""
diff --git
a/regression-test/suites/nereids_rules_p0/mv/nested/nested_mv_delete.groovy
b/regression-test/suites/nereids_rules_p0/mv/nested/nested_mv_delete.groovy
new file mode 100644
index 00000000000..e7556094e71
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/mv/nested/nested_mv_delete.groovy
@@ -0,0 +1,77 @@
+package mv.nested
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("nested_mv_delete") {
+
+ String db = context.config.getDbNameByFile(context.file)
+ sql "use ${db}"
+ sql "set runtime_filter_mode=OFF";
+ sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'"
+
+ sql """
+ drop table if exists orders_1
+ """
+
+ sql """
+ CREATE TABLE IF NOT EXISTS orders_1 (
+ o_orderkey INTEGER NOT NULL,
+ o_custkey INTEGER NOT NULL,
+ o_orderstatus CHAR(1) NOT NULL,
+ o_totalprice DECIMALV3(15,2) NOT NULL,
+ o_orderdate DATE NOT NULL,
+ o_orderpriority CHAR(15) NOT NULL,
+ o_clerk CHAR(15) NOT NULL,
+ o_shippriority INTEGER NOT NULL,
+ o_comment VARCHAR(79) NOT NULL,
+ public_col INT NULL
+ )
+ DUPLICATE KEY(o_orderkey, o_custkey)
+ DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+
+ sql """
+ insert into orders_1 values
+ (1, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy', 1),
+ (1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy', null),
+ (2, 1, 'o', 11.5, '2023-12-09', 'a', 'b', 1, 'yy', 2),
+ (3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy', null),
+ (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy', 3),
+ (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm', null),
+ (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi', 4),
+ (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi', null);
+ """
+
+ sql """alter table orders_1 modify column o_comment set stats
('row_count'='8');"""
+
+
+ create_async_mv(db, "mv_level_1", """
+ select * from orders_1;
+ """)
+
+ create_async_mv(db, "mv_level_2", """
+ select * from mv_level_1;
+ """)
+
+ sql """drop materialized view mv_level_1;"""
+
+ order_qt_query_after_delete "select * from mv_level_2"
+ sql """ DROP MATERIALIZED VIEW IF EXISTS mv_level_2"""
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
