This is an automated email from the ASF dual-hosted git repository.
airborne pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new ea07a39284e [fix](inverted index) Fix errors caused by
enable_need_read_data_opt (#42064) (#42362)
ea07a39284e is described below
commit ea07a39284e50a9014b0bc2f44b6be35e1d0533a
Author: Sun Chenyang <[email protected]>
AuthorDate: Thu Oct 24 17:47:29 2024 +0800
[fix](inverted index) Fix errors caused by enable_need_read_data_opt
(#42064) (#42362)
## Proposed changes
pick from master #42064
---
.../rules/implementation/AggregateStrategies.java | 22 +++-
.../data/inverted_index_p0/test_index_rqg_bug8.out | 4 +
.../inverted_index_p0/test_index_rqg_bug8.groovy | 136 +++++++++++++++++++++
3 files changed, 160 insertions(+), 2 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java
index 1482ba4d013..f2b332570b9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java
@@ -130,7 +130,8 @@ public class AggregateStrategies implements
ImplementationRuleFactory {
Set<Slot> aggSlots = funcs.stream()
.flatMap(f -> f.getInputSlots().stream())
.collect(Collectors.toSet());
- return conjuncts.stream().allMatch(expr ->
checkSlotInOrExpression(expr, aggSlots));
+ return conjuncts.stream().allMatch(expr ->
checkSlotInOrExpression(expr, aggSlots)
+ &&
checkIsNullExpr(expr, aggSlots));
})
.thenApply(ctx -> {
LogicalAggregate<LogicalFilter<LogicalOlapScan>> agg =
ctx.root;
@@ -166,7 +167,8 @@ public class AggregateStrategies implements
ImplementationRuleFactory {
Set<Slot> aggSlots = funcs.stream()
.flatMap(f -> f.getInputSlots().stream())
.collect(Collectors.toSet());
- return conjuncts.stream().allMatch(expr ->
checkSlotInOrExpression(expr, aggSlots));
+ return conjuncts.stream().allMatch(expr ->
checkSlotInOrExpression(expr, aggSlots)
+ &&
checkIsNullExpr(expr, aggSlots));
})
.thenApply(ctx -> {
LogicalAggregate<LogicalProject<LogicalFilter<LogicalOlapScan>>> agg = ctx.root;
@@ -373,6 +375,22 @@ public class AggregateStrategies implements
ImplementationRuleFactory {
return true;
}
+ private boolean checkIsNullExpr(Expression expr, Set<Slot> aggSlots) {
+ if (expr instanceof IsNull) {
+ Set<Slot> slots = expr.getInputSlots();
+ if (slots.stream().anyMatch(aggSlots::contains)) {
+ return false;
+ }
+ } else {
+ for (Expression child : expr.children()) {
+ if (!checkIsNullExpr(child, aggSlots)) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
private boolean isDupOrMowKeyTable(LogicalOlapScan logicalScan) {
if (logicalScan != null) {
KeysType keysType = logicalScan.getTable().getKeysType();
diff --git a/regression-test/data/inverted_index_p0/test_index_rqg_bug8.out
b/regression-test/data/inverted_index_p0/test_index_rqg_bug8.out
new file mode 100644
index 00000000000..a21f3b8748e
--- /dev/null
+++ b/regression-test/data/inverted_index_p0/test_index_rqg_bug8.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+0
+
diff --git
a/regression-test/suites/inverted_index_p0/test_index_rqg_bug8.groovy
b/regression-test/suites/inverted_index_p0/test_index_rqg_bug8.groovy
new file mode 100644
index 00000000000..9e0051d0a16
--- /dev/null
+++ b/regression-test/suites/inverted_index_p0/test_index_rqg_bug8.groovy
@@ -0,0 +1,136 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+suite("test_index_rqg_bug8", "test_index_rqg_bug8"){
+ def table1 = "test_index_rqg_bug8"
+
+ sql "drop table if exists ${table1}"
+
+ sql """
+ CREATE TABLE ${table1} (
+ `pk` int NULL,
+ `col_int_undef_signed_index_inverted` int NULL,
+ `col_boolean_undef_signed` boolean NULL,
+ `col_boolean_undef_signed_not_null` boolean NOT NULL,
+ `col_tinyint_undef_signed` tinyint NULL,
+ `col_tinyint_undef_signed_index_inverted` tinyint NULL,
+ `col_tinyint_undef_signed_not_null` tinyint NOT NULL,
+ `col_tinyint_undef_signed_not_null_index_inverted` tinyint NOT NULL,
+ `col_smallint_undef_signed` smallint NULL,
+ `col_smallint_undef_signed_index_inverted` smallint NULL,
+ `col_smallint_undef_signed_not_null` smallint NOT NULL,
+ `col_smallint_undef_signed_not_null_index_inverted` smallint NOT NULL,
+ `col_int_undef_signed` int NULL,
+ `col_int_undef_signed_not_null` int NOT NULL,
+ `col_int_undef_signed_not_null_index_inverted` int NOT NULL,
+ `col_bigint_undef_signed` bigint NULL,
+ `col_bigint_undef_signed_index_inverted` bigint NULL,
+ `col_bigint_undef_signed_not_null` bigint NOT NULL,
+ `col_bigint_undef_signed_not_null_index_inverted` bigint NOT NULL,
+ `col_decimal_16__8__undef_signed` decimal(16,8) NULL,
+ `col_decimal_16__8__undef_signed_index_inverted` decimal(16,8) NULL,
+ `col_decimal_16__8__undef_signed_not_null` decimal(16,8) NOT NULL,
+ `col_decimal_16__8__undef_signed_not_null_index_inverted`
decimal(16,8) NOT NULL,
+ `col_decimal_38__9__undef_signed` decimal(38,9) NULL,
+ `col_decimal_38__9__undef_signed_index_inverted` decimal(38,9) NULL,
+ `col_decimal_38__9__undef_signed_not_null` decimal(38,9) NOT NULL,
+ `col_decimal_38__9__undef_signed_not_null_index_inverted`
decimal(38,9) NOT NULL,
+ `col_decimal_38__30__undef_signed` decimal(38,30) NULL,
+ `col_decimal_38__30__undef_signed_index_inverted` decimal(38,30) NULL,
+ `col_decimal_38__30__undef_signed_not_null` decimal(38,30) NOT NULL,
+ `col_decimal_38__30__undef_signed_not_null_index_inverted`
decimal(38,30) NOT NULL,
+ `col_date_undef_signed` date NULL,
+ `col_date_undef_signed_index_inverted` date NULL,
+ `col_date_undef_signed_not_null` date NOT NULL,
+ `col_date_undef_signed_not_null_index_inverted` date NOT NULL,
+ `col_datetime_undef_signed` datetime NULL,
+ `col_datetime_undef_signed_index_inverted` datetime NULL,
+ `col_datetime_undef_signed_not_null` datetime NOT NULL,
+ `col_datetime_undef_signed_not_null_index_inverted` datetime NOT NULL,
+ `col_datetime_3__undef_signed` datetime(3) NULL,
+ `col_datetime_3__undef_signed_index_inverted` datetime(3) NULL,
+ `col_datetime_3__undef_signed_not_null` datetime(3) NOT NULL,
+ `col_datetime_3__undef_signed_not_null_index_inverted` datetime(3) NOT
NULL,
+ `col_datetime_6__undef_signed` datetime(6) NULL,
+ `col_datetime_6__undef_signed_index_inverted` datetime(6) NULL,
+ `col_datetime_6__undef_signed_not_null` datetime(6) NOT NULL,
+ `col_datetime_6__undef_signed_not_null_index_inverted` datetime(6) NOT
NULL,
+ `col_char_255__undef_signed` character(255) NULL,
+ `col_char_255__undef_signed_index_inverted` character(255) NULL,
+ `col_char_255__undef_signed_index_inverted_p_e` character(255) NULL,
+ `col_char_255__undef_signed_index_inverted_p_u` character(255) NULL,
+ `col_char_255__undef_signed_not_null` character(255) NOT NULL,
+ `col_char_255__undef_signed_not_null_index_inverted` character(255)
NOT NULL,
+ `col_char_255__undef_signed_not_null_index_inverted_p_e`
character(255) NOT NULL,
+ `col_char_255__undef_signed_not_null_index_inverted_p_u`
character(255) NOT NULL,
+ `col_varchar_1024__undef_signed` varchar(1024) NULL,
+ `col_varchar_1024__undef_signed_index_inverted` varchar(1024) NULL,
+ `col_varchar_1024__undef_signed_index_inverted_p_e` varchar(1024) NULL,
+ `col_varchar_1024__undef_signed_index_inverted_p_u` varchar(1024) NULL,
+ `col_varchar_1024__undef_signed_not_null` varchar(1024) NOT NULL,
+ `col_varchar_1024__undef_signed_not_null_index_inverted` varchar(1024)
NOT NULL,
+ `col_varchar_1024__undef_signed_not_null_index_inverted_p_e`
varchar(1024) NOT NULL,
+ `col_varchar_1024__undef_signed_not_null_index_inverted_p_u`
varchar(1024) NOT NULL,
+ INDEX col_tinyint_undef_signed_index_inverted_idx
(`col_tinyint_undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_tinyint_undef_signed_not_null_index_inverted_idx
(`col_tinyint_undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_smallint_undef_signed_index_inverted_idx
(`col_smallint_undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_smallint_undef_signed_not_null_index_inverted_idx
(`col_smallint_undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_int_undef_signed_index_inverted_idx
(`col_int_undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_int_undef_signed_not_null_index_inverted_idx
(`col_int_undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_bigint_undef_signed_index_inverted_idx
(`col_bigint_undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_bigint_undef_signed_not_null_index_inverted_idx
(`col_bigint_undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_decimal_16__8__undef_signed_index_inverted_idx
(`col_decimal_16__8__undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_decimal_16__8__undef_signed_not_null_index_inverted_idx
(`col_decimal_16__8__undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_decimal_38__9__undef_signed_index_inverted_idx
(`col_decimal_38__9__undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_decimal_38__9__undef_signed_not_null_index_inverted_idx
(`col_decimal_38__9__undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_decimal_38__30__undef_signed_index_inverted_idx
(`col_decimal_38__30__undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_decimal_38__30__undef_signed_not_null_index_inverted_idx
(`col_decimal_38__30__undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_date_undef_signed_index_inverted_idx
(`col_date_undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_date_undef_signed_not_null_index_inverted_idx
(`col_date_undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_datetime_undef_signed_index_inverted_idx
(`col_datetime_undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_datetime_undef_signed_not_null_index_inverted_idx
(`col_datetime_undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_datetime_3__undef_signed_index_inverted_idx
(`col_datetime_3__undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_datetime_3__undef_signed_not_null_index_inverted_idx
(`col_datetime_3__undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_datetime_6__undef_signed_index_inverted_idx
(`col_datetime_6__undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_datetime_6__undef_signed_not_null_index_inverted_idx
(`col_datetime_6__undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_char_255__undef_signed_index_inverted_idx
(`col_char_255__undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_char_255__undef_signed_index_inverted_p_e_idx
(`col_char_255__undef_signed_index_inverted_p_e`) USING INVERTED
PROPERTIES("parser" = "english", "lower_case" = "true", "support_phrase" =
"true"),
+ INDEX col_char_255__undef_signed_index_inverted_p_u_idx
(`col_char_255__undef_signed_index_inverted_p_u`) USING INVERTED
PROPERTIES("parser" = "unicode", "lower_case" = "true", "support_phrase" =
"true"),
+ INDEX col_char_255__undef_signed_not_null_index_inverted_idx
(`col_char_255__undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_char_255__undef_signed_not_null_index_inverted_p_e_idx
(`col_char_255__undef_signed_not_null_index_inverted_p_e`) USING INVERTED
PROPERTIES("parser" = "english", "lower_case" = "true", "support_phrase" =
"true"),
+ INDEX col_char_255__undef_signed_not_null_index_inverted_p_u_idx
(`col_char_255__undef_signed_not_null_index_inverted_p_u`) USING INVERTED
PROPERTIES("parser" = "unicode", "lower_case" = "true", "support_phrase" =
"true"),
+ INDEX col_varchar_1024__undef_signed_index_inverted_idx
(`col_varchar_1024__undef_signed_index_inverted`) USING INVERTED,
+ INDEX col_varchar_1024__undef_signed_index_inverted_p_e_idx
(`col_varchar_1024__undef_signed_index_inverted_p_e`) USING INVERTED
PROPERTIES("parser" = "english", "lower_case" = "true", "support_phrase" =
"true"),
+ INDEX col_varchar_1024__undef_signed_index_inverted_p_u_idx
(`col_varchar_1024__undef_signed_index_inverted_p_u`) USING INVERTED
PROPERTIES("parser" = "unicode", "lower_case" = "true", "support_phrase" =
"true"),
+ INDEX col_varchar_1024__undef_signed_not_null_index_inverted_idx
(`col_varchar_1024__undef_signed_not_null_index_inverted`) USING INVERTED,
+ INDEX col_varchar_1024__undef_signed_not_null_index_inverted_p_e_idx
(`col_varchar_1024__undef_signed_not_null_index_inverted_p_e`) USING INVERTED
PROPERTIES("parser" = "english", "lower_case" = "true", "support_phrase" =
"true"),
+ INDEX col_varchar_1024__undef_signed_not_null_index_inverted_p_u_idx
(`col_varchar_1024__undef_signed_not_null_index_inverted_p_u`) USING INVERTED
PROPERTIES("parser" = "unicode", "lower_case" = "true", "support_phrase" =
"true")
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`pk`, `col_int_undef_signed_index_inverted`)
+ DISTRIBUTED BY HASH(`pk`) BUCKETS 10
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ sql """ insert into ${table1}
(pk,col_boolean_undef_signed,col_boolean_undef_signed_not_null,col_tinyint_undef_signed,col_tinyint_undef_signed_index_inverted,col_tinyint_undef_signed_not_null,col_tinyint_undef_signed_not_null_index_inverted,col_smallint_undef_signed,col_smallint_undef_signed_index_inverted,col_smallint_undef_signed_not_null,col_smallint_undef_signed_not_null_index_inverted,col_int_undef_signed,col_int_undef_signed_index_inverted,col_int_undef_signed_not_null,col_int_
[...]
+
+ qt_sql """
+ SELECT count(col_decimal_16__8__undef_signed_index_inverted) FROM
${table1} where col_decimal_16__8__undef_signed_index_inverted is null;
+ """
+
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
