This is an automated email from the ASF dual-hosted git repository.
morrySnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 084144bad25 [fix](fd) Guard aggregate uniqueness with injectivity
proofs (#67878)
084144bad25 is described below
commit 084144bad25b8dd58a41c84d845020e63c47f419
Author: morrySnow <[email protected]>
AuthorDate: Wed Sep 23 11:45:27 2026 +0800
[fix](fd) Guard aggregate uniqueness with injectivity proofs (#67878)
## Problem
Nereids inferred that a grouped aggregate output was unique whenever its
input slot was unique. A non-injective argument expression or a lossy
result conversion can make different groups produce the same value, so
eliminating a later `GROUP BY` can return duplicate rows or incorrect
counts.
## Fix
Trace the aggregate argument back to a slot through injective casts
using the existing `getExpressionCoveredBySafetyCast` helper. For `MIN`
and `MAX`, this is sufficient for single-row groups. For `SUM` and
`AVG`, also require an injective conversion from the original slot type
to the aggregate result type, using `DataType.isInjectiveCastTo`
(updated on master by #68134).
This reuses the shared cast rules instead of maintaining an
aggregate-specific cast whitelist. Truncating `CHAR`/`VARCHAR` casts
remain non-injective because their parsed argument includes a
`substring` expression, which cannot be traced back through injective
casts. The same aggregate check is used by logical and physical traits.
---
.../apache/doris/nereids/util/ExpressionUtils.java | 22 ++++++-
.../doris/nereids/properties/UniqueTest.java | 32 +++++++++
.../eliminate_gby_key/eliminate_group_by.out | 12 ++++
.../eliminate_gby_key/eliminate_group_by.groovy | 76 +++++++++++++++++++++-
4 files changed, 139 insertions(+), 3 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
index 1b0403e9423..8aa62473e39 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
@@ -1207,9 +1207,27 @@ public class ExpressionUtils {
return expression instanceof Slot;
}
- // if the input is unique, the output of agg is unique, too
+ /**
+ * Whether this aggregate preserves the uniqueness of its argument for
single-row groups.
+ *
+ * <p>The argument must trace back to one slot through injective casts
only. MIN and MAX then
+ * return that argument value unchanged. SUM and AVG can additionally
coerce the argument to
+ * their result type, so that conversion must also be injective over the
original slot type.</p>
+ */
public static boolean isInjectiveAgg(Expression agg) {
- return agg instanceof Sum || agg instanceof Avg || agg instanceof Max
|| agg instanceof Min;
+ if (!(agg instanceof Sum || agg instanceof Avg || agg instanceof Max
|| agg instanceof Min)) {
+ return false;
+ }
+
+ Expression source = getExpressionCoveredBySafetyCast(agg.child(0));
+ if (!(source instanceof Slot)) {
+ return false;
+ }
+
+ if (agg instanceof Max || agg instanceof Min) {
+ return true;
+ }
+ return source.getDataType().isInjectiveCastTo(agg.getDataType());
}
/**
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/UniqueTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/UniqueTest.java
index d91a0ed9eb9..8fa6b3d9c09 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/UniqueTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/UniqueTest.java
@@ -45,6 +45,16 @@ class UniqueTest extends TestWithFeService {
+ "UNIQUE KEY(id)\n"
+ "distributed by hash(id) buckets 10\n"
+ "properties('replication_num' = '1');");
+ createTable("create table test.bigint_uni (\n"
+ + "id bigint not null)\n"
+ + "UNIQUE KEY(id)\n"
+ + "distributed by hash(id) buckets 10\n"
+ + "properties('replication_num' = '1');");
+ createTable("create table test.datetime_uni (\n"
+ + "id datetimev2(0) not null)\n"
+ + "UNIQUE KEY(id)\n"
+ + "distributed by hash(id) buckets 10\n"
+ + "properties('replication_num' = '1');");
connectContext.setDatabase("test");
connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION");
}
@@ -78,6 +88,28 @@ class UniqueTest extends TestWithFeService {
}
+ @Test
+ void testAggregateOutputInjectivity() {
+ assertAggregateOutputUnique("select sum(abs(id)) from agg group by
id", false);
+ assertAggregateOutputUnique("select avg(cast(id as bigint)) from agg
group by id", true);
+ assertAggregateOutputUnique("select avg(id) from bigint_uni group by
id", false);
+ assertAggregateOutputUnique("select sum(cast(id as bigint)) from agg
group by id", true);
+ assertAggregateOutputUnique("select sum(cast(id as tinyint)) from agg
group by id", false);
+ assertAggregateOutputUnique("select max(cast(id as char(1))) from agg
group by id", false);
+ assertAggregateOutputUnique("select max(cast(id as varchar(1))) from
agg group by id", false);
+ assertAggregateOutputUnique(
+ "select max(cast(id as datetimev2(6))) from datetime_uni group
by id", true);
+ }
+
+ private void assertAggregateOutputUnique(String sql, boolean expected) {
+ Plan plan = PlanChecker.from(connectContext)
+ .analyze(sql)
+ .getPlan();
+ Assertions.assertEquals(expected,
+
plan.getLogicalProperties().getTrait().isUnique(plan.getOutput().get(0)),
+ sql + "\n" + plan.treeString());
+ }
+
@Test
void testScan() throws Exception {
// test agg key
diff --git
a/regression-test/data/nereids_rules_p0/eliminate_gby_key/eliminate_group_by.out
b/regression-test/data/nereids_rules_p0/eliminate_gby_key/eliminate_group_by.out
index 037e9672190..af28c4d4e95 100644
---
a/regression-test/data/nereids_rules_p0/eliminate_gby_key/eliminate_group_by.out
+++
b/regression-test/data/nereids_rules_p0/eliminate_gby_key/eliminate_group_by.out
@@ -198,3 +198,15 @@ PhysicalResultSink
----filter((test_unique2.__DORIS_DELETE_SIGN__ = 0))
------PhysicalOlapScan[test_unique2]
+-- !non_injective_agg_argument --
+1 2
+
+-- !non_injective_agg_result_cast --
+9007199254740992 2
+
+-- !truncating_agg_argument_cast --
+1 2
+
+-- !ambiguous_complex_agg_argument_cast --
+["a", "b"] 2
+
diff --git
a/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_group_by.groovy
b/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_group_by.groovy
index 97a858a5c2e..68b6ba97539 100644
---
a/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_group_by.groovy
+++
b/regression-test/suites/nereids_rules_p0/eliminate_gby_key/eliminate_group_by.groovy
@@ -51,4 +51,78 @@ suite("eliminate_group_by") {
qt_variance_samp_shape "explain shape plan select
a,variance_samp(b),variance_samp(null) from test_unique2 group by a order by
1,2,3;"
qt_sum0_shape "explain shape plan select a,sum0(b),sum0(null) from
test_unique2 group by a order by 1,2,3;"
qt_median_shape "explain shape plan select
a,median(b),any_value(b),percentile(a,0.1),percentile(b,0.9),percentile(b,0.4)
from test_unique2 group by a order by 1,2,3,4,5,6;"
-}
\ No newline at end of file
+ sql "drop table if exists test_agg_output_injectivity;"
+ sql """
+ create table test_agg_output_injectivity(pk bigint not null)
+ unique key(pk)
+ distributed by hash(pk) buckets 1
+ properties("replication_num"="1");
+ """
+ sql """
+ insert into test_agg_output_injectivity
+ values (-1), (1), (10), (11), (9007199254740992), (9007199254740993);
+ """
+ order_qt_non_injective_agg_argument """
+ select s, count(*) as n
+ from (
+ select pk, sum(abs(pk)) as s
+ from test_agg_output_injectivity
+ where abs(pk) = 1
+ group by pk
+ ) q
+ group by s
+ order by s, n;
+ """
+ order_qt_non_injective_agg_result_cast """
+ select a, count(*) as n
+ from (
+ select pk, avg(pk) as a
+ from test_agg_output_injectivity
+ where pk > 100
+ group by pk
+ ) q
+ group by a
+ order by a, n;
+ """
+ order_qt_truncating_agg_argument_cast """
+ select m, count(*) as n
+ from (
+ select pk, max(cast(pk as char(1))) as m
+ from test_agg_output_injectivity
+ where pk between 10 and 11
+ group by pk
+ ) q
+ group by m
+ order by m, n;
+ """
+
+ sql "drop table if exists test_agg_array_injectivity;"
+ sql """
+ create table test_agg_array_injectivity(
+ id int not null,
+ arr array<string> not null
+ )
+ duplicate key(id)
+ distributed by hash(id) buckets 1
+ properties("replication_num"="1");
+ """
+ sql """
+ insert into test_agg_array_injectivity values
+ (1, array('a", "b')),
+ (2, array('a', 'b'));
+ """
+ order_qt_ambiguous_complex_agg_argument_cast """
+ select m, count(*) as n
+ from (
+ select arr, max(cast(arr as string)) as m
+ from (
+ select arr
+ from test_agg_array_injectivity
+ group by arr
+ ) d
+ group by arr
+ ) q
+ group by m
+ order by m, n;
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
