This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new ebbfe1dfa27 branch-2.1: [fix](nereids) Fix query err when topn push
through union_all #54158 (#54258)
ebbfe1dfa27 is described below
commit ebbfe1dfa270580118a3a19b72180e01b8f622bd
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Wed Aug 6 11:47:37 2025 +0800
branch-2.1: [fix](nereids) Fix query err when topn push through union_all
#54158 (#54258)
Cherry-picked from #54158
Co-authored-by: seawinde <[email protected]>
---
.../rules/rewrite/PushDownTopNThroughUnion.java | 9 +-
.../push_down_top_n_through_union.out | Bin 4103 -> 10108 bytes
.../push_down_top_n_through_union.groovy | 238 +++++++++++++++++++++
3 files changed, 244 insertions(+), 3 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughUnion.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughUnion.java
index c13c1143efd..b02d7a1e45d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughUnion.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNThroughUnion.java
@@ -22,6 +22,7 @@ import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
import org.apache.doris.nereids.trees.plans.logical.LogicalTopN;
@@ -63,11 +64,13 @@ public class PushDownTopNThroughUnion implements
RewriteRuleFactory {
.then(topN -> {
LogicalUnion union = topN.child();
List<Plan> newChildren = new ArrayList<>();
- for (Plan child : union.children()) {
+ for (int j = 0; j < union.children().size(); j++) {
+ Plan child = union.child(j);
+ List<SlotReference> regularChildOutput =
union.getRegularChildOutput(j);
Map<Expression, Expression> replaceMap = new
HashMap<>();
- for (int i = 0; i < union.getOutputs().size();
++i) {
+ for (int i = 0; i < regularChildOutput.size();
++i) {
NamedExpression output =
union.getOutputs().get(i);
- replaceMap.put(output,
child.getOutput().get(i));
+ replaceMap.put(output,
regularChildOutput.get(i));
}
List<OrderKey> orderKeys =
topN.getOrderKeys().stream()
diff --git
a/regression-test/data/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.out
b/regression-test/data/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.out
index be8053d8349..6a32bbce857 100644
Binary files
a/regression-test/data/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.out
and
b/regression-test/data/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.out
differ
diff --git
a/regression-test/suites/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.groovy
b/regression-test/suites/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.groovy
index fa0eacd5b30..b8555432865 100644
---
a/regression-test/suites/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.groovy
+++
b/regression-test/suites/nereids_rules_p0/push_down_top_n/push_down_top_n_through_union.groovy
@@ -74,4 +74,242 @@ suite("push_down_top_n_through_union") {
qt_push_down_topn_union_complex_conditions """
explain shape plan select * from (select * from table1 t1 where
t1.score > 10 and t1.name = 'Test' union all select * from table1 t2 where
t2.id < 5 and t2.score < 20) sub order by id limit 10;
"""
+
+
+ sql "DROP TABLE IF EXISTS t1";
+
+ sql """
+ CREATE TABLE `t1` (
+
+ `logTimestamp` datetime NULL,
+
+ `args1` varchar(65533) NULL,
+
+ `args2` variant NULL,
+
+ `args3` int NULL,
+
+ `args4` text NULL,
+
+ `args5` varchar(65533) NULL,
+
+ `args6` varchar(65533) NULL,
+
+ `args7` varchar(200) NULL,
+
+ `args8` varchar(65533) NULL,
+
+ `args9` variant NULL,
+
+ `args10` variant NULL,
+
+ `args11` variant NULL,
+
+ `args12` variant NULL,
+
+ `args13` varchar(65533) NULL,
+
+ `args14` text NULL,
+
+ `args15` variant NULL,
+
+ `log` variant NULL
+ ) ENGINE=OLAP
+
+ DISTRIBUTED BY RANDOM BUCKETS 10
+
+ PROPERTIES (
+
+ "replication_num" = "1"
+
+ );"""
+
+ sql """
+ INSERT INTO t1 (
+ `logTimestamp`, `args1`, `args2`, `args3`, `args4`,
+ `args5`, `args6`, `args7`, `args8`, `args9`,
+ `args10`, `args11`, `args12`, `args13`, `args14`,
+ `args15`, `log`
+ ) VALUES
+
+ ('2025-07-31 09:15:22', 'username=admin&password=*****',
'{"action":"login","status":"success"}', 200, 'User authentication successful',
+ '192.168.1.100', '/api/v1/auth/login', '192.168.1.100',
'auth.example.com', '{"app":"authentication","env":"production"}',
+
'{"country":"CN","city":"Beijing","coordinates":{"lat":39.9042,"lon":116.4074}}',
'{"Content-Type":"application/json","Authorization":"Bearer
eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9"}',
'{"server":"auth-node-01","region":"east-1"}',
'https://portal.example.com/login', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)
AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
+ '{"source":"web","tags":["security","auth"]}',
'{"timestamp":"2025-07-31T09:15:22Z","level":"INFO","message":"Successful login
for user
admin","context":{"ip":"192.168.1.100","user_agent":"Chrome/125.0.0.0","session_id":"sess_xyz123"}}'),
+
+
+ ('2025-07-31 11:30:45', 'product_id=789&quantity=2&price=59.99',
'{"event":"purchase","payment_method":"credit_card"}', 201, 'Order processed
successfully',
+ '192.168.1.101', '/api/v1/orders', '192.168.1.101', 'shop.example.com',
'{"app":"ecommerce","env":"production","version":"3.2.1"}',
+ '{"country":"US","city":"New
York","coordinates":{"lat":40.7128,"lon":-74.0060}}',
'{"Content-Type":"application/json","X-CSRF-Token":"csrf_abc123","Accept":"application/json"}',
'{"server":"shop-node-03","region":"us-east-1"}',
'https://shop.example.com/product/789', 'Mozilla/5.0 (iPhone; CPU iPhone OS
16_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5
Mobile/15E148 Safari/604.1',
+ '{"source":"mobile","tags":["transaction","order"]}',
'{"timestamp":"2025-07-31T11:30:45Z","level":"INFO","message":"New order
created","context":{"order_id":"ord_789xyz","amount":119.98,"currency":"USD","customer_id":"cust_123abc"}}'),
+
+
+ ('2025-07-31 14:20:18', 'transaction_id=txn_987654&amount=199.99',
'{"type":"payment","gateway":"stripe","currency":"USD"}', 202, 'Payment
processing started',
+ '192.168.1.102', '/api/v1/payments', '192.168.1.102', 'pay.example.com',
'{"app":"payment","env":"production","features":["3d_secure","refunds"]}',
+
'{"country":"GB","city":"London","coordinates":{"lat":51.5074,"lon":-0.1278}}',
'{"Content-Type":"application/json","X-Request-ID":"req_987xyz","Idempotency-Key":"idemp_123"}',
'{"server":"payment-gw-02","region":"eu-west-2"}',
'https://checkout.example.com/payment', 'Mozilla/5.0 (Macintosh; Intel Mac OS X
10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
+ '{"source":"web","tags":["finance","payment"]}',
'{"timestamp":"2025-07-31T14:20:18Z","level":"INFO","message":"Payment
initiated","context":{"transaction_id":"txn_987654","amount":199.99,"currency":"USD","customer":"cust_456def"}}'),
+
+
+ ('2025-07-31 16:45:33',
'error_code=500&message=Database+connection+failed',
'{"severity":"critical","component":"database"}', 500, 'Database connection
error',
+ '192.168.1.103', '/api/v1/products', '192.168.1.103', 'api.example.com',
'{"app":"api-gateway","env":"production","dependencies":["mysql","redis"]}',
+
'{"country":"JP","city":"Tokyo","coordinates":{"lat":35.6762,"lon":139.6503}}',
'{"Content-Type":"application/json","X-Correlation-ID":"corr_456mno"}',
'{"server":"api-node-05","region":"ap-northeast-1"}',
'https://app.example.com/products', 'Mozilla/5.0 (Linux; Android 13; SM-S901B)
AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Mobile Safari/537.36',
+ '{"source":"mobile","tags":["error","database"]}',
'{"timestamp":"2025-07-31T16:45:33Z","level":"ERROR","message":"Failed to
connect to database","context":{"error":"Connection
timeout","attempts":3,"database":"products_db","duration_ms":5000}}'),
+
+
+ ('2025-07-31 19:05:07', 'query=smartphone&page=1&sort=price_asc',
'{"search_type":"product","filters":["in_stock","free_shipping"]}', 200,
'Search results returned',
+ '192.168.1.104', '/api/v1/search', '192.168.1.104', 'search.example.com',
'{"app":"search","env":"production","features":["autocomplete","faceted_search"]}',
+
'{"country":"DE","city":"Berlin","coordinates":{"lat":52.5200,"lon":13.4050}}',
'{"Content-Type":"application/json","X-Forwarded-For":"192.168.1.104,
203.0.113.5"}', '{"server":"search-node-01","region":"eu-central-1"}',
'https://www.example.com/search', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64;
rv:109.0) Gecko/20100101 Firefox/125.0',
+ '{"source":"web","tags":["search","query"]}',
'{"timestamp":"2025-07-31T19:05:07Z","level":"INFO","message":"Search query
processed","context":{"query":"smartphone","results_count":42,"response_time_ms":125,"filters_applied":2}}');
+ """
+
+ sql """
+ DROP TABLE IF EXISTS t2;
+ """
+
+ sql """
+ CREATE TABLE `t2` (
+
+ `logTimestamp` datetime NULL,
+
+ `args1` varchar(65533) NULL,
+
+ `args2` variant NULL,
+
+ `args3` int NULL,
+
+ `args4` text NULL,
+
+ `args5` varchar(65533) NULL,
+
+ `args6` varchar(65533) NULL,
+
+ `args7` varchar(200) NULL,
+
+ `args8` varchar(65533) NULL,
+
+ `args9` variant NULL,
+
+ `args10` variant NULL,
+
+ `args11` variant NULL,
+
+ `args12` variant NULL,
+
+ `args13` varchar(65533) NULL,
+
+ `args14` text NULL,
+
+ `args15` variant NULL,
+
+ `log` variant NULL
+
+ ) ENGINE=OLAP
+
+ DUPLICATE KEY(`logTimestamp`)
+
+ DISTRIBUTED BY RANDOM BUCKETS 10
+
+ PROPERTIES (
+
+ "replication_num" = "1"
+ );
+ """
+
+ sql """
+ INSERT INTO t2 (
+ `logTimestamp`, `args1`, `args2`, `args3`, `args4`,
+ `args5`, `args6`, `args7`, `args8`, `args9`,
+ `args10`, `args11`, `args12`, `args13`, `args14`,
+ `args15`, `log`
+ ) VALUES
+
+ ('2025-07-31 10:15:30', 'user_id=123&action=login',
'{"type":"authentication","status":"success"}', 200, 'User login successful',
+ '192.168.1.100', '/api/v1/login', '192.168.1.100', 'api.example.com',
'{"app":"web","env":"production"}',
+ '{"country":"China","city":"Beijing"}',
'{"Content-Type":"application/json","Authorization":"Bearer abc123"}',
'{"hostname":"web-server-01"}', 'https://referer.example.com', 'Mozilla/5.0
(Windows NT 10.0; Win64; x64)',
+ '{"input_type":"http","tags":["auth","prod"]}',
'{"timestamp":"2025-07-31T10:15:30Z","level":"info","message":"login
success"}'),
+
+
+ ('2025-07-31 11:30:45', 'product_id=456&category=electronics',
'{"type":"product_view","device":"mobile"}', 304, 'Product page viewed',
+ '192.168.1.101', '/products/456', '192.168.1.101', 'shop.example.com',
'{"app":"ecommerce","env":"staging"}',
+ '{"country":"USA","city":"New York"}',
'{"Content-Type":"text/html","Cache-Control":"max-age=3600"}',
'{"hostname":"shop-server-02"}', 'https://search.example.com', 'Mozilla/5.0
(iPhone; CPU iPhone OS 15_0 like Mac OS X)',
+ '{"input_type":"http","tags":["shop","mobile"]}',
'{"timestamp":"2025-07-31T11:30:45Z","level":"info","message":"product view"}'),
+
+
+ ('2025-07-31 14:45:20', 'order_id=789&amount=99.99',
'{"type":"payment","method":"credit_card"}', 201, 'Payment processed',
+ '192.168.1.102', '/api/v1/payments', '192.168.1.102', 'pay.example.com',
'{"app":"payment","env":"production"}',
+ '{"country":"UK","city":"London"}',
'{"Content-Type":"application/json","X-CSRF-Token":"xyz789"}',
'{"hostname":"payment-gateway-01"}', 'https://checkout.example.com',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)',
+ '{"input_type":"api","tags":["payment","prod"]}',
'{"timestamp":"2025-07-31T14:45:20Z","level":"info","message":"payment
success"}'),
+
+
+ ('2025-07-31 16:30:10', 'search_term=laptop&page=2',
'{"type":"search","results_count":25}', 200, 'Search results returned',
+ '192.168.1.103', '/search', '192.168.1.103', 'search.example.com',
'{"app":"search","env":"production"}',
+ '{"country":"Japan","city":"Tokyo"}',
'{"Content-Type":"application/json","Accept-Language":"en-US"}',
'{"hostname":"search-server-03"}', 'https://www.example.com', 'Mozilla/5.0
(Linux; Android 11; SM-G991B)',
+ '{"input_type":"search","tags":["search","prod"]}',
'{"timestamp":"2025-07-31T16:30:10Z","level":"info","message":"search query"}'),
+
+
+ ('2025-07-31 18:15:55', 'error_code=500&message=Internal+Server+Error',
'{"type":"error","severity":"high"}', 500, 'Internal server error occurred',
+ '192.168.1.104', '/api/v1/users', '192.168.1.104', 'api.example.com',
'{"app":"web","env":"production"}',
+ '{"country":"Germany","city":"Berlin"}',
'{"Content-Type":"application/json","X-Request-ID":"req123"}',
'{"hostname":"web-server-02"}', 'https://client.example.com', 'Mozilla/5.0
(Windows NT 10.0; Win64; x64)',
+ '{"input_type":"error","tags":["error","prod"]}',
'{"timestamp":"2025-07-31T18:15:55Z","level":"error","message":"server
error"}');
+ """
+
+
+ sql "DROP VIEW IF EXISTS view_test"
+
+
+ sql """
+ CREATE VIEW `view_test`AS
+ SELECT t1.`logTimestamp`,
+ t1.`args1`,
+ t1.`args2`,
+ t1.`args3`,
+ t1.`args4`,
+ t1.`args5`,
+ t1.`args6`,
+ t1.`args7`,
+ t1.`args8`,
+ t1.`args9`,
+ t1.`args10`,
+ t1.`args11`,
+ t1.`args12`,
+ t1.`args13`,
+ t1.`args14`,
+ t1.`args15`,
+ t1.`log`
+ FROM t1
+ UNION all
+ SELECT t2.`logTimestamp`,
+ t2.`args1`,
+ t2.`args2`,
+ t2.`args3`,
+ t2.`args4`,
+ t2.`args5`,
+ t2.`args6`,
+ t2.`args7`,
+ t2.`args8`,
+ t2.`args9`,
+ t2.`args10`,
+ t2.`args11`,
+ t2.`args12`,
+ t2.`args13`,
+ t2.`args14`,
+ t2.`args15`,
+ t2.`log`
+ FROM t2;
+ """
+
+
+ order_qt_union_all_push_down_top_n """
+ SELECT `args1`, `args2`, `args3`, `args4`, `args5`, `args6`, `args7`,
`args8`, `args9`, `args10`, `args11`, `args12`, `args13`, `args14`, `args15`,
`log`, logTimestamp
+ FROM view_test
+ ORDER BY logTimestamp desc
+ LIMIT 8;"""
+
+
+ order_qt_union_all_push_down_top_n_shape """
+ explain shape plan SELECT `args1`, `args2`, `args3`, `args4`, `args5`,
`args6`, `args7`, `args8`, `args9`, `args10`, `args11`, `args12`, `args13`,
`args14`, `args15`, `log`, logTimestamp
+ FROM view_test
+ ORDER BY logTimestamp desc
+ LIMIT 8;
+ """
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
