This is an automated email from the ASF dual-hosted git repository.
morrySnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new cba3cac5a2f [Fix](Query Stats) Add QueryStatsRecorder for column-level
query and filter - Part1 (#63067)
cba3cac5a2f is described below
commit cba3cac5a2fbeb7d23346285db1b0a7a47d5e969
Author: nsivarajan <[email protected]>
AuthorDate: Wed May 27 10:59:36 2026 +0530
[Fix](Query Stats) Add QueryStatsRecorder for column-level query and filter
- Part1 (#63067)
### What problem does this PR solve?
Problem Summary:
`Show Query Stats` returns only 0, as it is broken . This PR is intended
to make Query and Filter Stats to available when
below config is enabled.
```sql
admin set frontend config ("enable_query_hit_stats" = "true");
set enable_nereids_planner = true;
```
This is planned as Sequence of PR with incremental capture of more cases
later (like Group by, order by, Window and more).
---------
Co-authored-by: Sivarajan Narayanan <[email protected]>
---
.../org/apache/doris/nereids/NereidsPlanner.java | 4 +
.../org/apache/doris/nereids/StatementContext.java | 9 +
.../doris/statistics/query/QueryStatsRecorder.java | 212 ++++++++++
.../doris/statistics/query/QueryStatsUtil.java | 6 +-
.../statistics/query/QueryStatsRecorderTest.java | 435 +++++++++++++++++++++
.../data/query_p0/stats/query_stats_test.out | 52 ---
.../suites/query_p0/stats/query_stats_test.groovy | 120 +++++-
7 files changed, 776 insertions(+), 62 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index 7952d009e51..5ccca2a185f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -85,6 +85,7 @@ import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ResultSet;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.qe.VariableMgr;
+import org.apache.doris.statistics.query.QueryStatsRecorder;
import org.apache.doris.statistics.util.StatisticsUtil;
import org.apache.doris.thrift.TQueryCacheParam;
@@ -179,6 +180,9 @@ public class NereidsPlanner extends Planner {
LOG.info(getExplainString(new
ExplainOptions(ExplainLevel.SHAPE_PLAN, false)));
LOG.info(getExplainString(new
ExplainOptions(ExplainLevel.DISTRIBUTED_PLAN, false)));
}
+ if (physicalPlan != null) {
+ QueryStatsRecorder.record(physicalPlan, statementContext);
+ }
}
@VisibleForTesting
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
index 2af64752cc4..3659b1f79f5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
@@ -319,6 +319,7 @@ public class StatementContext implements Closeable {
// When true, data will be collected to a single node to avoid generating
too many small files
private boolean useGatherForIcebergRewrite = false;
private boolean hasNestedColumns;
+ private boolean queryStatsRecorded = false;
private final Set<CTEId> mustInlineCTE = new HashSet<>();
private final Set<String> usedAIResourceNames = new LinkedHashSet<>();
@@ -1174,6 +1175,14 @@ public class StatementContext implements Closeable {
return isInsert;
}
+ public boolean isQueryStatsRecorded() {
+ return queryStatsRecorded;
+ }
+
+ public void markQueryStatsRecorded() {
+ queryStatsRecorded = true;
+ }
+
public Optional<Map<TableIf, Set<Expression>>> getMvRefreshPredicates() {
return mvRefreshPredicates;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/query/QueryStatsRecorder.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/QueryStatsRecorder.java
new file mode 100644
index 00000000000..1b5b76a1bc5
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/QueryStatsRecorder.java
@@ -0,0 +1,212 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.statistics.query;
+
+import org.apache.doris.analysis.StatementBase;
+import org.apache.doris.catalog.DatabaseIf;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.common.Config;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.glue.LogicalPlanAdapter;
+import org.apache.doris.nereids.trees.expressions.ExprId;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.commands.Command;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter;
+import
org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterializeOlapScan;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
+import org.apache.doris.qe.ConnectContext;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Records column-level query-hit and filter-hit statistics from the Nereids
physical plan.
+ * Called once per query in NereidsPlanner after plan translation.
+ *
+ * <p>Scope (Part 1):
+ * <ul>
+ * <li>queryHit: base SELECT columns whose ExprId flows straight through to
the root
+ * plan's output without rewriting. Columns hidden by an alias, an
expression,
+ * or an aggregate function are NOT recorded yet (Part 2).</li>
+ * <li>filterHit: columns referenced in WHERE predicate conjuncts.</li>
+ * <li>Only OlapTable scans are recorded; external tables (Hive, Iceberg,
JDBC, …) are not.</li>
+ * <li>DML, EXPLAIN, and internal queries (e.g. auto-analyze) are
skipped.</li>
+ * <li>Per query, each table's count is incremented at most once regardless
of scan count.</li>
+ * </ul>
+ * GROUP BY, ORDER BY, window, JOIN, and aliased/projected columns are
deferred to Part 2.
+ */
+public class QueryStatsRecorder {
+ private static final Logger LOG =
LogManager.getLogger(QueryStatsRecorder.class);
+
+ private QueryStatsRecorder() {}
+
+ public static void record(PhysicalPlan plan, StatementContext stmtContext)
{
+ if (!shouldRecord(stmtContext)) {
+ return;
+ }
+ if (stmtContext.isQueryStatsRecorded()) {
+ return;
+ }
+ // Set the latch before the work so a partial-failure retry does not
double-count.
+ stmtContext.markQueryStatsRecorded();
+ try {
+ Map<String, StatsDelta> deltas = collectDeltas(plan);
+ for (StatsDelta delta : deltas.values()) {
+ if (!delta.empty()) {
+ try {
+ Env.getCurrentEnv().getQueryStats().addStats(delta);
+ } catch (Exception e) {
+ ConnectContext cc = stmtContext.getConnectContext();
+ String queryId = (cc != null && cc.queryId() != null)
+ ? cc.queryId().toString() : "unknown";
+ LOG.warn("Failed to record query stats for query={}",
queryId, e);
+ }
+ }
+ }
+ } catch (Exception e) {
+ ConnectContext cc = stmtContext.getConnectContext();
+ String queryId = (cc != null && cc.queryId() != null)
+ ? cc.queryId().toString() : "unknown";
+ LOG.warn("Failed to build query stats deltas for query={}",
queryId, e);
+ }
+ }
+
+ /**
+ * Builds the per-table StatsDelta map from the physical plan.
+ * Package-private so unit tests can verify recording logic without
touching Env.
+ */
+ static Map<String, StatsDelta> collectDeltas(PhysicalPlan plan) {
+ Map<ExprId, PhysicalOlapScan> exprIdToScan = new HashMap<>();
+ Map<String, StatsDelta> deltas = new HashMap<>();
+ walkPlan(plan, exprIdToScan, deltas);
+ if (exprIdToScan.isEmpty()) {
+ return deltas;
+ }
+ for (Slot slot : plan.getOutput()) {
+ if (!(slot instanceof SlotReference)) {
+ continue;
+ }
+ SlotReference sr = (SlotReference) slot;
+ PhysicalOlapScan sourceScan = exprIdToScan.get(sr.getExprId());
+ if (sourceScan == null) {
+ continue;
+ }
+ StatsDelta delta = getOrCreateDelta(deltas, sourceScan);
+ if (delta != null) {
+ sr.getOriginalColumn().ifPresent(col ->
delta.addQueryStats(col.getName()));
+ }
+ }
+ return deltas;
+ }
+
+ // Package-private for testing.
+ static boolean shouldRecord(StatementContext ctx) {
+ if (!Config.enable_query_hit_stats) {
+ return false;
+ }
+ ConnectContext connectContext = ctx.getConnectContext();
+ if (connectContext != null && connectContext.getState().isInternal()) {
+ return false;
+ }
+ StatementBase stmt = ctx.getParsedStatement();
+ if (stmt == null || stmt.isExplain()) {
+ return false;
+ }
+ // isInsert guards INSERT INTO … SELECT: parsedStmt may be the SELECT
sub-plan,
+ // not the INSERT Command, when NereidsPlanner re-enters for execution
planning.
+ if (ctx.isInsert()) {
+ return false;
+ }
+ if (stmt instanceof LogicalPlanAdapter
+ && ((LogicalPlanAdapter) stmt).getLogicalPlan() instanceof
Command) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Single-pass tree walk: registers scan output slots into exprIdToScan,
+ * and records filterHit for PhysicalFilter conjuncts.
+ * Children are visited before the current node so scans are registered
+ * before parent filters look them up.
+ * PhysicalLazyMaterializeOlapScan is checked before PhysicalOlapScan
+ * because it is a subclass; the inner scan's metadata must be used.
+ */
+ private static void walkPlan(Plan plan,
+ Map<ExprId, PhysicalOlapScan> exprIdToScan,
+ Map<String, StatsDelta> deltas) {
+ if (plan instanceof PhysicalLazyMaterializeOlapScan) {
+ PhysicalOlapScan inner =
+ ((PhysicalLazyMaterializeOlapScan) plan).getScan();
+ for (Slot slot : plan.getOutput()) {
+ exprIdToScan.put(slot.getExprId(), inner);
+ }
+ return;
+ }
+ if (plan instanceof PhysicalOlapScan) {
+ PhysicalOlapScan scan = (PhysicalOlapScan) plan;
+ for (Slot slot : scan.getOutput()) {
+ exprIdToScan.put(slot.getExprId(), scan);
+ }
+ return;
+ }
+ for (Plan child : plan.children()) {
+ walkPlan(child, exprIdToScan, deltas);
+ }
+ if (plan instanceof PhysicalFilter) {
+ PhysicalFilter<?> filter = (PhysicalFilter<?>) plan;
+ for (Expression conjunct : filter.getConjuncts()) {
+ conjunct.getInputSlots().forEach(slot -> {
+ if (!(slot instanceof SlotReference)) {
+ return;
+ }
+ SlotReference sr = (SlotReference) slot;
+ PhysicalOlapScan sourceScan =
exprIdToScan.get(sr.getExprId());
+ if (sourceScan == null) {
+ return;
+ }
+ StatsDelta delta = getOrCreateDelta(deltas, sourceScan);
+ if (delta != null) {
+ sr.getOriginalColumn().ifPresent(col ->
delta.addFilterStats(col.getName()));
+ }
+ });
+ }
+ }
+ }
+
+ private static StatsDelta getOrCreateDelta(Map<String, StatsDelta> deltas,
+ PhysicalOlapScan scan) {
+ OlapTable t = scan.getTable();
+ DatabaseIf<?> db = scan.getDatabase();
+ if (t == null || db == null) {
+ return null;
+ }
+ String key = t.getCatalogId() + "_" + db.getId() + "_" + t.getId()
+ + "_" + scan.getSelectedIndexId();
+ return deltas.computeIfAbsent(key, k ->
+ new StatsDelta(t.getCatalogId(), db.getId(), t.getId(),
scan.getSelectedIndexId()));
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/query/QueryStatsUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/QueryStatsUtil.java
index 3359fb54767..a8cf306f271 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/query/QueryStatsUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/query/QueryStatsUtil.java
@@ -24,6 +24,7 @@ import org.apache.doris.common.Pair;
import org.apache.doris.common.UserException;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.system.Frontend;
+import org.apache.doris.system.SystemInfoService.HostInfo;
import org.apache.doris.thrift.FrontendService;
import org.apache.doris.thrift.TGetQueryStatsRequest;
import org.apache.doris.thrift.TNetworkAddress;
@@ -171,8 +172,11 @@ public class QueryStatsUtil {
private static List<TQueryStatsResult> getStats(TGetQueryStatsRequest
request) {
List<TQueryStatsResult> results = new ArrayList<>();
+ HostInfo selfHostInfo = Env.getCurrentEnv().getSelfNode();
for (Frontend fe : Env.getCurrentEnv().getFrontends(null /* all */)) {
- if (!fe.isAlive() ||
fe.getHost().equals(Env.getCurrentEnv().getSelfNode().getHost())) {
+ if (!fe.isAlive()
+ || (fe.getHost().equals(selfHostInfo.getHost())
+ && fe.getEditLogPort() == selfHostInfo.getPort())) {
continue;
}
FrontendService.Client client = null;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/query/QueryStatsRecorderTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/query/QueryStatsRecorderTest.java
new file mode 100644
index 00000000000..a06a6eba4ad
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/query/QueryStatsRecorderTest.java
@@ -0,0 +1,435 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.statistics.query;
+
+import org.apache.doris.analysis.ExplainOptions;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.DatabaseIf;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.common.Config;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.glue.LogicalPlanAdapter;
+import org.apache.doris.nereids.trees.expressions.ExprId;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.plans.commands.Command;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter;
+import
org.apache.doris.nereids.trees.plans.physical.PhysicalLazyMaterializeOlapScan;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.QueryState;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+public class QueryStatsRecorderTest {
+
+ private boolean originalConfigValue;
+
+ @BeforeEach
+ public void setUp() {
+ originalConfigValue = Config.enable_query_hit_stats;
+ }
+
+ @AfterEach
+ public void tearDown() {
+ // Restore config to avoid affecting other tests.
+ Config.enable_query_hit_stats = originalConfigValue;
+ }
+
+ // ── shouldRecord guard tests
─────────────────────────────────────────────
+
+ @Test
+ public void testShouldNotRecordWhenConfigOff() {
+ Config.enable_query_hit_stats = false;
+ StatementContext ctx = new StatementContext();
+ LogicalPlanAdapter stmt = new LogicalPlanAdapter(
+
Mockito.mock(org.apache.doris.nereids.trees.plans.logical.LogicalPlan.class),
ctx);
+ ctx.setParsedStatement(stmt);
+ Assertions.assertFalse(QueryStatsRecorder.shouldRecord(ctx));
+ }
+
+ @Test
+ public void testShouldNotRecordWhenStatementIsNull() {
+ Config.enable_query_hit_stats = true;
+ StatementContext ctx = new StatementContext();
+ // parsedStatement not set — getParsedStatement() returns null
+ Assertions.assertFalse(QueryStatsRecorder.shouldRecord(ctx));
+ }
+
+ @Test
+ public void testShouldNotRecordExplain() {
+ Config.enable_query_hit_stats = true;
+ StatementContext ctx = new StatementContext();
+ LogicalPlanAdapter stmt = new LogicalPlanAdapter(
+
Mockito.mock(org.apache.doris.nereids.trees.plans.logical.LogicalPlan.class),
ctx);
+ // isExplain() returns true when explainOptions is non-null.
+ stmt.setIsExplain(new ExplainOptions(false, false, false));
+ ctx.setParsedStatement(stmt);
+ Assertions.assertFalse(QueryStatsRecorder.shouldRecord(ctx));
+ }
+
+ @Test
+ public void testShouldNotRecordDml() {
+ Config.enable_query_hit_stats = true;
+ StatementContext ctx = new StatementContext();
+ Command dmlCommand = Mockito.mock(Command.class);
+ LogicalPlanAdapter stmt = new LogicalPlanAdapter(dmlCommand, ctx);
+ ctx.setParsedStatement(stmt);
+ Assertions.assertFalse(QueryStatsRecorder.shouldRecord(ctx));
+ }
+
+ @Test
+ public void testShouldNotRecordInternalQuery() {
+ Config.enable_query_hit_stats = true;
+ StatementContext ctx = new StatementContext();
+ org.apache.doris.nereids.trees.plans.logical.LogicalPlan selectPlan =
+
Mockito.mock(org.apache.doris.nereids.trees.plans.logical.LogicalPlan.class);
+ LogicalPlanAdapter stmt = new LogicalPlanAdapter(selectPlan, ctx);
+ ctx.setParsedStatement(stmt);
+ QueryState state = new QueryState();
+ state.setInternal(true);
+ ConnectContext connectContext = Mockito.mock(ConnectContext.class);
+ Mockito.when(connectContext.getState()).thenReturn(state);
+ ctx.setConnectContext(connectContext);
+ Assertions.assertFalse(QueryStatsRecorder.shouldRecord(ctx));
+ }
+
+ @Test
+ public void testShouldNotRecordInsert() {
+ Config.enable_query_hit_stats = true;
+ StatementContext ctx = new StatementContext();
+ org.apache.doris.nereids.trees.plans.logical.LogicalPlan selectPlan =
+
Mockito.mock(org.apache.doris.nereids.trees.plans.logical.LogicalPlan.class);
+ LogicalPlanAdapter stmt = new LogicalPlanAdapter(selectPlan, ctx);
+ ctx.setParsedStatement(stmt);
+ ctx.setIsInsert(true);
+ Assertions.assertFalse(QueryStatsRecorder.shouldRecord(ctx));
+ }
+
+ @Test
+ public void testShouldRecordNormalSelect() {
+ Config.enable_query_hit_stats = true;
+ StatementContext ctx = new StatementContext();
+ org.apache.doris.nereids.trees.plans.logical.LogicalPlan selectPlan =
+
Mockito.mock(org.apache.doris.nereids.trees.plans.logical.LogicalPlan.class);
+ LogicalPlanAdapter stmt = new LogicalPlanAdapter(selectPlan, ctx);
+ ctx.setParsedStatement(stmt);
+ Assertions.assertTrue(QueryStatsRecorder.shouldRecord(ctx));
+ }
+
+ // ── collectDeltas (walkPlan) tests
───────────────────────────────────────
+
+ /**
+ * Plan: Filter(k2=1) → Scan[k1(id1), k2(id2)], root output = [k1]
+ * Expected: k1.queryHit=true (SELECT), k2.filterHit=true (WHERE), no
cross-contamination.
+ */
+ @Test
+ public void testFilterHitRecorded() {
+ ExprId id1 = new ExprId(1);
+ ExprId id2 = new ExprId(2);
+ SlotReference k1Slot = mockSlot(id1, "k1");
+ SlotReference k2Slot = mockSlot(id2, "k2");
+ PhysicalOlapScan scan = mockScan(1L, 1L, 1L, 1L,
ImmutableList.of(k1Slot, k2Slot));
+
+ Expression conjunct = Mockito.mock(Expression.class);
+
Mockito.when(conjunct.getInputSlots()).thenReturn(ImmutableSet.of(k2Slot));
+
+ PhysicalFilter<?> filter = Mockito.mock(PhysicalFilter.class);
+ Mockito.when(filter.children()).thenReturn(ImmutableList.of(scan));
+
Mockito.when(filter.getConjuncts()).thenReturn(ImmutableSet.of(conjunct));
+ // Root output: only k1 (SELECT k1 WHERE k2=1)
+ Mockito.when(filter.getOutput()).thenReturn(ImmutableList.of(k1Slot));
+
+ Map<String, StatsDelta> deltas =
QueryStatsRecorder.collectDeltas((PhysicalPlan) filter);
+
+ Assertions.assertEquals(1, deltas.size());
+ StatsDelta delta = deltas.values().iterator().next();
+ // k1: queryHit only (in SELECT output)
+ Assertions.assertNotNull(delta.getColumnStats().get("k1"));
+ Assertions.assertTrue(delta.getColumnStats().get("k1").queryHit);
+ Assertions.assertFalse(delta.getColumnStats().get("k1").filterHit);
+ // k2: filterHit only (in WHERE conjunct, not in SELECT output)
+ Assertions.assertNotNull(delta.getColumnStats().get("k2"));
+ Assertions.assertTrue(delta.getColumnStats().get("k2").filterHit);
+ Assertions.assertFalse(delta.getColumnStats().get("k2").queryHit);
+ }
+
+ /**
+ * Plan: Scan[k1(id1), k2(id2)] as root, root output = [k1]
+ * Expected: k1.queryHit=true, k2 not in delta.
+ */
+ @Test
+ public void testQueryHitFromRootOutput() {
+ ExprId id1 = new ExprId(1);
+ ExprId id2 = new ExprId(2);
+ SlotReference k1Slot = mockSlot(id1, "k1");
+ SlotReference k2Slot = mockSlot(id2, "k2");
+ PhysicalOlapScan scan = mockScan(1L, 1L, 1L, 1L,
ImmutableList.of(k1Slot, k2Slot));
+ // Root output exposes only k1
+ Mockito.when(scan.getOutput()).thenReturn(ImmutableList.of(k1Slot));
+
+ Map<String, StatsDelta> deltas =
QueryStatsRecorder.collectDeltas(scan);
+
+ Assertions.assertEquals(1, deltas.size());
+ StatsDelta delta = deltas.values().iterator().next();
+ Assertions.assertNotNull(delta.getColumnStats().get("k1"));
+ Assertions.assertTrue(delta.getColumnStats().get("k1").queryHit);
+ Assertions.assertFalse(delta.getColumnStats().get("k1").filterHit);
+ // k2 not in root output — no delta entry
+ Assertions.assertNull(delta.getColumnStats().get("k2"));
+ }
+
+ /**
+ * Calling record() twice on the same StatementContext must record only
once.
+ * The latch (isQueryStatsRecorded) is checked after shouldRecord passes,
+ * so we need a valid statement context for shouldRecord to return true.
+ */
+ @Test
+ public void testRecordIsIdempotent() {
+ Config.enable_query_hit_stats = true;
+ StatementContext ctx = new StatementContext();
+ // Set up a proper SELECT so shouldRecord() returns true
+ org.apache.doris.nereids.trees.plans.logical.LogicalPlan logicalPlan =
+
Mockito.mock(org.apache.doris.nereids.trees.plans.logical.LogicalPlan.class);
+ LogicalPlanAdapter stmt = new LogicalPlanAdapter(logicalPlan, ctx);
+ ctx.setParsedStatement(stmt);
+ // Pre-set the latch — simulates a second call after first recording
+ ctx.markQueryStatsRecorded();
+
+ PhysicalOlapScan plan = Mockito.mock(PhysicalOlapScan.class);
+ QueryStatsRecorder.record(plan, ctx);
+ // isQueryStatsRecorded() is true → record() returns before touching
the plan
+ Mockito.verify(plan, Mockito.never()).getOutput();
+ }
+
+ /**
+ * PhysicalLazyMaterializeOlapScan wrapping an inner scan:
+ * the delta key and table metadata must come from the inner scan.
+ */
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testDeferMaterializeScanUsesInnerScan() {
+ ExprId id1 = new ExprId(1);
+ SlotReference k1Slot = mockSlot(id1, "k1");
+
+ // Inner scan has catalogId=2, dbId=2, tableId=2, indexId=2
+ PhysicalOlapScan inner = mockScan(2L, 2L, 2L, 2L,
ImmutableList.of(k1Slot));
+
+ PhysicalLazyMaterializeOlapScan defer =
+ Mockito.mock(PhysicalLazyMaterializeOlapScan.class);
+ Mockito.when(defer.getScan()).thenReturn(inner);
+ // Both walkPlan and collectDeltas root-output loop call getOutput()
+ Mockito.when(defer.getOutput()).thenReturn(ImmutableList.of(k1Slot));
+
+ Map<String, StatsDelta> deltas =
QueryStatsRecorder.collectDeltas((PhysicalPlan) defer);
+
+ Assertions.assertEquals(1, deltas.size());
+ Assertions.assertTrue(deltas.containsKey("2_2_2_2"),
+ "Delta key must use inner scan's table identifiers");
+ StatsDelta delta = deltas.get("2_2_2_2");
+ Assertions.assertNotNull(delta.getColumnStats().get("k1"));
+ Assertions.assertTrue(delta.getColumnStats().get("k1").queryHit);
+ }
+
+ /**
+ * Plan: Filter(k1=1) → Scan[k1(id1)], root output = [k1].
+ * k1 appears in both the WHERE predicate and the SELECT output.
+ * Expected: k1.queryHit=true AND k1.filterHit=true simultaneously.
+ */
+ @Test
+ public void testColumnInBothSelectAndWhere() {
+ ExprId id1 = new ExprId(1);
+ SlotReference k1Slot = mockSlot(id1, "k1");
+ PhysicalOlapScan scan = mockScan(1L, 1L, 1L, 1L,
ImmutableList.of(k1Slot));
+
+ Expression conjunct = Mockito.mock(Expression.class);
+
Mockito.when(conjunct.getInputSlots()).thenReturn(ImmutableSet.of(k1Slot));
+
+ PhysicalFilter<?> filter = Mockito.mock(PhysicalFilter.class);
+ Mockito.when(filter.children()).thenReturn(ImmutableList.of(scan));
+
Mockito.when(filter.getConjuncts()).thenReturn(ImmutableSet.of(conjunct));
+ Mockito.when(filter.getOutput()).thenReturn(ImmutableList.of(k1Slot));
+
+ Map<String, StatsDelta> deltas =
QueryStatsRecorder.collectDeltas((PhysicalPlan) filter);
+
+ Assertions.assertEquals(1, deltas.size());
+ StatsDelta delta = deltas.values().iterator().next();
+ Assertions.assertNotNull(delta.getColumnStats().get("k1"));
+ Assertions.assertTrue(delta.getColumnStats().get("k1").queryHit,
"k1.queryHit must be true");
+ Assertions.assertTrue(delta.getColumnStats().get("k1").filterHit,
"k1.filterHit must be true");
+ }
+
+ /**
+ * Plan has no OlapScan nodes: collectDeltas should return an empty map
without throwing.
+ */
+ @Test
+ public void testNoPlanNodesReturnsEmptyDeltas() {
+ PhysicalPlan leafPlan = Mockito.mock(PhysicalPlan.class);
+ Mockito.when(leafPlan.children()).thenReturn(ImmutableList.of());
+
+ Map<String, StatsDelta> deltas =
QueryStatsRecorder.collectDeltas(leafPlan);
+
+ Assertions.assertTrue(deltas.isEmpty());
+ }
+
+ /**
+ * SlotReference whose getOriginalColumn() returns Optional.empty():
+ * no column entry must be added to the delta (no NPE, no phantom entry).
+ */
+ @Test
+ public void testSlotWithNoOriginalColumnIsSkipped() {
+ ExprId id1 = new ExprId(1);
+ SlotReference slotNoCol = Mockito.mock(SlotReference.class);
+ Mockito.when(slotNoCol.getExprId()).thenReturn(id1);
+
Mockito.when(slotNoCol.getOriginalColumn()).thenReturn(Optional.empty());
+
+ PhysicalOlapScan scan = mockScan(1L, 1L, 1L, 1L,
ImmutableList.of(slotNoCol));
+
+ Map<String, StatsDelta> deltas =
QueryStatsRecorder.collectDeltas(scan);
+
+ Assertions.assertEquals(1, deltas.size());
+
Assertions.assertTrue(deltas.values().iterator().next().getColumnStats().isEmpty(),
+ "No column stats should be recorded for a slot with no
originalColumn");
+ }
+
+ /**
+ * Filter with two conjuncts each referencing a different column:
+ * both columns must receive filterHit; neither must receive queryHit.
+ */
+ @Test
+ public void testMultipleConjunctsAllGetFilterHit() {
+ ExprId id1 = new ExprId(1);
+ ExprId id2 = new ExprId(2);
+ ExprId id3 = new ExprId(3);
+ SlotReference k1Slot = mockSlot(id1, "k1");
+ SlotReference k2Slot = mockSlot(id2, "k2");
+ SlotReference k3Slot = mockSlot(id3, "k3");
+ PhysicalOlapScan scan = mockScan(1L, 1L, 1L, 1L,
+ ImmutableList.of(k1Slot, k2Slot, k3Slot));
+
+ Expression conj1 = Mockito.mock(Expression.class);
+
Mockito.when(conj1.getInputSlots()).thenReturn(ImmutableSet.of(k2Slot));
+ Expression conj2 = Mockito.mock(Expression.class);
+
Mockito.when(conj2.getInputSlots()).thenReturn(ImmutableSet.of(k3Slot));
+
+ PhysicalFilter<?> filter = Mockito.mock(PhysicalFilter.class);
+ Mockito.when(filter.children()).thenReturn(ImmutableList.of(scan));
+ Mockito.when(filter.getConjuncts()).thenReturn(ImmutableSet.of(conj1,
conj2));
+ Mockito.when(filter.getOutput()).thenReturn(ImmutableList.of(k1Slot));
+
+ Map<String, StatsDelta> deltas =
QueryStatsRecorder.collectDeltas((PhysicalPlan) filter);
+
+ Assertions.assertEquals(1, deltas.size());
+ StatsDelta delta = deltas.values().iterator().next();
+ Assertions.assertTrue(delta.getColumnStats().get("k2").filterHit,
"k2.filterHit must be true");
+ Assertions.assertFalse(delta.getColumnStats().get("k2").queryHit,
"k2.queryHit must be false");
+ Assertions.assertTrue(delta.getColumnStats().get("k3").filterHit,
"k3.filterHit must be true");
+ Assertions.assertFalse(delta.getColumnStats().get("k3").queryHit,
"k3.queryHit must be false");
+ }
+
+ /**
+ * Two scans from different tables in one plan: each must produce an
independent
+ * StatsDelta with a distinct key, recording only its own table's columns.
+ */
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testTwoDifferentTablesProduceSeparateDeltas() {
+ ExprId id1 = new ExprId(1);
+ ExprId id2 = new ExprId(2);
+ SlotReference k1Slot = mockSlot(id1, "k1");
+ SlotReference k2Slot = mockSlot(id2, "k2");
+
+ PhysicalOlapScan scan1 = mockScan(1L, 1L, 1L, 1L,
ImmutableList.of(k1Slot));
+ PhysicalOlapScan scan2 = mockScan(2L, 2L, 2L, 2L,
ImmutableList.of(k2Slot));
+
+ PhysicalPlan join = Mockito.mock(PhysicalPlan.class);
+ Mockito.when(join.children()).thenReturn(ImmutableList.of(scan1,
scan2));
+ Mockito.when(join.getOutput()).thenReturn(ImmutableList.of(k1Slot,
k2Slot));
+
+ Map<String, StatsDelta> deltas =
QueryStatsRecorder.collectDeltas(join);
+
+ Assertions.assertEquals(2, deltas.size(), "Each table must have its
own delta");
+ Assertions.assertTrue(deltas.containsKey("1_1_1_1"), "scan1 delta
missing");
+ Assertions.assertTrue(deltas.containsKey("2_2_2_2"), "scan2 delta
missing");
+
Assertions.assertTrue(deltas.get("1_1_1_1").getColumnStats().get("k1").queryHit);
+ Assertions.assertNull(deltas.get("1_1_1_1").getColumnStats().get("k2"),
+ "scan1 must not record scan2's column");
+
Assertions.assertTrue(deltas.get("2_2_2_2").getColumnStats().get("k2").queryHit);
+ Assertions.assertNull(deltas.get("2_2_2_2").getColumnStats().get("k1"),
+ "scan2 must not record scan1's column");
+ }
+
+ /**
+ * Scan whose getTable() returns null: getOrCreateDelta returns null and
+ * no stats are recorded — no NPE.
+ */
+ @Test
+ public void testNullTableInScanDoesNotCrash() {
+ ExprId id1 = new ExprId(1);
+ SlotReference k1Slot = mockSlot(id1, "k1");
+
+ PhysicalOlapScan scan = Mockito.mock(PhysicalOlapScan.class);
+ Mockito.when(scan.getTable()).thenReturn(null);
+ Mockito.when(scan.getOutput()).thenReturn(ImmutableList.of(k1Slot));
+ Mockito.when(scan.children()).thenReturn(ImmutableList.of());
+
+ Map<String, StatsDelta> deltas =
QueryStatsRecorder.collectDeltas(scan);
+
+ Assertions.assertTrue(deltas.isEmpty(), "Null-table scan must not
create a delta");
+ }
+
+ // ── helpers
──────────────────────────────────────────────────────────────
+
+ private SlotReference mockSlot(ExprId exprId, String columnName) {
+ SlotReference slot = Mockito.mock(SlotReference.class);
+ Mockito.when(slot.getExprId()).thenReturn(exprId);
+ Column col = Mockito.mock(Column.class);
+ Mockito.when(col.getName()).thenReturn(columnName);
+ Mockito.when(slot.getOriginalColumn()).thenReturn(Optional.of(col));
+ return slot;
+ }
+
+ @SuppressWarnings("unchecked")
+ private PhysicalOlapScan mockScan(long catalogId, long dbId, long tableId,
long indexId,
+ List<Slot> outputSlots) {
+ PhysicalOlapScan scan = Mockito.mock(PhysicalOlapScan.class);
+ OlapTable table = Mockito.mock(OlapTable.class);
+ DatabaseIf<OlapTable> db = (DatabaseIf<OlapTable>)
Mockito.mock(DatabaseIf.class);
+ Mockito.when(table.getCatalogId()).thenReturn(catalogId);
+ Mockito.when(table.getId()).thenReturn(tableId);
+ Mockito.when(db.getId()).thenReturn(dbId);
+ Mockito.when(scan.getTable()).thenReturn(table);
+ Mockito.when(scan.getDatabase()).thenReturn(db);
+ Mockito.when(scan.getSelectedIndexId()).thenReturn(indexId);
+ Mockito.when(scan.getOutput()).thenReturn((List<Slot>) (List<?>)
outputSlots);
+ Mockito.when(scan.children()).thenReturn(ImmutableList.of());
+ return scan;
+ }
+}
diff --git a/regression-test/data/query_p0/stats/query_stats_test.out
b/regression-test/data/query_p0/stats/query_stats_test.out
deleted file mode 100644
index 984413ec2e0..00000000000
--- a/regression-test/data/query_p0/stats/query_stats_test.out
+++ /dev/null
@@ -1,52 +0,0 @@
--- This file is automatically generated. You should know what you did if you
want to edit this
--- !sql --
-k0 0 0
-k1 0 0
-k2 0 0
-k3 0 0
-k4 0 0
-k5 0 0
-k6 0 0
-k10 0 0
-k11 0 0
-k7 0 0
-k8 0 0
-k9 0 0
-k12 0 0
-k13 0 0
-
--- !sql --
-k0 0 0
-k1 0 0
-k2 0 0
-k3 0 0
-k4 0 0
-k5 0 0
-k6 0 0
-k10 0 0
-k11 0 0
-k7 0 0
-k8 0 0
-k9 0 0
-k12 0 0
-k13 0 0
-
--- !sql --
-stats_table 0
-
--- !sql --
-stats_table k0 0 0
- k1 0 0
- k2 0 0
- k3 0 0
- k4 0 0
- k5 0 0
- k6 0 0
- k10 0 0
- k11 0 0
- k7 0 0
- k8 0 0
- k9 0 0
- k12 0 0
- k13 0 0
-
diff --git a/regression-test/suites/query_p0/stats/query_stats_test.groovy
b/regression-test/suites/query_p0/stats/query_stats_test.groovy
index 492cca65899..aa9b7e5a0b6 100644
--- a/regression-test/suites/query_p0/stats/query_stats_test.groovy
+++ b/regression-test/suites/query_p0/stats/query_stats_test.groovy
@@ -38,19 +38,121 @@ suite("query_stats_test") {
) engine=olap
DISTRIBUTED BY HASH(`k1`) BUCKETS 1 properties("replication_num" = "1")
"""
- sql "admin set frontend config (\"enable_query_hit_stats\"=\"true\");"
+
+ def origNereids = sql("select @@enable_nereids_planner")[0][0]
+ def origCache = sql("select @@enable_query_cache")[0][0]
+
+ sql "admin set all frontends config (\"enable_query_hit_stats\"=\"true\");"
+ sql "set enable_nereids_planner = true"
+ sql "set enable_query_cache = false"
+
+ sql """INSERT INTO ${tbName} VALUES
+ (true, 1, 100, 1000, 10000, 123.456, 'A1234', '2024-01-01',
'2024-01-01 10:00:00', 'alpha', 1.23, 4.56, 'text1', 12345678901234567890),
+ (false, -5, 200, 2000, 20000, 234.567, 'B2345', '2024-02-02',
'2024-02-02 11:30:00', 'beta', 2.34, 5.67, 'text2', 22345678901234567890),
+ (true, 10, -300, 3000, 30000, 345.678, 'C3456', '2024-03-03',
'2024-03-03 12:45:00', 'gamma', 3.45, 6.78, 'text3', 32345678901234567890),
+ (NULL, 0, 0, 0, 0, 0.000, 'D4567', '2024-04-04',
'2024-04-04 00:00:00', 'delta', 0.00, 0.00, 'text4', 42345678901234567890),
+ (false, 127, 32767, 2147483647, 9223372036854775807, 999.999,
'E5678', '2024-05-05', '2024-05-05 23:59:59', 'epsilon', 9.99, 9.99, 'text5',
52345678901234567890)"""
+
sql "clean all query stats"
- explain {
- sql("select k1 from ${tbName} where k1 = 1")
- }
- qt_sql "show query stats from ${tbName}"
+ explain { sql("select k1 from ${tbName} where k1 = 1") }
+ def explainStats = sql "show query stats from ${tbName}"
+ for (row in explainStats) {
+ assertEquals(0, row[1] as int)
+ assertEquals(0, row[2] as int)
+ }
+ // Counts may exceed 1 in multi-FE clusters; verify right columns hit,
wrong ones stay 0.
sql "select k1 from ${tbName} where k0 = 1"
sql "select k4 from ${tbName} where k2 = 1991"
+ def stats1 = sql "show query stats from ${tbName}"
+ def s1k1 = stats1.find { it[0] == "k1" }
+ def s1k0 = stats1.find { it[0] == "k0" }
+ def s1k4 = stats1.find { it[0] == "k4" }
+ def s1k2 = stats1.find { it[0] == "k2" }
+ assertNotNull(s1k1)
+ assertNotNull(s1k0)
+ assertNotNull(s1k4)
+ assertNotNull(s1k2)
+ assertTrue((s1k1[1] as int) >= 1)
+ assertTrue((s1k0[2] as int) >= 1)
+ assertTrue((s1k4[1] as int) >= 1)
+ assertTrue((s1k2[2] as int) >= 1)
+ assertEquals(0, s1k1[2] as int)
+ assertEquals(0, s1k0[1] as int)
+ for (col in ["k3", "k5", "k6", "k10", "k11", "k7", "k8", "k9", "k12",
"k13"]) {
+ def row = stats1.find { it[0] == col }
+ assertNotNull(row)
+ assertEquals(0, row[1] as int)
+ assertEquals(0, row[2] as int)
+ }
+ def allStats1 = sql "show query stats from ${tbName} all"
+ assertTrue((allStats1[0][1] as int) >= 2)
+ def verboseStats1 = sql "show query stats from ${tbName} all verbose"
+ assertTrue(verboseStats1.size() > 0)
+
+ sql "clean all query stats"
+ sql "select k3 from ${tbName} where k5 = 1.0"
+ def stats2 = sql "show query stats from ${tbName}"
+ def s2k3 = stats2.find { it[0] == "k3" }
+ def s2k5 = stats2.find { it[0] == "k5" }
+ assertNotNull(s2k3)
+ assertNotNull(s2k5)
+ assertTrue((s2k3[1] as int) >= 1)
+ assertTrue((s2k5[2] as int) >= 1)
+ assertEquals(0, s2k3[2] as int)
+ assertEquals(0, s2k5[1] as int)
+ for (col in ["k0", "k1", "k2", "k4", "k6", "k10", "k11", "k7", "k8", "k9",
"k12", "k13"]) {
+ def row = stats2.find { it[0] == col }
+ assertNotNull(row)
+ assertEquals(0, row[1] as int)
+ assertEquals(0, row[2] as int)
+ }
+ def allStats2 = sql "show query stats from ${tbName} all"
+ assertTrue((allStats2[0][1] as int) >= 1)
+
+ // Column in both SELECT and WHERE.
+ sql "clean all query stats"
+ sql "select k1 from ${tbName} where k1 = 1"
+ def stats3 = sql "show query stats from ${tbName}"
+ def s3k1 = stats3.find { it[0] == "k1" }
+ assertNotNull(s3k1)
+ assertTrue((s3k1[1] as int) >= 1)
+ assertTrue((s3k1[2] as int) >= 1)
+ for (col in ["k0", "k2", "k3", "k4", "k5", "k6", "k10", "k11", "k7", "k8",
"k9", "k12", "k13"]) {
+ def row = stats3.find { it[0] == col }
+ assertNotNull(row)
+ assertEquals(0, row[1] as int)
+ assertEquals(0, row[2] as int)
+ }
+
+ // INSERT INTO ... SELECT must not record stats.
+ sql "clean all query stats"
+ sql "insert into ${tbName} select * from ${tbName} where k1 > 0"
+ def insertStats = sql "show query stats from ${tbName}"
+ for (row in insertStats) {
+ assertEquals(0, row[1] as int)
+ assertEquals(0, row[2] as int)
+ }
+
+ // Alias gap: k1.queryHit = 0 until Part 2 walks Project nodes.
+ sql "clean all query stats"
+ sql "select k1 as x from ${tbName} where k2 = 1"
+ def aliasResult = sql "show query stats from ${tbName}"
+ def arK1 = aliasResult.find { it[0] == "k1" }
+ def arK2 = aliasResult.find { it[0] == "k2" }
+ assertNotNull(arK1)
+ assertNotNull(arK2)
+ assertEquals(0, arK1[1] as int)
+ assertTrue((arK2[2] as int) >= 1)
+
+ // Self-join: StatsDelta dedup keeps table count = 1 per FE.
+ sql "clean all query stats"
+ sql "select a.k1 from ${tbName} a, ${tbName} b where a.k1 = b.k1"
+ def joinResult = sql "show query stats from ${tbName} all"
+ assertTrue((joinResult[0][1] as int) >= 1)
- qt_sql "show query stats from ${tbName}"
- qt_sql "show query stats from ${tbName} all"
- qt_sql "show query stats from ${tbName} all verbose"
- sql "admin set frontend config (\"enable_query_hit_stats\"=\"false\");"
+ sql "admin set all frontends config
(\"enable_query_hit_stats\"=\"false\");"
+ sql "set enable_nereids_planner = ${origNereids}"
+ sql "set enable_query_cache = ${origCache}"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
