This is an automated email from the ASF dual-hosted git repository.
englefly pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new cd71f991355 [fix](fe) Do not scale numNulls of partition statistics
again (#69636)
cd71f991355 is described below
commit cd71f9913556f65ce3b08696fbae8769dbf80be6
Author: minghong <[email protected]>
AuthorDate: Tue Oct 27 15:49:03 2026 +0800
[fix](fe) Do not scale numNulls of partition statistics again (#69636)
### What problem does this PR solve?
Problem Summary:
When only a subset of partitions is selected,
`StatsCalculator.computeOlapScan` scales `numNulls` of the column
statistics by `selectedPartitionsRowCount / tableRowCount`, because the
table level statistics cover the whole table.
With `enable_partition_analyze = true` the statistics of the selected
partitions are used instead, and their `numNulls` already only covers
the selected partitions. The scaling was applied to them anyway, so
`numNulls` was scaled a second time and became too small.
Reproduction: a table with 3 partitions of 4 rows each and 1 null in
`val` per partition. Pruning to one partition (4 of 12 rows) reports
`numNulls = 0.3333` in `EXPLAIN MEMO PLAN` instead of `1`, and `select *
from t where val is null` is estimated to return `0.33` rows instead of
`1`:
```
val#1 -> ndv=1.0000, min=2.000000(2), max=2.000000(2), count=4.0000,
numNulls=0.3333, ...
```
Fix:
- make the partition statistics helper return `null` instead of silently
falling back to the table level statistics, so that the caller knows
which scope the returned statistic has;
- scale `numNulls` only when the table level statistics are really used,
i.e. when `enable_partition_analyze` is off, or when the partition
statistics are unavailable and the scan falls back to the table level
ones.
### Release note
None
### Check List (For Author)
- Test: Unit Test
-
`StatsCalculatorTest#testComputeOlapScanDoesNotScaleNumNullsOfPartitionStatistics`
reproduces the bug: without the fix it fails with `expected: <1.0> but
was: <0.3333333333333333>`.
-
`StatsCalculatorTest#testComputeOlapScanFallsBackToTableStatisticsWhenPartitionStatisticIsUnknown`
covers the fallback path, which still has to be scaled.
-
`StatsCalculatorTest#testComputeOlapScanScalesNumNullsForSelectedPartitions`
(existing case, table level statistics) still passes.
- `mvn -pl fe-core -am test -Dtest=StatsCalculatorTest` passes (19
tests), checkstyle passes.
- Behavior changed: No
- Does this need documentation: No
---
.../doris/nereids/stats/StatsCalculator.java | 84 +++++++-----
.../doris/nereids/stats/StatsCalculatorTest.java | 145 +++++++++++++++++----
2 files changed, 171 insertions(+), 58 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
index e3071cfd25f..e1482c1bdcd 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -588,10 +588,20 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
&&
connectContext.getSessionVariable().enablePartitionAnalyze;
for (SlotReference slot : visibleOutputSlots) {
ColumnStatistic cache;
+ // numNulls of the table level statistics covers the whole
table, so it has to be scaled
+ // to the selected partitions. numNulls of the partition level
statistics is already
+ // scoped to them and must not be scaled again.
+ boolean scaleNumNulls;
if (enablePartitionStatics) {
- cache = getColumnStatistic(olapTableStats, slot.getName(),
selectedPartitionNames);
+ cache = getPartitionColumnStatistic(olapTableStats,
slot.getName(), selectedPartitionNames);
+ scaleNumNulls = cache == null;
+ if (scaleNumNulls) {
+ // partition level statistics are unavailable, fall
back to table level statistics
+ cache =
olapTableStats.getColumnStatistics(slot.getName(), connectContext);
+ }
} else {
cache = olapTableStats.getColumnStatistics(slot.getName(),
connectContext);
+ scaleNumNulls = true;
}
if (slot.getOriginalColumn().isPresent()) {
cache = updateMinMaxForPartitionKey(olapTable,
selectedPartitionNames, slot, cache);
@@ -599,9 +609,10 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
ColumnStatisticBuilder colStatsBuilder = new
ColumnStatisticBuilder(cache,
selectedPartitionsRowCount);
colStatsBuilder.normalizeAvgSizeByte(slot.getDataType());
- //scale null_num
- double scale = tableRowCount == 0 ? 1 :
selectedPartitionsRowCount / tableRowCount;
- colStatsBuilder.setNumNulls(colStatsBuilder.getNumNulls() *
scale);
+ if (scaleNumNulls) {
+ double scale = tableRowCount == 0 ? 1 :
selectedPartitionsRowCount / tableRowCount;
+ colStatsBuilder.setNumNulls(colStatsBuilder.getNumNulls()
* scale);
+ }
builder.putColumnStatistics(slot, colStatsBuilder.build());
}
checkIfUnknownStatsUsedAsKey(builder);
@@ -1206,46 +1217,47 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
}
/**
- * - partition pruned: try to get partition col stats, if failed, then
fall back to table level col stats
- * - no partition pruned: get table level col stats
+ * Merge the column statistics of the given partitions.
+ *
+ * <p>The returned statistic is scoped to the given partitions, i.e. its
count and numNulls only
+ * cover them. Returns null if the statistics of any partition is unknown,
so that the caller can
+ * decide whether to fall back to the table level column statistics.
*/
- private ColumnStatistic getColumnStatistic(
+ private ColumnStatistic getPartitionColumnStatistic(
OlapTableStatistics olapTableStatistics, String colName,
List<String> partitionNames) {
if (connectContext != null &&
connectContext.getState().isPlanWithUnKnownColumnStats()) {
return ColumnStatistic.UNKNOWN;
}
- if (!partitionNames.isEmpty()) {
- PartitionColumnStatisticBuilder builder = new
PartitionColumnStatisticBuilder();
- boolean hasUnknown = false;
- // check if there is any unknown stats to avoid unnecessary
partition column stats merge.
- List<PartitionColumnStatistic> pColStatsLists = new
ArrayList<>(partitionNames.size());
- for (String partitionName : partitionNames) {
- PartitionColumnStatistic pcolStats
- =
olapTableStatistics.getPartitionColumnStatistics(partitionName, colName,
connectContext);
- if (pcolStats.isUnKnown) {
- hasUnknown = true;
- break;
- } else {
- pColStatsLists.add(pcolStats);
- }
+ if (partitionNames.isEmpty()) {
+ return null;
+ }
+ PartitionColumnStatisticBuilder builder = new
PartitionColumnStatisticBuilder();
+ boolean hasUnknown = false;
+ // check if there is any unknown stats to avoid unnecessary partition
column stats merge.
+ List<PartitionColumnStatistic> pColStatsLists = new
ArrayList<>(partitionNames.size());
+ for (String partitionName : partitionNames) {
+ PartitionColumnStatistic pcolStats
+ =
olapTableStatistics.getPartitionColumnStatistics(partitionName, colName,
connectContext);
+ if (pcolStats.isUnKnown) {
+ hasUnknown = true;
+ break;
}
- if (!hasUnknown) {
- boolean isFirst = true;
- // try to merge partition column stats
- for (PartitionColumnStatistic pcolStats : pColStatsLists) {
- if (isFirst) {
- builder = new
PartitionColumnStatisticBuilder(pcolStats);
- isFirst = false;
- } else {
- builder.merge(pcolStats);
- }
- }
- return builder.toColumnStatistics();
+ pColStatsLists.add(pcolStats);
+ }
+ if (hasUnknown) {
+ return null;
+ }
+ boolean isFirst = true;
+ // try to merge partition column stats
+ for (PartitionColumnStatistic pcolStats : pColStatsLists) {
+ if (isFirst) {
+ builder = new PartitionColumnStatisticBuilder(pcolStats);
+ isFirst = false;
+ } else {
+ builder.merge(pcolStats);
}
}
-
- // if any partition-col-stats is unknown, fall back to table level col
stats
- return olapTableStatistics.getColumnStatistics(colName,
connectContext);
+ return builder.toColumnStatistics();
}
/**
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
index 078368cb71a..f5bbc72e3db 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
@@ -61,7 +61,10 @@ import org.apache.doris.statistics.analysis.AnalysisManager;
import org.apache.doris.statistics.cache.StatisticsCache;
import org.apache.doris.statistics.model.ColumnStatistic;
import org.apache.doris.statistics.model.ColumnStatisticBuilder;
+import org.apache.doris.statistics.model.PartitionColumnStatistic;
+import org.apache.doris.statistics.model.PartitionColumnStatisticBuilder;
import org.apache.doris.statistics.model.Statistics;
+import org.apache.doris.statistics.util.Hll128;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
@@ -214,9 +217,98 @@ public class StatsCalculatorTest {
@Test
public void testComputeOlapScanScalesNumNullsForSelectedPartitions() {
+ // table level statistics: the whole table has 12 rows and 3 nulls,
the selected partition has 4 rows
+ try (MockPartitionPrunedScan fixture = mockPartitionPrunedScan(false,
newTableStatistic(1), null)) {
+ Statistics statistics = fixture.computeOlapScan();
+ ColumnStatistic result =
statistics.findColumnStatistics(fixture.slot);
+
+ Assertions.assertEquals(4, statistics.getRowCount(), 0.001);
+ Assertions.assertNotNull(result);
+ Assertions.assertEquals(1, result.ndv, 0.001);
+ Assertions.assertEquals(2, result.minValue, 0.001);
+ Assertions.assertEquals(2, result.maxValue, 0.001);
+ Assertions.assertEquals(4, result.count, 0.001);
+ Assertions.assertEquals(1, result.numNulls, 0.001);
+ Assertions.assertEquals("2", result.minExpr.getStringValue());
+ Assertions.assertEquals("2", result.maxExpr.getStringValue());
+ }
+ }
+
+ @Test
+ public void testComputeOlapScanDoesNotScaleNumNullsOfPartitionStatistics()
{
+ // partition level statistics of the selected partition: 4 rows and 1
of them is null
+ PartitionColumnStatistic partitionStatistic = new
PartitionColumnStatisticBuilder(4)
+ .setNdv(newHll128(2))
+ .setNumNulls(1)
+ .setMinValue(2)
+ .setMaxValue(2)
+ .setMinExpr(new IntLiteral(2))
+ .setMaxExpr(new IntLiteral(2))
+ .build();
+ try (MockPartitionPrunedScan fixture = mockPartitionPrunedScan(true,
+ newTableStatistic(2), partitionStatistic)) {
+ Statistics statistics = fixture.computeOlapScan();
+ ColumnStatistic result =
statistics.findColumnStatistics(fixture.slot);
+
+ Assertions.assertEquals(4, statistics.getRowCount(), 0.001);
+ Assertions.assertNotNull(result);
+ // ndv comes from the partition level statistics, the table level
ones have ndv=2
+ Assertions.assertEquals(1, result.ndv, 0.001);
+ Assertions.assertEquals(4, result.count, 0.001);
+ // numNulls of the partition level statistics already covers the
selected partition only,
+ // it must not be scaled by selectedPartitionsRowCount /
tableRowCount (1 * 4 / 12) again
+ Assertions.assertEquals(1, result.numNulls, 0.001);
+ }
+ }
+
+ @Test
+ public void
testComputeOlapScanFallsBackToTableStatisticsWhenPartitionStatisticIsUnknown() {
+ try (MockPartitionPrunedScan fixture = mockPartitionPrunedScan(true,
+ newTableStatistic(2), PartitionColumnStatistic.UNKNOWN)) {
+ Statistics statistics = fixture.computeOlapScan();
+ ColumnStatistic result =
statistics.findColumnStatistics(fixture.slot);
+
+ Assertions.assertEquals(4, statistics.getRowCount(), 0.001);
+ Assertions.assertNotNull(result);
+ // ndv comes from the table level statistics, i.e. it fell back to
them
+ Assertions.assertEquals(2, result.ndv, 0.001);
+ Assertions.assertEquals(4, result.count, 0.001);
+ // the table level statistic covers the whole table, so numNulls
is scaled: 3 * 4 / 12
+ Assertions.assertEquals(1, result.numNulls, 0.001);
+ }
+ }
+
+ private static ColumnStatistic newTableStatistic(double ndv) {
+ return new ColumnStatisticBuilder(12)
+ .setNdv(ndv)
+ .setMinValue(2)
+ .setMaxValue(2)
+ .setMinExpr(new IntLiteral(2))
+ .setMaxExpr(new IntLiteral(2))
+ .setNumNulls(3)
+ .build();
+ }
+
+ private static Hll128 newHll128(long value) {
+ Hll128 hll128 = new Hll128();
+ hll128.update(value);
+ return hll128;
+ }
+
+ /**
+ * Mock an OLAP scan over a table with 3 partitions and 12 rows, whose
partition pruning selects
+ * exactly one partition with 4 rows.
+ *
+ * @param enablePartitionStatics value of the session variable
enable_partition_analyze
+ * @param tableStatistic statistics of the whole table
+ * @param partitionStatistic statistics of the selected partition, null if
not stubbed
+ */
+ private static MockPartitionPrunedScan mockPartitionPrunedScan(boolean
enablePartitionStatics,
+ ColumnStatistic tableStatistic, PartitionColumnStatistic
partitionStatistic) {
ConnectContext previousContext = ConnectContext.get();
ConnectContext connectContext = new ConnectContext();
connectContext.setThreadLocalInfo();
+ connectContext.getSessionVariable().enablePartitionAnalyze =
enablePartitionStatics;
Env env = Mockito.mock(Env.class);
AnalysisManager analysisManager = Mockito.mock(AnalysisManager.class);
@@ -232,19 +324,15 @@ public class StatsCalculatorTest {
Column column = new Column("val", PrimitiveType.INT);
SlotReference slot = new SlotReference(new ExprId(1), "val",
IntegerType.INSTANCE, true,
ImmutableList.of("test", "tbl"), table, column, table, column);
- ColumnStatistic tableStatistic = new ColumnStatisticBuilder(12)
- .setNdv(1)
- .setMinValue(2)
- .setMaxValue(2)
- .setMinExpr(new IntLiteral(2))
- .setMaxExpr(new IntLiteral(2))
- .setNumNulls(3)
- .build();
Mockito.when(env.getAnalysisManager()).thenReturn(analysisManager);
Mockito.when(env.getStatisticsCache()).thenReturn(statisticsCache);
Mockito.when(statisticsCache.getOlapTableStats(scan)).thenReturn(olapTableStatistics);
Mockito.when(olapTableStatistics.getColumnStatistics("val",
connectContext)).thenReturn(tableStatistic);
+ if (partitionStatistic != null) {
+
Mockito.when(olapTableStatistics.getPartitionColumnStatistics("p1", "val",
connectContext))
+ .thenReturn(partitionStatistic);
+ }
Mockito.when(scan.getTable()).thenReturn(table);
Mockito.when(scan.getSelectedIndexId()).thenReturn(baseIndexId);
Mockito.when(scan.getSelectedPartitionIds()).thenReturn(ImmutableList.of(selectedPartitionId));
@@ -258,22 +346,35 @@ public class StatsCalculatorTest {
Mockito.when(table.getQualifiedDbName()).thenReturn("test");
Mockito.when(selectedPartition.getName()).thenReturn("p1");
- try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
- mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class);
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ return new MockPartitionPrunedScan(previousContext, scan, slot,
mockedEnv);
+ }
- Statistics statistics = new StatsCalculator((CascadesContext)
null).computeOlapScan(scan);
- ColumnStatistic result = statistics.findColumnStatistics(slot);
+ /**
+ * Mocks of {@link #mockPartitionPrunedScan}, they have to be closed after
computing the statistics.
+ */
+ private static final class MockPartitionPrunedScan implements
AutoCloseable {
+ private final ConnectContext previousContext;
+ private final LogicalOlapScan scan;
+ private final SlotReference slot;
+ private final MockedStatic<Env> mockedEnv;
+
+ private MockPartitionPrunedScan(ConnectContext previousContext,
LogicalOlapScan scan, SlotReference slot,
+ MockedStatic<Env> mockedEnv) {
+ this.previousContext = previousContext;
+ this.scan = scan;
+ this.slot = slot;
+ this.mockedEnv = mockedEnv;
+ }
- Assertions.assertEquals(4, statistics.getRowCount(), 0.001);
- Assertions.assertNotNull(result);
- Assertions.assertEquals(1, result.ndv, 0.001);
- Assertions.assertEquals(2, result.minValue, 0.001);
- Assertions.assertEquals(2, result.maxValue, 0.001);
- Assertions.assertEquals(4, result.count, 0.001);
- Assertions.assertEquals(1, result.numNulls, 0.001);
- Assertions.assertEquals("2", result.minExpr.getStringValue());
- Assertions.assertEquals("2", result.maxExpr.getStringValue());
- } finally {
+ private Statistics computeOlapScan() {
+ return new StatsCalculator((CascadesContext)
null).computeOlapScan(scan);
+ }
+
+ @Override
+ public void close() {
+ mockedEnv.close();
ConnectContext.remove();
if (previousContext != null) {
previousContext.setThreadLocalInfo();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
