This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new d8e8897a725 branch-4.2: [fix](statistics) full analyze not collect hot
value by default #63625 (#68298)
d8e8897a725 is described below
commit d8e8897a725771afb651145f2e66d81df7978058
Author: yujun <[email protected]>
AuthorDate: Mon Sep 21 18:15:45 2026 +0800
branch-4.2: [fix](statistics) full analyze not collect hot value by default
#63625 (#68298)
cherry-pick: #63625
Conflicts are limited to FE test files, because branch-4.2 statistics
tests still use JMockit while master has moved to Mockito, and
`TableNameInfo`/`PartitionNamesInfo` live in `org.apache.doris.info` on
branch-4.2. The test additions were re-expressed in the branch-4.2
idiom; main-source changes are identical to #63625.
`testDoSampleAlwaysCollectsHotValue` also stubs
`getSampleCollectInfo(..)` with a `SampleCollectInfo` instead of
`useLinearAnalyzeTemplate()`, since branch-4.2 `doSample` picks the
template through the sample-algorithm decision, and
`AnalysisManagerTest` now uses the 4-arg `mockAnalyzeCommand(..)` helper
so the Paimon row-count case keeps a non-null `AnalyzeProperties`.
FE UT: AnalyzeTableCommandTest, OlapAnalysisTaskTest,
AnalysisManagerTest, HMSAnalysisTaskTest, StatisticsUtilTest
---
.../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 1 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 2 +
.../apache/doris/analysis/AnalyzeProperties.java | 20 ++++
.../doris/nereids/parser/LogicalPlanBuilder.java | 2 +
.../trees/plans/commands/AnalyzeCommand.java | 6 ++
.../org/apache/doris/statistics/AnalysisInfo.java | 10 +-
.../doris/statistics/AnalysisInfoBuilder.java | 11 +-
.../apache/doris/statistics/AnalysisManager.java | 3 +
.../apache/doris/statistics/BaseAnalysisTask.java | 23 ++++
.../doris/statistics/ExternalAnalysisTask.java | 16 +--
.../apache/doris/statistics/OlapAnalysisTask.java | 12 ++-
.../doris/statistics/StatisticsAutoCollector.java | 1 +
.../plans/commands/AnalyzeTableCommandTest.java | 66 +++++++++++-
.../doris/statistics/AnalysisManagerTest.java | 96 ++++++++++++++---
.../doris/statistics/HMSAnalysisTaskTest.java | 60 +++++++++++
.../doris/statistics/OlapAnalysisTaskTest.java | 117 +++++++++++++++++++++
.../doris/statistics/util/StatisticsUtilTest.java | 4 +
.../distinct_split/disitinct_split.out | 57 ++++------
.../statistics/test_full_analyze_hot_value.groovy | 59 +++++++----
.../suites/statistics/test_hot_value.groovy | 2 +-
20 files changed, 481 insertions(+), 87 deletions(-)
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
index 4a762c33201..4334e001e38 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
@@ -287,6 +287,7 @@ HISTOGRAM: 'HISTOGRAM';
HLL: 'HLL';
HLL_UNION: 'HLL_UNION';
HOSTNAME: 'HOSTNAME';
+HOT: 'HOT';
HOTSPOT: 'HOTSPOT';
HOUR: 'HOUR';
HOURS: 'HOURS';
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 9ceaeef405f..372429744dc 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -917,6 +917,7 @@ analyzeProperties
| FULL
| SQL
| HISTOGRAM
+ | (HOT VALUE)
| (SAMPLE ((ROWS rows=INTEGER_VALUE) | (PERCENT percent=INTEGER_VALUE)) )
| (BUCKETS bucket=INTEGER_VALUE)
| (PERIOD periodInSecond=INTEGER_VALUE)
@@ -2175,6 +2176,7 @@ nonReserved
| HISTOGRAM
| HLL_UNION
| HOSTNAME
+ | HOT
| HOTSPOT
| HOUR
| HOURS
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeProperties.java
index 8c7deafb430..8c5f3671b77 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyzeProperties.java
@@ -45,6 +45,7 @@ public class AnalyzeProperties {
public static final String PROPERTY_FORCE_FULL = "force.full";
public static final String PROPERTY_EXTERNAL_TABLE_USE_SQL =
"external.table.use.sql";
public static final String PROPERTY_USE_AUTO_ANALYZER =
"use.auto.analyzer";
+ public static final String PROPERTY_COLLECT_HOT_VALUE =
"collect.hot.value";
public static final AnalyzeProperties DEFAULT_PROP = new
AnalyzeProperties(new HashMap<String, String>() {
{
@@ -74,6 +75,7 @@ public class AnalyzeProperties {
.add(PROPERTY_FORCE_FULL)
.add(PROPERTY_EXTERNAL_TABLE_USE_SQL)
.add(PROPERTY_USE_AUTO_ANALYZER)
+ .add(PROPERTY_COLLECT_HOT_VALUE)
.build();
public AnalyzeProperties(Map<String, String> properties) {
@@ -96,6 +98,7 @@ public class AnalyzeProperties {
checkAnalysisMode(msgTemplate);
checkAnalysisType(msgTemplate);
checkScheduleType(msgTemplate);
+ checkCollectHotValue(msgTemplate);
checkPeriod();
}
@@ -240,6 +243,15 @@ public class AnalyzeProperties {
}
}
+ private void checkCollectHotValue(String msgTemplate) throws
AnalysisException {
+ if (properties.containsKey(PROPERTY_COLLECT_HOT_VALUE)) {
+ String value = properties.get(PROPERTY_COLLECT_HOT_VALUE);
+ if (!"true".equalsIgnoreCase(value) &&
!"false".equalsIgnoreCase(value)) {
+ throw new AnalysisException(String.format(msgTemplate,
PROPERTY_COLLECT_HOT_VALUE, value));
+ }
+ }
+ }
+
private void checkPeriod() throws AnalysisException {
if (properties.containsKey(PROPERTY_PERIOD_SECONDS)
&& properties.containsKey(PROPERTY_PERIOD_CRON)) {
@@ -287,6 +299,14 @@ public class AnalyzeProperties {
return properties.containsKey(PROPERTY_EXTERNAL_TABLE_USE_SQL);
}
+ public boolean hasCollectHotValue() {
+ return properties.containsKey(PROPERTY_COLLECT_HOT_VALUE);
+ }
+
+ public boolean collectHotValue() {
+ return
Boolean.parseBoolean(properties.get(PROPERTY_COLLECT_HOT_VALUE));
+ }
+
public String toSQL() {
StringBuilder sb = new StringBuilder();
sb.append("PROPERTIES(");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 8804be4406c..43f762fe982 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -8584,6 +8584,8 @@ public class LogicalPlanBuilder extends
DorisParserBaseVisitor<Object> {
properties.put(AnalyzeProperties.PROPERTY_EXTERNAL_TABLE_USE_SQL,
"true");
} else if (ctx.HISTOGRAM() != null) {
properties.put(AnalyzeProperties.PROPERTY_ANALYSIS_TYPE,
AnalysisInfo.AnalysisType.HISTOGRAM.toString());
+ } else if (ctx.HOT() != null) {
+ properties.put(AnalyzeProperties.PROPERTY_COLLECT_HOT_VALUE,
"true");
} else if (ctx.SAMPLE() != null) {
if (ctx.ROWS() != null) {
properties.put(AnalyzeProperties.PROPERTY_SAMPLE_ROWS,
ctx.INTEGER_VALUE().getText());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AnalyzeCommand.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AnalyzeCommand.java
index 7c3d8ebe2c1..34323496fb4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AnalyzeCommand.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AnalyzeCommand.java
@@ -138,10 +138,16 @@ public abstract class AnalyzeCommand extends Command
implements ForwardWithSync
return analyzeProperties.usingSqlForExternalTable();
}
+ /**
+ * Validate analyze command properties.
+ */
public void validate(ConnectContext ctx) throws UserException {
if (analyzeProperties != null) {
analyzeProperties.check();
}
+ if (analyzeProperties.hasCollectHotValue() && getAnalysisMethod() ==
AnalysisInfo.AnalysisMethod.SAMPLE) {
+ throw new AnalysisException("Sample analyze always collects hot
value");
+ }
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
index 58b2c3e3d1f..d2c293b33b1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
@@ -168,6 +168,9 @@ public class AnalysisInfo implements Writable {
@SerializedName("usingSqlForExternalTable")
public final boolean usingSqlForExternalTable;
+ @SerializedName("chv")
+ public final Boolean collectHotValue;
+
@SerializedName("createTime")
public final long createTime = System.currentTimeMillis();
@@ -209,8 +212,9 @@ public class AnalysisInfo implements Writable {
long lastExecTimeInMs, long timeCostInMs, AnalysisState state,
ScheduleType scheduleType,
boolean partitionOnly, boolean samplingPartition,
boolean isAllPartition, long partitionCount, CronExpression
cronExpression, boolean forceFull,
- boolean usingSqlForExternalTable, long tblUpdateTime, long
rowCount, boolean userInject, long updateRows,
- long tableVersion, JobPriority priority, Map<Long, Long>
partitionUpdateRows, boolean enablePartition) {
+ boolean usingSqlForExternalTable, Boolean collectHotValue, long
tblUpdateTime, long rowCount,
+ boolean userInject, long updateRows, long tableVersion,
JobPriority priority,
+ Map<Long, Long> partitionUpdateRows, boolean enablePartition) {
this.jobId = jobId;
this.taskId = taskId;
this.taskIds = taskIds;
@@ -243,6 +247,7 @@ public class AnalysisInfo implements Writable {
}
this.forceFull = forceFull;
this.usingSqlForExternalTable = usingSqlForExternalTable;
+ this.collectHotValue = collectHotValue;
this.tblUpdateTime = tblUpdateTime;
this.rowCount = rowCount;
this.userInject = userInject;
@@ -293,6 +298,7 @@ public class AnalysisInfo implements Writable {
}
sj.add("forceFull: " + forceFull);
sj.add("usingSqlForExternalTable: " + usingSqlForExternalTable);
+ sj.add("collectHotValue: " + collectHotValue);
sj.add("rowCount: " + rowCount);
sj.add("userInject: " + userInject);
sj.add("updateRows: " + updateRows);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
index bbd0d616495..d9fe7c21220 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
@@ -59,6 +59,7 @@ public class AnalysisInfoBuilder {
private CronExpression cronExpression;
private boolean forceFull;
private boolean usingSqlForExternalTable;
+ private Boolean collectHotValue;
private long tblUpdateTime;
private long rowCount;
private boolean userInject = false;
@@ -101,6 +102,7 @@ public class AnalysisInfoBuilder {
cronExpression = info.cronExpression;
forceFull = info.forceFull;
usingSqlForExternalTable = info.usingSqlForExternalTable;
+ collectHotValue = info.collectHotValue;
tblUpdateTime = info.tblUpdateTime;
rowCount = info.rowCount;
userInject = info.userInject;
@@ -256,6 +258,11 @@ public class AnalysisInfoBuilder {
return this;
}
+ public AnalysisInfoBuilder setCollectHotValue(Boolean collectHotValue) {
+ this.collectHotValue = collectHotValue;
+ return this;
+ }
+
public AnalysisInfoBuilder setTblUpdateTime(long tblUpdateTime) {
this.tblUpdateTime = tblUpdateTime;
return this;
@@ -301,8 +308,8 @@ public class AnalysisInfoBuilder {
colName, indexId, jobType, analysisMethod, analysisType,
samplePercent,
sampleRows, maxBucketNum, periodTimeInMs, message,
lastExecTimeInMs, timeCostInMs, state, scheduleType,
partitionOnly, samplingPartition, isAllPartition,
partitionCount,
- cronExpression, forceFull, usingSqlForExternalTable,
tblUpdateTime, rowCount, userInject, updateRows,
- tableVersion, priority, partitionUpdateRows, enablePartition);
+ cronExpression, forceFull, usingSqlForExternalTable,
collectHotValue, tblUpdateTime, rowCount,
+ userInject, updateRows, tableVersion, priority,
partitionUpdateRows, enablePartition);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index f6ac836a42d..1de58345dc1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -381,6 +381,9 @@ public class AnalysisManager implements Writable {
infoBuilder.setCronExpression(cronExpression);
infoBuilder.setForceFull(command.forceFull());
infoBuilder.setUsingSqlForExternalTable(command.usingSqlForExternalTable());
+ AnalyzeProperties analyzeProperties = command.getAnalyzeProperties();
+ infoBuilder.setCollectHotValue((analyzeProperties.hasCollectHotValue()
+ && analyzeProperties.collectHotValue()) || analysisMethod ==
AnalysisMethod.SAMPLE);
if (analysisMethod == AnalysisMethod.SAMPLE) {
infoBuilder.setSamplePercent(samplePercent);
infoBuilder.setSampleRows(sampleRows);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
index b9d8cd78dc6..2e0c5cc4a98 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
@@ -120,6 +120,25 @@ public abstract class BaseAnalysisTask {
+ "GROUP BY `hash_value` ORDER BY `count` DESC LIMIT
${hotValueCollectCount}) t) "
+ "SELECT * FROM cte2 CROSS JOIN cte3";
+ protected static final String FULL_ANALYZE_WITHOUT_HOT_VALUE_TEMPLATE =
+ "SELECT CONCAT(${tblId}, '-', ${idxId}, '-', '${colId}') AS `id`, "
+ + "${catalogId} AS `catalog_id`, "
+ + "${dbId} AS `db_id`, "
+ + "${tblId} AS `tbl_id`, "
+ + "${idxId} AS `idx_id`, "
+ + "'${colId}' AS `col_id`, "
+ + "NULL AS `part_id`, "
+ + "COUNT(1) AS `row_count`, "
+ + "NDV(`${colName}`) AS `ndv`, "
+ + "COUNT(1) - COUNT(`${colName}`) AS `null_count`, "
+ + "SUBSTRING(CAST(MIN(`${colName}`) AS STRING), 1, 1024) AS
`min`, "
+ + "SUBSTRING(CAST(MAX(`${colName}`) AS STRING), 1, 1024) AS
`max`, "
+ + "${dataSizeFunction} AS `data_size`, "
+ + "NOW() AS `update_time`, "
+ + "null as `hot_value` "
+ + "FROM (SELECT `${colName}`${lengthAssert} "
+ + "FROM `${catalogName}`.`${dbName}`.`${tblName}` ${index})
__lc_t";
+
protected static final String LINEAR_ANALYZE_TEMPLATE = "WITH cte1 AS ("
+ "SELECT `${colName}`${lengthAssert} "
+ "FROM `${catalogName}`.`${dbName}`.`${tblName}` ${index}
${sampleHints} ${limit} ${preAggHint}), "
@@ -591,6 +610,10 @@ public abstract class BaseAnalysisTask {
protected abstract void deleteNotExistPartitionStats(AnalysisInfo jobInfo)
throws DdlException;
+ protected boolean shouldCollectHotValue() {
+ return Boolean.TRUE.equals(info.collectHotValue);
+ }
+
protected String getPartitionInfo(String partitionName) {
return "";
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java
index 8c4eccb1165..d37b3740f9d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java
@@ -58,18 +58,22 @@ public class ExternalAnalysisTask extends BaseAnalysisTask {
}
protected void doFull() throws Exception {
- StringBuilder sb = new StringBuilder();
Map<String, String> params = buildSqlParams();
params.put("dataSizeFunction", getDataSizeFunction(col, false));
- params.put("hotValueCollectCount",
String.valueOf(SessionVariable.getHotValueCollectCount()));
- params.put("subStringColName", getStringTypeColName(col));
- params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE `${colName}`
IS NOT NULL)");
if (LOG.isDebugEnabled()) {
LOG.debug("Will do full collection for column {}", col.getName());
}
- sb.append(FULL_ANALYZE_TEMPLATE);
+ String template;
+ if (shouldCollectHotValue()) {
+ params.put("hotValueCollectCount",
String.valueOf(SessionVariable.getHotValueCollectCount()));
+ params.put("subStringColName", getStringTypeColName(col));
+ params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE
`${colName}` IS NOT NULL)");
+ template = FULL_ANALYZE_TEMPLATE;
+ } else {
+ template = FULL_ANALYZE_WITHOUT_HOT_VALUE_TEMPLATE;
+ }
StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
- String sql = stringSubstitutor.replace(sb.toString());
+ String sql = stringSubstitutor.replace(template);
runQuery(sql);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
index 287a5d7b3b9..a7ac364c984 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
@@ -378,11 +378,15 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
doPartitionTable();
} else {
Map<String, String> params = buildSqlParams();
- params.put("hotValueCollectCount",
String.valueOf(SessionVariable.getHotValueCollectCount()));
- params.put("subStringColName", getStringTypeColName(col));
- params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE
`${colName}` IS NOT NULL)");
StringSubstitutor stringSubstitutor = new
StringSubstitutor(params);
- runQuery(stringSubstitutor.replace(FULL_ANALYZE_TEMPLATE));
+ if (shouldCollectHotValue()) {
+ params.put("hotValueCollectCount",
String.valueOf(SessionVariable.getHotValueCollectCount()));
+ params.put("subStringColName", getStringTypeColName(col));
+ params.put("rowCount2", "(SELECT COUNT(1) FROM cte1 WHERE
`${colName}` IS NOT NULL)");
+ runQuery(stringSubstitutor.replace(FULL_ANALYZE_TEMPLATE));
+ } else {
+
runQuery(stringSubstitutor.replace(FULL_ANALYZE_WITHOUT_HOT_VALUE_TEMPLATE));
+ }
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
index 2ce0f05229b..0dbe97f1ba1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
@@ -256,6 +256,7 @@ public class StatisticsAutoCollector extends MasterDaemon {
.setPartitionNames(Collections.emptySet())
.setSampleRows(analysisMethod.equals(AnalysisMethod.SAMPLE)
? StatisticsUtil.getHugeTableSampleRows() : -1)
+
.setCollectHotValue(analysisMethod.equals(AnalysisMethod.SAMPLE))
.setScheduleType(ScheduleType.AUTOMATIC)
.setState(AnalysisState.PENDING)
.setTaskIds(new ArrayList<>())
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AnalyzeTableCommandTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AnalyzeTableCommandTest.java
index 5d8f14938a0..a5f16003984 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AnalyzeTableCommandTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/AnalyzeTableCommandTest.java
@@ -26,7 +26,10 @@ import org.apache.doris.info.PartitionNamesInfo;
import org.apache.doris.info.TableNameInfo;
import org.apache.doris.mysql.privilege.AccessControllerManager;
import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.parser.NereidsParser;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.statistics.AnalysisInfo;
import com.google.common.collect.ImmutableList;
import mockit.Expectations;
@@ -34,6 +37,9 @@ import mockit.Mocked;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import java.util.HashMap;
+import java.util.Map;
+
public class AnalyzeTableCommandTest {
private static final String internalCtl =
InternalCatalog.INTERNAL_CATALOG_NAME;
@Mocked
@@ -82,5 +88,63 @@ public class AnalyzeTableCommandTest {
() ->
analyzeTableCommand.checkAnalyzePrivilege(tableNameInfo2),
"ANALYZE command denied to user 'null'@'null' for table
'test_db: test_tbl2'");
}
-}
+ @Test
+ void testCollectHotValueDefaultAndOverride() {
+ TableNameInfo tableNameInfo = new TableNameInfo(internalCtl,
+ CatalogMocker.TEST_DB_NAME, CatalogMocker.TEST_TBL_NAME);
+
+ AnalyzeTableCommand fullAnalyzeCommand = new
AnalyzeTableCommand(tableNameInfo,
+ null, null, new AnalyzeProperties(defaultAnalyzeProperties()));
+
Assertions.assertFalse(fullAnalyzeCommand.getAnalyzeProperties().hasCollectHotValue());
+ Assertions.assertEquals(AnalysisInfo.AnalysisMethod.FULL,
fullAnalyzeCommand.getAnalysisMethod());
+
+ Map<String, String> sampleProperties = defaultAnalyzeProperties();
+ sampleProperties.put(AnalyzeProperties.PROPERTY_SAMPLE_ROWS, "100");
+ AnalyzeTableCommand sampleAnalyzeCommand = new
AnalyzeTableCommand(tableNameInfo,
+ null, null, new AnalyzeProperties(sampleProperties));
+
Assertions.assertFalse(sampleAnalyzeCommand.getAnalyzeProperties().hasCollectHotValue());
+ Assertions.assertEquals(AnalysisInfo.AnalysisMethod.SAMPLE,
sampleAnalyzeCommand.getAnalysisMethod());
+
+ Map<String, String> forcedFullHotValueProperties =
defaultAnalyzeProperties();
+
forcedFullHotValueProperties.put(AnalyzeProperties.PROPERTY_COLLECT_HOT_VALUE,
"true");
+ AnalyzeTableCommand forcedFullHotValueCommand = new
AnalyzeTableCommand(tableNameInfo,
+ null, null, new
AnalyzeProperties(forcedFullHotValueProperties));
+
Assertions.assertTrue(forcedFullHotValueCommand.getAnalyzeProperties().collectHotValue());
+ }
+
+ @Test
+ void testCollectHotValuePropertyValidation() {
+ Map<String, String> properties = defaultAnalyzeProperties();
+ properties.put(AnalyzeProperties.PROPERTY_COLLECT_HOT_VALUE,
"invalid");
+ AnalyzeProperties analyzeProperties = new
AnalyzeProperties(properties);
+ Assertions.assertThrows(AnalysisException.class,
analyzeProperties::check);
+ }
+
+ @Test
+ void testParseWithHotValue() {
+ NereidsParser parser = new NereidsParser();
+ LogicalPlan plan = parser.parseSingle("ANALYZE TABLE test_db.test_tbl
WITH HOT VALUE");
+ Assertions.assertTrue(plan instanceof AnalyzeTableCommand);
+ AnalyzeTableCommand command = (AnalyzeTableCommand) plan;
+
Assertions.assertTrue(command.getAnalyzeProperties().hasCollectHotValue());
+
Assertions.assertTrue(command.getAnalyzeProperties().collectHotValue());
+ }
+
+ @Test
+ void testSampleAnalyzeWithHotValueRejected() {
+ NereidsParser parser = new NereidsParser();
+ AnalyzeTableCommand command = (AnalyzeTableCommand) parser.parseSingle(
+ "ANALYZE TABLE test_db.test_tbl WITH SAMPLE ROWS 100 WITH HOT
VALUE");
+ AnalysisException exception =
Assertions.assertThrows(AnalysisException.class,
+ () -> command.validate(ctx));
+ Assertions.assertEquals("Sample analyze always collects hot value",
exception.getDetailMessage());
+ }
+
+ private Map<String, String> defaultAnalyzeProperties() {
+ Map<String, String> properties = new HashMap<>();
+ properties.put(AnalyzeProperties.PROPERTY_SYNC, "false");
+ properties.put(AnalyzeProperties.PROPERTY_ANALYSIS_TYPE,
"FUNDAMENTALS");
+ return properties;
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
index 4a290a2436b..f2c1f313044 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
@@ -17,6 +17,7 @@
package org.apache.doris.statistics;
+import org.apache.doris.analysis.AnalyzeProperties;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.DatabaseIf;
@@ -27,6 +28,7 @@ import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
+import org.apache.doris.common.Pair;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.datasource.paimon.PaimonExternalCatalog;
@@ -34,6 +36,7 @@ import
org.apache.doris.datasource.paimon.PaimonExternalDatabase;
import org.apache.doris.datasource.paimon.PaimonExternalTable;
import org.apache.doris.datasource.paimon.PaimonUtils;
import org.apache.doris.info.PartitionNamesInfo;
+import org.apache.doris.info.TableNameInfo;
import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
@@ -77,6 +80,67 @@ import java.util.concurrent.atomic.AtomicInteger;
// CHECKSTYLE OFF
public class AnalysisManagerTest {
+ @Test
+ public void testBuildAnalysisJobInfoCollectHotValueDefault() {
+ AnalysisManager manager = new AnalysisManager();
+ Env env = Mockito.mock(Env.class);
+ try (MockedStatic<Env> envMockedStatic =
Mockito.mockStatic(Env.class)) {
+ envMockedStatic.when(Env::getCurrentEnv).thenReturn(env);
+ Mockito.when(env.getNextId()).thenReturn(1L, 2L, 3L, 4L);
+
+ AnalysisInfo fullOnce = manager.buildAnalysisJobInfo(
+ mockAnalyzeCommand(AnalysisMethod.FULL, ScheduleType.ONCE,
false, false));
+ Assertions.assertFalse(fullOnce.collectHotValue);
+
+ AnalysisInfo samplePeriod = manager.buildAnalysisJobInfo(
+ mockAnalyzeCommand(AnalysisMethod.SAMPLE,
ScheduleType.PERIOD, false, true));
+ Assertions.assertTrue(samplePeriod.collectHotValue);
+
+ AnalysisInfo automatic = manager.buildAnalysisJobInfo(
+ mockAnalyzeCommand(AnalysisMethod.FULL,
ScheduleType.AUTOMATIC, false, true));
+ Assertions.assertFalse(automatic.collectHotValue);
+
+ AnalysisInfo automaticSample = manager.buildAnalysisJobInfo(
+ mockAnalyzeCommand(AnalysisMethod.SAMPLE,
ScheduleType.AUTOMATIC, false, true));
+ Assertions.assertTrue(automaticSample.collectHotValue);
+
+ AnalysisInfo explicitAutomatic = manager.buildAnalysisJobInfo(
+ mockAnalyzeCommand(AnalysisMethod.FULL,
ScheduleType.AUTOMATIC, true, false));
+ Assertions.assertFalse(explicitAutomatic.collectHotValue);
+ }
+ }
+
+ @Test
+ public void testBuildAnalysisJobInfoAutoSampleCommandCollectsHotValue() {
+ AnalysisManager manager = new AnalysisManager();
+ Env env = Mockito.mock(Env.class);
+ try (MockedStatic<Env> envMockedStatic =
Mockito.mockStatic(Env.class)) {
+ envMockedStatic.when(Env::getCurrentEnv).thenReturn(env);
+ Mockito.when(env.getNextId()).thenReturn(1L);
+
+ Map<String, String> properties = new HashMap<>();
+ properties.put(AnalyzeProperties.PROPERTY_SYNC, "false");
+ properties.put(AnalyzeProperties.PROPERTY_ANALYSIS_TYPE,
AnalysisType.FUNDAMENTALS.toString());
+ properties.put(AnalyzeProperties.PROPERTY_AUTOMATIC, "true");
+ properties.put(AnalyzeProperties.PROPERTY_SAMPLE_ROWS, "100");
+ AnalyzeTableCommand command = Mockito.spy(new AnalyzeTableCommand(
+ new TableNameInfo(InternalCatalog.INTERNAL_CATALOG_NAME,
"testDb", "testTbl"),
+ null, ImmutableList.of("testCol"), new
AnalyzeProperties(properties)));
+ TableIf table = Mockito.mock(TableIf.class);
+ Mockito.when(table.getId()).thenReturn(30001L);
+ Mockito.when(table.getColumnIndexPairs(Mockito.any()))
+ .thenReturn(Collections.singleton(Pair.of("testTbl",
"testCol")));
+ Mockito.doReturn(table).when(command).getTable();
+ Mockito.doReturn(10001L).when(command).getCatalogId();
+ Mockito.doReturn(20001L).when(command).getDbId();
+
+ AnalysisInfo analysisInfo = manager.buildAnalysisJobInfo(command);
+ Assertions.assertEquals(ScheduleType.AUTOMATIC,
analysisInfo.scheduleType);
+ Assertions.assertEquals(AnalysisMethod.SAMPLE,
analysisInfo.analysisMethod);
+ Assertions.assertTrue(analysisInfo.collectHotValue);
+ }
+ }
+
@Test
public void testUpdateTaskStatus(@Mocked BaseAnalysisTask task1,
@Mocked BaseAnalysisTask task2) {
@@ -429,7 +493,7 @@ public class AnalysisManagerTest {
PaimonExternalTable table = Mockito.spy(new
TestPaimonExternalTable(catalog, database));
Mockito.doReturn(-1L).when(table).getRowCount();
Mockito.doReturn(Collections.emptySet()).when(table).getColumnIndexPairs(Mockito.any());
- AnalyzeTableCommand command = mockAnalyzeCommand(AnalysisMethod.FULL,
ScheduleType.ONCE);
+ AnalyzeTableCommand command = mockAnalyzeCommand(AnalysisMethod.FULL,
ScheduleType.ONCE, false, false);
Mockito.when(command.getTable()).thenReturn(table);
AnalysisManager manager = new AnalysisManager();
Env env = Mockito.mock(Env.class);
@@ -475,11 +539,26 @@ public class AnalysisManagerTest {
Assertions.assertTrue(count.get() <= 20);
}
- private AnalyzeTableCommand mockAnalyzeCommand(AnalysisMethod
analysisMethod, ScheduleType scheduleType) {
+ private static class TestPaimonExternalTable extends PaimonExternalTable {
+ private TestPaimonExternalTable(PaimonExternalCatalog catalog,
PaimonExternalDatabase database) {
+ super(30001L, "table", "table", catalog, database);
+ }
+
+ @Override
+ protected synchronized void makeSureInitialized() {
+ }
+ }
+
+ private AnalyzeTableCommand mockAnalyzeCommand(AnalysisMethod
analysisMethod, ScheduleType scheduleType,
+ boolean hasCollectHotValue, boolean collectHotValue) {
AnalyzeTableCommand command = Mockito.mock(AnalyzeTableCommand.class);
TableIf table = Mockito.mock(TableIf.class);
+ Map<String, String> properties = new HashMap<>();
+ if (hasCollectHotValue) {
+ properties.put(AnalyzeProperties.PROPERTY_COLLECT_HOT_VALUE,
String.valueOf(collectHotValue));
+ }
+ AnalyzeProperties analyzeProperties = new
AnalyzeProperties(properties);
Mockito.when(table.getId()).thenReturn(30001L);
-
Mockito.when(table.getColumnIndexPairs(Mockito.any())).thenReturn(Collections.emptySet());
Mockito.when(command.getTable()).thenReturn(table);
Mockito.when(command.getColumnNames()).thenReturn(Collections.emptySet());
Mockito.when(command.isPartitionOnly()).thenReturn(false);
@@ -497,16 +576,7 @@ public class AnalysisManagerTest {
Mockito.when(command.getPartitionNames()).thenReturn(Collections.emptySet());
Mockito.when(command.forceFull()).thenReturn(false);
Mockito.when(command.usingSqlForExternalTable()).thenReturn(false);
+
Mockito.when(command.getAnalyzeProperties()).thenReturn(analyzeProperties);
return command;
}
-
- private static class TestPaimonExternalTable extends PaimonExternalTable {
- private TestPaimonExternalTable(PaimonExternalCatalog catalog,
PaimonExternalDatabase database) {
- super(30001L, "table", "table", catalog, database);
- }
-
- @Override
- protected synchronized void makeSureInitialized() {
- }
- }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/HMSAnalysisTaskTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/HMSAnalysisTaskTest.java
index 078ca196df2..33721e5f8e4 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/HMSAnalysisTaskTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/HMSAnalysisTaskTest.java
@@ -199,6 +199,65 @@ public class HMSAnalysisTaskTest {
}
};
+ new MockUp<HMSAnalysisTask>() {
+ @Mock
+ public void runQuery(String sql) {
+ Assertions.assertEquals("SELECT CONCAT(30001, '-', -1, '-',
'hour') AS `id`, "
+ + "10001 AS `catalog_id`, 20001 AS `db_id`, 30001 AS
`tbl_id`, "
+ + "-1 AS `idx_id`, 'hour' AS `col_id`, NULL AS
`part_id`, "
+ + "COUNT(1) AS `row_count`, NDV(`hour`) AS `ndv`, "
+ + "COUNT(1) - COUNT(`hour`) AS `null_count`, "
+ + "SUBSTRING(CAST(MIN(`hour`) AS STRING), 1, 1024) AS
`min`, "
+ + "SUBSTRING(CAST(MAX(`hour`) AS STRING), 1, 1024) AS
`max`, "
+ + "COUNT(1) * 4 AS `data_size`, NOW() AS
`update_time`, null as `hot_value` "
+ + "FROM (SELECT `hour` FROM `hms`.`default`.`test` )
__lc_t", sql);
+ }
+ };
+
+ HMSAnalysisTask task = new HMSAnalysisTask();
+ task.col = new Column("hour", PrimitiveType.INT);
+ task.tbl = tableIf;
+ task.catalog = catalogIf;
+ task.db = databaseIf;
+ task.setTable(tableIf);
+
+ AnalysisInfoBuilder analysisInfoBuilder = new AnalysisInfoBuilder();
+ analysisInfoBuilder.setColName("hour");
+ analysisInfoBuilder.setJobType(AnalysisInfo.JobType.MANUAL);
+ analysisInfoBuilder.setUsingSqlForExternalTable(true);
+ task.info = analysisInfoBuilder.build();
+
+ task.doExecute();
+ }
+
+ @Test
+ public void testOrdinaryStatsWithHotValue(@Mocked CatalogIf catalogIf,
@Mocked DatabaseIf databaseIf, @Mocked HMSExternalTable tableIf)
+ throws Exception {
+
+ new Expectations() {
+ {
+ tableIf.getId();
+ result = 30001;
+ tableIf.getName();
+ result = "test";
+ catalogIf.getId();
+ result = 10001;
+ catalogIf.getName();
+ result = "hms";
+ databaseIf.getId();
+ result = 20001;
+ databaseIf.getFullName();
+ result = "default";
+ }
+ };
+
+ new MockUp<HMSExternalTable>() {
+ @Mock
+ public Set<String> getPartitionNames() {
+ return ImmutableSet.of("date=20230101/hour=12");
+ }
+ };
+
new MockUp<SessionVariable>() {
@Mock
public int getHotValueCollectCount() {
@@ -242,6 +301,7 @@ public class HMSAnalysisTaskTest {
analysisInfoBuilder.setColName("hour");
analysisInfoBuilder.setJobType(AnalysisInfo.JobType.MANUAL);
analysisInfoBuilder.setUsingSqlForExternalTable(true);
+ analysisInfoBuilder.setCollectHotValue(true);
task.info = analysisInfoBuilder.build();
task.doExecute();
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java
index af8c9c18f66..dd0113bf930 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/OlapAnalysisTaskTest.java
@@ -41,6 +41,7 @@ import org.apache.doris.common.FeConstants;
import org.apache.doris.common.Pair;
import org.apache.doris.common.util.DebugPointUtil;
import org.apache.doris.datasource.CatalogIf;
+import org.apache.doris.persist.gson.GsonUtils;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod;
import org.apache.doris.statistics.AnalysisInfo.JobType;
@@ -58,6 +59,7 @@ import mockit.Mocked;
import org.apache.commons.text.StringSubstitutor;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentMatchers;
import org.mockito.MockedStatic;
import org.mockito.Mockito;
@@ -91,6 +93,30 @@ public class OlapAnalysisTaskTest {
Assertions.assertFalse(tableSample.isPercent());
}
+ @Test
+ public void testShouldCollectHotValue() {
+ OlapAnalysisTask olapAnalysisTask = new OlapAnalysisTask();
+ olapAnalysisTask.info = new AnalysisInfoBuilder().build();
+ Assertions.assertFalse(olapAnalysisTask.shouldCollectHotValue());
+
+ olapAnalysisTask.info = new
AnalysisInfoBuilder().setCollectHotValue(true).build();
+ Assertions.assertTrue(olapAnalysisTask.shouldCollectHotValue());
+
+ olapAnalysisTask.info = new
AnalysisInfoBuilder().setCollectHotValue(false).build();
+ Assertions.assertFalse(olapAnalysisTask.shouldCollectHotValue());
+ }
+
+ @Test
+ public void testCollectHotValueUseShortSerializedName() {
+ AnalysisInfo info = new
AnalysisInfoBuilder().setCollectHotValue(false).build();
+ String json = GsonUtils.GSON.toJson(info);
+ Assertions.assertTrue(json.contains("\"chv\":false"));
+ Assertions.assertFalse(json.contains("collectHotValue"));
+
+ AnalysisInfo deserialized = GsonUtils.GSON.fromJson(json,
AnalysisInfo.class);
+ Assertions.assertFalse(deserialized.collectHotValue);
+ }
+
// test auto small table
@Test
public void testSample3(@Mocked OlapTable tbl) {
@@ -1040,6 +1066,7 @@ public class OlapAnalysisTaskTest {
AnalysisInfoBuilder builder = new AnalysisInfoBuilder();
builder.setJobType(AnalysisInfo.JobType.MANUAL);
builder.setColName("testCol");
+ builder.setCollectHotValue(true);
task.info = builder.build();
task.catalog = catalogIf;
task.db = databaseIf;
@@ -1047,6 +1074,95 @@ public class OlapAnalysisTaskTest {
}
}
+ @Test
+ public void testDoFullWithoutHotValue() throws Exception {
+ CatalogIf catalogIf = Mockito.mock(CatalogIf.class);
+ DatabaseIf databaseIf = Mockito.mock(DatabaseIf.class);
+ OlapTable tableIf = Mockito.mock(OlapTable.class);
+ Mockito.when(tableIf.getId()).thenReturn(30001L);
+ Mockito.when(tableIf.getName()).thenReturn("testTbl");
+ Mockito.when(catalogIf.getId()).thenReturn(10001L);
+ Mockito.when(catalogIf.getName()).thenReturn("catalogName");
+ Mockito.when(databaseIf.getId()).thenReturn(20001L);
+ Mockito.when(databaseIf.getFullName()).thenReturn("testDb");
+
+ try (MockedStatic<StatisticsUtil> mockedStatisticsUtil =
Mockito.mockStatic(
+ StatisticsUtil.class, Mockito.CALLS_REAL_METHODS)) {
+
mockedStatisticsUtil.when(StatisticsUtil::enablePartitionAnalyze).thenReturn(false);
+
+ OlapAnalysisTask task = Mockito.spy(new OlapAnalysisTask());
+ Mockito.doAnswer(invocation -> {
+ String sql = invocation.getArgument(0);
+ Assertions.assertTrue(sql.startsWith("SELECT CONCAT(30001,
'-', -1, '-', 'testCol') AS `id`"), sql);
+ Assertions.assertTrue(sql.contains("null as `hot_value`"),
sql);
+ Assertions.assertTrue(sql.contains(
+ "FROM (SELECT `testCol` FROM
`catalogName`.`testDb`.`testTbl` ) __lc_t"), sql);
+ Assertions.assertFalse(sql.contains("CROSS JOIN cte3"), sql);
+ Assertions.assertFalse(sql.contains("GROUP BY `hash_value`"),
sql);
+ return null;
+ }).when(task).runQuery(Mockito.anyString());
+
+ task.col = new Column("testCol",
Type.fromPrimitiveType(PrimitiveType.INT),
+ true, null, null, null);
+ task.tbl = tableIf;
+ AnalysisInfoBuilder builder = new AnalysisInfoBuilder();
+ builder.setJobType(AnalysisInfo.JobType.MANUAL);
+ builder.setColName("testCol");
+ builder.setCollectHotValue(false);
+ task.info = builder.build();
+ task.catalog = catalogIf;
+ task.db = databaseIf;
+ task.doFull();
+ }
+ }
+
+ @Test
+ public void testDoSampleAlwaysCollectsHotValue() throws Exception {
+ CatalogIf catalogIf = Mockito.mock(CatalogIf.class);
+ DatabaseIf databaseIf = Mockito.mock(DatabaseIf.class);
+ OlapTable tableIf = Mockito.mock(OlapTable.class);
+ Mockito.when(tableIf.getId()).thenReturn(30001L);
+ Mockito.when(tableIf.getName()).thenReturn("testTbl");
+ Mockito.when(tableIf.getRowCount()).thenReturn(1000L);
+ Mockito.when(catalogIf.getId()).thenReturn(10001L);
+ Mockito.when(catalogIf.getName()).thenReturn("catalogName");
+ Mockito.when(databaseIf.getId()).thenReturn(20001L);
+ Mockito.when(databaseIf.getFullName()).thenReturn("testDb");
+
+ OlapAnalysisTask task = Mockito.spy(new OlapAnalysisTask());
+ Mockito.doReturn(new ResultRow(Lists.newArrayList("1",
"2"))).when(task).collectMinMax();
+ Mockito.doNothing().when(task).getSampleParams(ArgumentMatchers.any(),
ArgumentMatchers.anyLong(),
+ ArgumentMatchers.any());
+ Mockito.doAnswer(invocation -> {
+ String sql = invocation.getArgument(0);
+ Assertions.assertTrue(sql.contains("as `hot_value`"), sql);
+ Assertions.assertTrue(sql.contains("cte3"), sql);
+ Assertions.assertTrue(sql.contains("CROSS JOIN cte3"), sql);
+ Assertions.assertTrue(sql.contains("LIMIT 10"), sql);
+ Assertions.assertFalse(sql.contains("null as `hot_value`"), sql);
+ return null;
+ }).when(task).runQuery(Mockito.anyString());
+
+ task.col = new Column("testCol",
Type.fromPrimitiveType(PrimitiveType.INT),
+ true, null, null, null);
+ task.tbl = tableIf;
+ AnalysisInfoBuilder builder = new AnalysisInfoBuilder();
+ builder.setJobType(AnalysisInfo.JobType.MANUAL);
+ builder.setColName("testCol");
+ builder.setCollectHotValue(false);
+ task.info = builder.build();
+ task.catalog = catalogIf;
+ task.db = databaseIf;
+ task.tableSample = new TableSample(false, 100L);
+
+ Mockito.doReturn(new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.LINEAR, null))
+ .when(task).getSampleCollectInfo(ArgumentMatchers.anyLong());
+ task.doSample();
+ Mockito.doReturn(new
OlapAnalysisTask.SampleCollectInfo(AnalyzeSampleAlgorithm.DUJ1, null))
+ .when(task).getSampleCollectInfo(ArgumentMatchers.anyLong());
+ task.doSample();
+ }
+
@Test
public void testDoFullHotValueStringColumn() throws Exception {
CatalogIf catalogIf = Mockito.mock(CatalogIf.class);
@@ -1080,6 +1196,7 @@ public class OlapAnalysisTaskTest {
AnalysisInfoBuilder builder = new AnalysisInfoBuilder();
builder.setJobType(AnalysisInfo.JobType.MANUAL);
builder.setColName("strCol");
+ builder.setCollectHotValue(true);
task.info = builder.build();
task.catalog = catalogIf;
task.db = databaseIf;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java
index 02356bb094e..2cf25135784 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/util/StatisticsUtilTest.java
@@ -643,6 +643,10 @@ class StatisticsUtilTest {
@Test
void testGetHotValues() {
+ Assertions.assertNull(StatisticsUtil.getHotValues(null, Type.INT));
+ Assertions.assertNull(StatisticsUtil.getHotValues("null", Type.INT));
+ Assertions.assertTrue(StatisticsUtil.getHotValues("",
Type.INT).isEmpty());
+
String value1 = "1234 :0.35 ;222 :0.34";
Map<Literal, Float> hotValues = StatisticsUtil.getHotValues(value1,
Type.INT);
Map<Literal, Float> hotValuesAfterFilter =
StatisticsUtil.getHotValuesWithOriginalThreshold(hotValues, 100);
diff --git
a/regression-test/data/nereids_rules_p0/distinct_split/disitinct_split.out
b/regression-test/data/nereids_rules_p0/distinct_split/disitinct_split.out
index 9a18cab3879..b239f4d54f8 100644
--- a/regression-test/data/nereids_rules_p0/distinct_split/disitinct_split.out
+++ b/regression-test/data/nereids_rules_p0/distinct_split/disitinct_split.out
@@ -369,13 +369,11 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
------hashAgg[DISTINCT_GLOBAL]
--------hashAgg[DISTINCT_LOCAL]
----------hashAgg[GLOBAL]
-------------hashAgg[LOCAL]
---------------PhysicalCteConsumer ( cteId=CTEId#0 )
+------------PhysicalCteConsumer ( cteId=CTEId#0 )
------hashAgg[DISTINCT_GLOBAL]
--------hashAgg[DISTINCT_LOCAL]
----------hashAgg[GLOBAL]
-------------hashAgg[LOCAL]
---------------PhysicalCteConsumer ( cteId=CTEId#0 )
+------------PhysicalCteConsumer ( cteId=CTEId#0 )
-- !multi_count_mulitcols_with_gby --
PhysicalCteAnchor ( cteId=CTEId#0 )
@@ -386,13 +384,11 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
------hashAgg[GLOBAL]
--------hashAgg[LOCAL]
----------hashAgg[GLOBAL]
-------------hashAgg[LOCAL]
---------------PhysicalCteConsumer ( cteId=CTEId#0 )
+------------PhysicalCteConsumer ( cteId=CTEId#0 )
------hashAgg[GLOBAL]
--------hashAgg[LOCAL]
----------hashAgg[GLOBAL]
-------------hashAgg[LOCAL]
---------------PhysicalCteConsumer ( cteId=CTEId#0 )
+------------PhysicalCteConsumer ( cteId=CTEId#0 )
-- !three_count_mulitcols_without_gby --
PhysicalCteAnchor ( cteId=CTEId#0 )
@@ -404,18 +400,15 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
--------hashAgg[DISTINCT_GLOBAL]
----------hashAgg[DISTINCT_LOCAL]
------------hashAgg[GLOBAL]
---------------hashAgg[LOCAL]
-----------------PhysicalCteConsumer ( cteId=CTEId#0 )
+--------------PhysicalCteConsumer ( cteId=CTEId#0 )
--------hashAgg[DISTINCT_GLOBAL]
----------hashAgg[DISTINCT_LOCAL]
------------hashAgg[GLOBAL]
---------------hashAgg[LOCAL]
-----------------PhysicalCteConsumer ( cteId=CTEId#0 )
+--------------PhysicalCteConsumer ( cteId=CTEId#0 )
------hashAgg[DISTINCT_GLOBAL]
--------hashAgg[DISTINCT_LOCAL]
----------hashAgg[GLOBAL]
-------------hashAgg[LOCAL]
---------------PhysicalCteConsumer ( cteId=CTEId#0 )
+------------PhysicalCteConsumer ( cteId=CTEId#0 )
-- !four_count_mulitcols_with_gby --
PhysicalCteAnchor ( cteId=CTEId#0 )
@@ -428,23 +421,19 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
----------hashAgg[GLOBAL]
------------hashAgg[LOCAL]
--------------hashAgg[GLOBAL]
-----------------hashAgg[LOCAL]
-------------------PhysicalCteConsumer ( cteId=CTEId#0 )
+----------------PhysicalCteConsumer ( cteId=CTEId#0 )
----------hashAgg[GLOBAL]
------------hashAgg[LOCAL]
--------------hashAgg[GLOBAL]
-----------------hashAgg[LOCAL]
-------------------PhysicalCteConsumer ( cteId=CTEId#0 )
+----------------PhysicalCteConsumer ( cteId=CTEId#0 )
--------hashAgg[GLOBAL]
----------hashAgg[LOCAL]
------------hashAgg[GLOBAL]
---------------hashAgg[LOCAL]
-----------------PhysicalCteConsumer ( cteId=CTEId#0 )
+--------------PhysicalCteConsumer ( cteId=CTEId#0 )
------hashAgg[GLOBAL]
--------hashAgg[LOCAL]
----------hashAgg[GLOBAL]
-------------hashAgg[LOCAL]
---------------PhysicalCteConsumer ( cteId=CTEId#0 )
+------------PhysicalCteConsumer ( cteId=CTEId#0 )
-- !has_other_func --
PhysicalCteAnchor ( cteId=CTEId#0 )
@@ -478,13 +467,11 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
----------hashAgg[GLOBAL]
------------hashAgg[LOCAL]
--------------hashAgg[GLOBAL]
-----------------hashAgg[LOCAL]
-------------------PhysicalCteConsumer ( cteId=CTEId#0 )
+----------------PhysicalCteConsumer ( cteId=CTEId#0 )
----------hashAgg[GLOBAL]
------------hashAgg[LOCAL]
--------------hashAgg[GLOBAL]
-----------------hashAgg[LOCAL]
-------------------PhysicalCteConsumer ( cteId=CTEId#0 )
+----------------PhysicalCteConsumer ( cteId=CTEId#0 )
-- !multi_count_with_gby --
PhysicalCteAnchor ( cteId=CTEId#0 )
@@ -495,13 +482,11 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
------hashAgg[GLOBAL]
--------hashAgg[LOCAL]
----------hashAgg[GLOBAL]
-------------hashAgg[LOCAL]
---------------PhysicalCteConsumer ( cteId=CTEId#0 )
+------------PhysicalCteConsumer ( cteId=CTEId#0 )
------hashAgg[GLOBAL]
--------hashAgg[LOCAL]
----------hashAgg[GLOBAL]
-------------hashAgg[LOCAL]
---------------PhysicalCteConsumer ( cteId=CTEId#0 )
+------------PhysicalCteConsumer ( cteId=CTEId#0 )
-- !multi_sum_with_gby --
PhysicalCteAnchor ( cteId=CTEId#0 )
@@ -512,13 +497,11 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
------hashAgg[GLOBAL]
--------hashAgg[LOCAL]
----------hashAgg[GLOBAL]
-------------hashAgg[LOCAL]
---------------PhysicalCteConsumer ( cteId=CTEId#0 )
+------------PhysicalCteConsumer ( cteId=CTEId#0 )
------hashAgg[GLOBAL]
--------hashAgg[LOCAL]
----------hashAgg[GLOBAL]
-------------hashAgg[LOCAL]
---------------PhysicalCteConsumer ( cteId=CTEId#0 )
+------------PhysicalCteConsumer ( cteId=CTEId#0 )
-- !sum_count_with_gby --
PhysicalCteAnchor ( cteId=CTEId#0 )
@@ -529,12 +512,10 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
------hashAgg[GLOBAL]
--------hashAgg[LOCAL]
----------hashAgg[GLOBAL]
-------------hashAgg[LOCAL]
---------------PhysicalCteConsumer ( cteId=CTEId#0 )
+------------PhysicalCteConsumer ( cteId=CTEId#0 )
------hashAgg[GLOBAL]
--------hashAgg[GLOBAL]
-----------hashAgg[LOCAL]
-------------PhysicalCteConsumer ( cteId=CTEId#0 )
+----------PhysicalCteConsumer ( cteId=CTEId#0 )
-- !has_grouping --
PhysicalResultSink
diff --git
a/regression-test/suites/statistics/test_full_analyze_hot_value.groovy
b/regression-test/suites/statistics/test_full_analyze_hot_value.groovy
index 92211e69d05..f2e5031eb19 100644
--- a/regression-test/suites/statistics/test_full_analyze_hot_value.groovy
+++ b/regression-test/suites/statistics/test_full_analyze_hot_value.groovy
@@ -22,7 +22,7 @@ suite("test_full_analyze_hot_value") {
sql """use test_full_analyze_hot_value"""
sql """set global enable_auto_analyze=false"""
- // Test 1: Full analyze collects hot_value for varchar column with skewed
distribution
+ // Test 1: Full analyze does not collect hot_value by default.
sql """drop table if exists full_hot_skew"""
sql """CREATE TABLE full_hot_skew (
key1 int NULL,
@@ -43,7 +43,20 @@ suite("test_full_analyze_hot_value") {
logger.info("Test1 result: " + result)
assertEquals(1, result.size())
assertEquals("100.0", result[0][2])
- // Full analyze should now collect hot_value
+ assertEquals("null", String.valueOf(result[0][17]))
+
+ result = sql """show column cached stats full_hot_skew(value1)"""
+ logger.info("Test1 cached result: " + result)
+ assertEquals(1, result.size())
+ assertEquals("null", String.valueOf(result[0][17]))
+
+ // Test 2: Full analyze collects hot_value when explicitly enabled.
+ sql """drop stats full_hot_skew"""
+ sql """analyze table full_hot_skew with sync with hot value"""
+ result = sql """show column stats full_hot_skew(value1)"""
+ logger.info("Test2 result: " + result)
+ assertEquals(1, result.size())
+ assertEquals("100.0", result[0][2])
assertTrue(result[0][17].contains(":"), "Full analyze should collect
hot_value, but got " + result[0][17])
String[] hotValues = result[0][17].split(";")
assertEquals(2, hotValues.length)
@@ -52,22 +65,22 @@ suite("test_full_analyze_hot_value") {
// Verify cached stats also have hot_value
result = sql """show column cached stats full_hot_skew(value1)"""
- logger.info("Test1 cached result: " + result)
+ logger.info("Test2 cached result: " + result)
assertEquals(1, result.size())
hotValues = result[0][17].split(";")
assertEquals(2, hotValues.length)
assertTrue(hotValues[0].trim() == "'1':0.5" || hotValues[0].trim() ==
"'0':0.5")
assertTrue(hotValues[1].trim() == "'1':0.5" || hotValues[1].trim() ==
"'0':0.5")
- // Test 2: Full analyze collects hot_value for int column
+ // Test 3: Explicit full hot value collection works for int column.
result = sql """show column stats full_hot_skew(key1)"""
- logger.info("Test2 result: " + result)
+ logger.info("Test3 result: " + result)
assertEquals(1, result.size())
assertEquals("100.0", result[0][2])
// key1 has 100 unique values, top 10 will each have proportion 0.01 ->
ROUND to 0.01
assertTrue(result[0][17].contains(":"), "Full analyze should collect
hot_value for int column")
- // Test 3: Full analyze with special characters in values
+ // Test 4: Explicit full hot value collection works with special
characters in values.
sql """drop table if exists full_hot_special"""
sql """CREATE TABLE full_hot_special (
key1 int NULL,
@@ -82,18 +95,18 @@ suite("test_full_analyze_hot_value") {
"""
sql """insert into full_hot_special select number, " : ;a" from
numbers("number"="100")"""
- sql """analyze table full_hot_special with sync"""
+ sql """analyze table full_hot_special with sync with hot value"""
result = sql """show column stats full_hot_special(value1)"""
- logger.info("Test3 result: " + result)
+ logger.info("Test4 result: " + result)
assertEquals(1, result.size())
// All 100 rows have the same value " : ;a", so it should appear with
ratio 1.0
assertEquals("' : ;a':1.0", result[0][17])
- // Test 4: Full analyze then sample analyze, hot_value should be updated
+ // Test 5: Sample analyze collects hot_value by default.
sql """drop stats full_hot_skew"""
sql """analyze table full_hot_skew with sample rows 400 with sync"""
result = sql """show column stats full_hot_skew(value1)"""
- logger.info("Test4 result: " + result)
+ logger.info("Test5 result: " + result)
assertEquals(1, result.size())
assertTrue(result[0][17].contains(":"), "Sample analyze should also
collect hot_value")
hotValues = result[0][17].split(";")
@@ -101,11 +114,17 @@ suite("test_full_analyze_hot_value") {
assertTrue(hotValues[0].trim() == "'1':0.5" || hotValues[0].trim() ==
"'0':0.5")
assertTrue(hotValues[1].trim() == "'1':0.5" || hotValues[1].trim() ==
"'0':0.5")
- // Test 5: Verify full analyze produces same hot_value as sample analyze
for same data
+ // Test 6: Sample analyze rejects WITH HOT VALUE because it always
collects hot_value.
+ test {
+ sql """analyze table full_hot_skew with sample rows 400 with sync with
hot value"""
+ exception "Sample analyze always collects hot value"
+ }
+
+ // Test 7: Explicit full analyze produces same hot_value as sample analyze
for same data.
sql """drop stats full_hot_skew"""
- sql """analyze table full_hot_skew with sync"""
+ sql """analyze table full_hot_skew with sync with hot value"""
def fullResult = sql """show column stats full_hot_skew(value1)"""
- logger.info("Test5 full result: " + fullResult)
+ logger.info("Test7 full result: " + fullResult)
assertEquals(1, fullResult.size())
assertTrue(fullResult[0][17].contains(":"))
def fullParts = fullResult[0][17].split(";").collect { it.trim() }.sort()
@@ -113,14 +132,14 @@ suite("test_full_analyze_hot_value") {
sql """drop stats full_hot_skew"""
sql """analyze table full_hot_skew with sample rows 40000 with sync"""
def sampleResult = sql """show column stats full_hot_skew(value1)"""
- logger.info("Test5 sample result: " + sampleResult)
+ logger.info("Test7 sample result: " + sampleResult)
assertEquals(1, sampleResult.size())
assertTrue(sampleResult[0][17].contains(":"))
// Both full and sample should produce the same hot_value entries (order
may differ)
def sampleParts = sampleResult[0][17].split(";").collect { it.trim()
}.sort()
assertEquals(fullParts, sampleParts)
- // Test 6: Full analyze on empty table should produce an empty hot_value
string
+ // Test 8: Explicit full analyze on empty table should produce an empty
hot_value string.
sql """drop table if exists full_hot_empty"""
sql """CREATE TABLE full_hot_empty (
key1 int NULL,
@@ -133,14 +152,14 @@ suite("test_full_analyze_hot_value") {
"replication_num" = "1"
)
"""
- sql """analyze table full_hot_empty with sync"""
+ sql """analyze table full_hot_empty with sync with hot value"""
result = sql """show column stats full_hot_empty(value1)"""
- logger.info("Test6 empty table result: " + result)
+ logger.info("Test8 empty table result: " + result)
assertEquals(1, result.size())
assertEquals("0.0", result[0][2])
assertEquals("''", result[0][17])
- // Test 7: Full analyze on all-NULL column should produce an empty
hot_value string
+ // Test 9: Explicit full analyze on all-NULL column should produce an
empty hot_value string.
sql """drop table if exists full_hot_all_null"""
sql """CREATE TABLE full_hot_all_null (
key1 int NULL,
@@ -154,9 +173,9 @@ suite("test_full_analyze_hot_value") {
)
"""
sql """insert into full_hot_all_null select number, null from
numbers("number"="100")"""
- sql """analyze table full_hot_all_null with sync"""
+ sql """analyze table full_hot_all_null with sync with hot value"""
result = sql """show column stats full_hot_all_null(value1)"""
- logger.info("Test7 all-null result: " + result)
+ logger.info("Test9 all-null result: " + result)
assertEquals(1, result.size())
assertEquals("100.0", result[0][2])
assertEquals("100.0", result[0][4])
diff --git a/regression-test/suites/statistics/test_hot_value.groovy
b/regression-test/suites/statistics/test_hot_value.groovy
index 4d8ee5643df..8b62ff715ec 100644
--- a/regression-test/suites/statistics/test_hot_value.groovy
+++ b/regression-test/suites/statistics/test_hot_value.groovy
@@ -80,7 +80,7 @@ suite("test_hot_value") {
sql """insert into test2 select number, " : ;a" from
numbers("number"="10000")"""
wait_row_count_reported("test_hot_value", "test1", 0, 4, "10000")
wait_row_count_reported("test_hot_value", "test2", 0, 4, "10000")
- sql """analyze table test1 with sync"""
+ sql """analyze table test1 with sync with hot value"""
explain {
sql("memo plan select * from test1")
contains "hotValues=("
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
