This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a commit to branch high-priority-column
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/high-priority-column by this
push:
new 43e9a70e3a3 Improve waiting empty table logic. (#33472)
43e9a70e3a3 is described below
commit 43e9a70e3a3ff6d9626a351d9cb9d1e9f266c9ee
Author: Jibing-Li <64681310+jibing...@users.noreply.github.com>
AuthorDate: Wed Apr 10 15:40:31 2024 +0800
Improve waiting empty table logic. (#33472)
---
.../apache/doris/statistics/AnalysisManager.java | 6 +++---
.../doris/statistics/ExternalAnalysisTask.java | 5 +----
.../apache/doris/statistics/OlapAnalysisTask.java | 24 +++++++---------------
.../doris/statistics/StatisticsAutoCollector.java | 4 ++--
.../doris/statistics/util/StatisticsUtil.java | 2 +-
5 files changed, 14 insertions(+), 27 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index b59ee808e75..405715a9e5a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -236,8 +236,9 @@ public class AnalysisManager implements Writable {
@VisibleForTesting
protected AnalysisInfo buildAndAssignJob(AnalyzeTblStmt stmt) throws
DdlException {
AnalysisInfo jobInfo = buildAnalysisJobInfo(stmt);
- if (jobInfo.jobColumns.isEmpty()) {
+ if (jobInfo.jobColumns == null || jobInfo.jobColumns.isEmpty()) {
// No statistics need to be collected or updated
+ LOG.info("Job columns are empty, skip analyze table {}",
stmt.getTblName().toString());
return null;
}
// Only OlapTable and Hive HMSExternalTable support sample analyze.
@@ -371,8 +372,7 @@ public class AnalysisManager implements Writable {
infoBuilder.setColName(stringJoiner.toString());
infoBuilder.setTaskIds(Lists.newArrayList());
infoBuilder.setTblUpdateTime(table.getUpdateTime());
- long rowCount = table.getRowCount();
- infoBuilder.setRowCount(rowCount);
+ infoBuilder.setRowCount(StatisticsUtil.isEmptyTable(table,
analysisMethod) ? 0 : table.getRowCount());
TableStatsMeta tableStatsStatus = findTableStatsStatus(table.getId());
infoBuilder.setUpdateRows(tableStatsStatus == null ? 0 :
tableStatsStatus.updatedRows.get());
infoBuilder.setPriority(JobPriority.MANUAL);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java
index 16e741f4563..7d3c9af2548 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/ExternalAnalysisTask.java
@@ -256,9 +256,6 @@ public class ExternalAnalysisTask extends BaseAnalysisTask {
}
target = columnSize * tableSample.getSampleValue();
}
- if (sizeToRead > LIMIT_SIZE && sizeToRead > target * LIMIT_FACTOR) {
- return true;
- }
- return false;
+ return sizeToRead > LIMIT_SIZE && sizeToRead > target * LIMIT_FACTOR;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
index c1f7765ea54..41e492fbc04 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
@@ -37,7 +37,6 @@ import org.apache.commons.text.StringSubstitutor;
import java.security.SecureRandom;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
@@ -65,11 +64,11 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
}
public void doExecute() throws Exception {
- Set<Pair<String, String>> columnList = info.jobColumns;
- if (StatisticsUtil.isEmptyTable(tbl, info.analysisMethod) ||
columnList == null || columnList.isEmpty()) {
+ // For empty table, write empty result directly, no need to run SQL to
collect stats.
+ if (info.rowCount == 0) {
StatsId statsId = new StatsId(concatColumnStatsId(),
info.catalogId, info.dbId,
info.tblId, info.indexId, info.colName, null);
- job.appendBuf(this, Arrays.asList(new ColStatsData(statsId)));
+ job.appendBuf(this, Collections.singletonList(new
ColStatsData(statsId)));
return;
}
if (tableSample != null) {
@@ -84,7 +83,7 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
* 2. estimate partition stats
* 3. insert col stats and partition stats
*/
- protected void doSample() throws Exception {
+ protected void doSample() {
if (LOG.isDebugEnabled()) {
LOG.debug("Will do sample collection for column {}",
col.getName());
}
@@ -209,7 +208,7 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
* 2. insert partition in batch
* 3. calculate column stats based on partition stats
*/
- protected void doFull() throws Exception {
+ protected void doFull() {
if (LOG.isDebugEnabled()) {
LOG.debug("Will do full collection for column {}", col.getName());
}
@@ -315,10 +314,7 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
return false;
}
// Partition column need to scan tablets from all partitions.
- if (tbl.isPartitionColumn(col.getName())) {
- return false;
- }
- return true;
+ return !tbl.isPartitionColumn(col.getName());
}
/**
@@ -381,12 +377,6 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
}
protected String concatColumnStatsId() {
- StringBuilder stringBuilder = new StringBuilder();
- stringBuilder.append(info.tblId);
- stringBuilder.append("-");
- stringBuilder.append(info.indexId);
- stringBuilder.append("-");
- stringBuilder.append(info.colName);
- return stringBuilder.toString();
+ return info.tblId + "-" + info.indexId + "-" + info.colName;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
index 0b37ede5a49..479610ccea2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
@@ -114,7 +114,7 @@ public class StatisticsAutoCollector extends MasterDaemon {
return Pair.of(job.get(), JobPriority.MID);
}
job = fetchJobFromMap(manager.lowPriorityJobs);
- return job.map(tableNameSetEntry -> Pair.of(tableNameSetEntry,
JobPriority.LOW)).orElse(null);
+ return job.map(entry -> Pair.of(entry, JobPriority.LOW)).orElse(null);
}
protected Optional<Map.Entry<TableName, Set<Pair<String, String>>>>
fetchJobFromMap(
@@ -183,7 +183,7 @@ public class StatisticsAutoCollector extends MasterDaemon {
? AnalysisMethod.SAMPLE : AnalysisMethod.FULL;
AnalysisManager manager = Env.getServingEnv().getAnalysisManager();
TableStatsMeta tableStatsStatus =
manager.findTableStatsStatus(table.getId());
- long rowCount = table.getRowCount();
+ long rowCount = StatisticsUtil.isEmptyTable(table, analysisMethod) ? 0
: table.getRowCount();
StringJoiner stringJoiner = new StringJoiner(",", "[", "]");
for (Pair<String, String> pair : jobColumns) {
stringJoiner.add(pair.toString());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
index 351fcbd892f..a25934dd3d1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -1020,7 +1020,7 @@ public class StatisticsUtil {
}
public static boolean isEmptyTable(TableIf table,
AnalysisInfo.AnalysisMethod method) {
- int waitRowCountReportedTime = 90;
+ int waitRowCountReportedTime = 75;
if (!(table instanceof OlapTable) ||
method.equals(AnalysisInfo.AnalysisMethod.FULL)) {
return false;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
