This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 74690cec927 [improvement](statistics)Optimize drop stats operation
(#30202)
74690cec927 is described below
commit 74690cec9274d3a6ce068ca99f7ba10c57c50ae7
Author: Jibing-Li <64681310+jibing...@users.noreply.github.com>
AuthorDate: Mon Jan 22 13:30:18 2024 +0800
[improvement](statistics)Optimize drop stats operation (#30202)
Before, drop stats operation need to call columns * followers times of
isMaster() function and the same times of rpc to drop remote column stats. This
pr is to reduce the rpc calls and use more efficient way to check master node
instead of using isMaster()
backport #30144
---
.../org/apache/doris/analysis/DropStatsStmt.java | 19 +++++++++++++
.../apache/doris/service/FrontendServiceImpl.java | 2 +-
.../apache/doris/statistics/AnalysisManager.java | 33 ++++++++++++++--------
.../doris/statistics/InvalidateStatsTarget.java | 16 +++++++++--
4 files changed, 55 insertions(+), 15 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropStatsStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropStatsStmt.java
index ac08f01f31e..5e3bd20c0f1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropStatsStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropStatsStmt.java
@@ -53,7 +53,10 @@ public class DropStatsStmt extends DdlStmt {
private Set<String> columnNames;
// Flag to drop external table row count in table_statistics.
private boolean dropTableRowCount;
+ private boolean isAllColumns;
+ private long catalogId;
+ private long dbId;
private long tblId;
public DropStatsStmt(boolean dropExpired) {
@@ -100,10 +103,13 @@ public class DropStatsStmt extends DdlStmt {
DatabaseIf db = catalog.getDbOrAnalysisException(dbName);
TableIf table = db.getTableOrAnalysisException(tblName);
tblId = table.getId();
+ dbId = db.getId();
+ catalogId = catalog.getId();
// check permission
checkAnalyzePriv(db.getFullName(), table.getName());
// check columnNames
if (columnNames != null) {
+ isAllColumns = false;
for (String cName : columnNames) {
if (table.getColumn(cName) == null) {
ErrorReport.reportAnalysisException(
@@ -115,6 +121,7 @@ public class DropStatsStmt extends DdlStmt {
}
}
} else {
+ isAllColumns = true;
columnNames =
table.getColumns().stream().map(Column::getName).collect(Collectors.toSet());
}
}
@@ -123,10 +130,22 @@ public class DropStatsStmt extends DdlStmt {
return tblId;
}
+ public long getDbId() {
+ return dbId;
+ }
+
+ public long getCatalogIdId() {
+ return catalogId;
+ }
+
public Set<String> getColumnNames() {
return columnNames;
}
+ public boolean isAllColumns() {
+ return isAllColumns;
+ }
+
public boolean dropTableRowCount() {
return dropTableRowCount;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index 198fdf1d10a..69d71dc814f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -3097,7 +3097,7 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
if (tableStats == null) {
return new TStatus(TStatusCode.OK);
}
- analysisManager.invalidateLocalStats(target.tableId, target.columns,
tableStats);
+ analysisManager.invalidateLocalStats(target.catalogId, target.dbId,
target.tableId, target.columns, tableStats);
return new TStatus(TStatusCode.OK);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index 742afe2957f..1bc6da5a6c2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -653,14 +653,16 @@ public class AnalysisManager implements Writable {
}
Set<String> cols = dropStatsStmt.getColumnNames();
+ long catalogId = dropStatsStmt.getCatalogIdId();
+ long dbId = dropStatsStmt.getDbId();
long tblId = dropStatsStmt.getTblId();
TableStatsMeta tableStats =
findTableStatsStatus(dropStatsStmt.getTblId());
if (tableStats == null) {
return;
}
- invalidateLocalStats(tblId, cols, tableStats);
+ invalidateLocalStats(catalogId, dbId, tblId, cols, tableStats);
// Drop stats ddl is master only operation.
- invalidateRemoteStats(tblId, cols);
+ invalidateRemoteStats(catalogId, dbId, tblId, cols,
dropStatsStmt.isAllColumns());
StatisticsRepository.dropStatistics(tblId, cols);
}
@@ -669,30 +671,37 @@ public class AnalysisManager implements Writable {
if (tableStats == null) {
return;
}
+ long catalogId = table.getDatabase().getCatalog().getId();
+ long dbId = table.getDatabase().getId();
+ long tableId = table.getId();
Set<String> cols =
table.getBaseSchema().stream().map(Column::getName).collect(Collectors.toSet());
- invalidateLocalStats(table.getId(), cols, tableStats);
+ invalidateLocalStats(catalogId, dbId, tableId, cols, tableStats);
// Drop stats ddl is master only operation.
- invalidateRemoteStats(table.getId(), cols);
+ invalidateRemoteStats(catalogId, dbId, tableId, cols, true);
StatisticsRepository.dropStatistics(table.getId(), cols);
}
- public void invalidateLocalStats(long tableId, Set<String> columns,
TableStatsMeta tableStats) {
+ public void invalidateLocalStats(long catalogId, long dbId, long tableId,
+ Set<String> columns, TableStatsMeta
tableStats) {
if (tableStats == null) {
return;
}
StatisticsCache statisticsCache =
Env.getCurrentEnv().getStatisticsCache();
- if (columns != null) {
- for (String column : columns) {
- tableStats.removeColumn(column);
- statisticsCache.invalidate(tableId, -1, column);
- }
+ if (columns == null) {
+ TableIf table = StatisticsUtil.findTable(catalogId, dbId, tableId);
+ columns =
table.getBaseSchema().stream().map(Column::getName).collect(Collectors.toSet());
+ }
+ for (String column : columns) {
+ tableStats.removeColumn(column);
+ statisticsCache.invalidate(tableId, -1, column);
}
tableStats.updatedTime = 0;
tableStats.userInjected = false;
}
- public void invalidateRemoteStats(long tableId, Set<String> columns) {
- InvalidateStatsTarget target = new InvalidateStatsTarget(tableId,
columns);
+ public void invalidateRemoteStats(long catalogId, long dbId, long tableId,
+ Set<String> columns, boolean
isAllColumns) {
+ InvalidateStatsTarget target = new InvalidateStatsTarget(catalogId,
dbId, tableId, columns, isAllColumns);
TInvalidateFollowerStatsCacheRequest request = new
TInvalidateFollowerStatsCacheRequest();
request.key = GsonUtils.GSON.toJson(target);
StatisticsCache statisticsCache =
Env.getCurrentEnv().getStatisticsCache();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/InvalidateStatsTarget.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/InvalidateStatsTarget.java
index 1ee7b745048..e49048f8946 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/InvalidateStatsTarget.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/InvalidateStatsTarget.java
@@ -23,14 +23,26 @@ import java.util.Set;
public class InvalidateStatsTarget {
+ @SerializedName("catalogId")
+ public final long catalogId;
+
+ @SerializedName("dbId")
+ public final long dbId;
+
@SerializedName("tableId")
public final long tableId;
@SerializedName("columns")
public final Set<String> columns;
- public InvalidateStatsTarget(long tableId, Set<String> columns) {
+ public InvalidateStatsTarget(long catalogId, long dbId, long tableId,
Set<String> columns, boolean isAllColumns) {
+ this.catalogId = catalogId;
+ this.dbId = dbId;
this.tableId = tableId;
- this.columns = columns;
+ if (isAllColumns) {
+ this.columns = null;
+ } else {
+ this.columns = columns;
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
