This is an automated email from the ASF dual-hosted git repository.
englefly pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 855140f8a35 [improvement](statistics)Return -1 to neredis if report
olap table row count for new table is not done for all tablets. (#40457)
855140f8a35 is described below
commit 855140f8a3563f88e01a253c9a319075768cdff0
Author: Jibing-Li <64681310+jibing...@users.noreply.github.com>
AuthorDate: Sat Sep 7 22:06:22 2024 +0800
[improvement](statistics)Return -1 to neredis if report olap table row
count for new table is not done for all tablets. (#40457)
Return -1 to neredis if report olap table row count for new table is not
done for all tablets.
After this change, nereids could know new table is empty or not. When
it's not empty but not reported yet, return -1 as row count to nereids.
---
be/src/olap/tablet_manager.cpp | 1 +
.../apache/doris/analysis/ShowTableStatsStmt.java | 66 +++++++----------
.../apache/doris/catalog/CloudTabletStatMgr.java | 1 +
.../apache/doris/catalog/MaterializedIndex.java | 10 +++
.../java/org/apache/doris/catalog/OlapTable.java | 7 +-
.../java/org/apache/doris/catalog/Replica.java | 10 +++
.../org/apache/doris/catalog/TabletStatMgr.java | 29 +++++++-
.../doris/nereids/stats/StatsCalculator.java | 2 +-
.../java/org/apache/doris/qe/ShowExecutor.java | 9 +--
.../apache/doris/service/FrontendServiceImpl.java | 3 +
.../org/apache/doris/statistics/AnalysisInfo.java | 5 --
.../apache/doris/statistics/AnalysisManager.java | 5 ++
.../apache/doris/statistics/BaseAnalysisTask.java | 2 -
.../apache/doris/statistics/OlapAnalysisTask.java | 2 +-
.../doris/statistics/StatisticsRepository.java | 1 -
.../apache/doris/statistics/TableStatsMeta.java | 28 ++-----
.../doris/transaction/DatabaseTransactionMgr.java | 3 +
gensrc/thrift/BackendService.thrift | 1 +
.../suites/statistics/analyze_stats.groovy | 2 +-
.../suites/statistics/test_analyze_mv.groovy | 85 +++++++++++++++++++---
20 files changed, 179 insertions(+), 93 deletions(-)
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 3bac9637033..6525e9185da 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -1065,6 +1065,7 @@ void
TabletManager::build_all_report_tablets_info(std::map<TTabletId, TTablet>*
t_tablet_stat.__set_row_count(tablet_info.row_count);
t_tablet_stat.__set_total_version_count(tablet_info.total_version_count);
t_tablet_stat.__set_visible_version_count(tablet_info.visible_version_count);
+ t_tablet_stat.__set_visible_version(tablet_info.version);
};
for_each_tablet(handler, filter_all_tablets);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java
index 54c8f652e12..ccffee3086d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java
@@ -75,8 +75,9 @@ public class ShowTableStatsStmt extends ShowStmt implements
NotFallbackInParser
new ImmutableList.Builder<String>()
.add("table_name")
.add("index_name")
- .add("row_count")
- .add("update_time")
+ .add("analyze_row_count")
+ .add("report_row_count")
+ .add("report_row_count_for_nereids")
.build();
private static final ImmutableList<String> COLUMN_PARTITION_TITLE_NAMES =
@@ -200,15 +201,15 @@ public class ShowTableStatsStmt extends ShowStmt
implements NotFallbackInParser
public ShowResultSet constructResultSet(TableStatsMeta tableStatistic,
TableIf table) {
if (indexName != null) {
- return constructIndexResultSet(tableStatistic);
+ return constructIndexResultSet(tableStatistic, table);
}
if (partitionNames == null) {
return constructTableResultSet(tableStatistic, table);
}
if (columnNames == null) {
- return constructPartitionResultSet(tableStatistic);
+ return constructPartitionResultSet(tableStatistic, table);
} else {
- return constructColumnPartitionResultSet(tableStatistic);
+ return constructColumnPartitionResultSet(tableStatistic, table);
}
}
@@ -216,31 +217,23 @@ public class ShowTableStatsStmt extends ShowStmt
implements NotFallbackInParser
return new ShowResultSet(getMetaData(), new ArrayList<>());
}
- public ShowResultSet constructResultSet(TableIf table) {
- List<List<String>> result = Lists.newArrayList();
- if (partitionNames != null) {
- // For partition, return empty result if table stats not exist.
- return new ShowResultSet(getMetaData(), result);
- }
- List<String> row = Lists.newArrayList();
- row.add("");
- row.add("");
- row.add(String.valueOf(table.getCachedRowCount()));
- row.add("");
- row.add("");
- row.add("");
- row.add("");
- row.add("");
- row.add(String.valueOf(table.autoAnalyzeEnabled()));
- result.add(row);
- return new ShowResultSet(getMetaData(), result);
- }
-
public ShowResultSet constructTableResultSet(TableStatsMeta
tableStatistic, TableIf table) {
- DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd
HH:mm:ss");
if (tableStatistic == null) {
- return new ShowResultSet(getMetaData(), new ArrayList<>());
+ List<List<String>> result = Lists.newArrayList();
+ List<String> row = Lists.newArrayList();
+ row.add("");
+ row.add("");
+ row.add(String.valueOf(table.getCachedRowCount()));
+ row.add("");
+ row.add("");
+ row.add("");
+ row.add("");
+ row.add("");
+ row.add(String.valueOf(table.autoAnalyzeEnabled()));
+ result.add(row);
+ return new ShowResultSet(getMetaData(), result);
}
+ DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd
HH:mm:ss");
List<List<String>> result = Lists.newArrayList();
List<String> row = Lists.newArrayList();
row.add(String.valueOf(tableStatistic.updatedRows));
@@ -260,9 +253,9 @@ public class ShowTableStatsStmt extends ShowStmt implements
NotFallbackInParser
return new ShowResultSet(getMetaData(), result);
}
- public ShowResultSet constructPartitionResultSet(TableStatsMeta
tableStatistic) {
+ public ShowResultSet constructPartitionResultSet(TableStatsMeta
tableStatistic, TableIf table) {
List<List<String>> result = Lists.newArrayList();
- if (!(table instanceof OlapTable)) {
+ if (!(table instanceof OlapTable) || tableStatistic == null) {
return new ShowResultSet(getMetaData(), result);
}
Collection<String> partitions = partitionNames.isStar()
@@ -286,7 +279,7 @@ public class ShowTableStatsStmt extends ShowStmt implements
NotFallbackInParser
return new ShowResultSet(getMetaData(), result);
}
- public ShowResultSet constructIndexResultSet(TableStatsMeta
tableStatistic) {
+ public ShowResultSet constructIndexResultSet(TableStatsMeta
tableStatistic, TableIf table) {
List<List<String>> result = Lists.newArrayList();
if (!(table instanceof OlapTable)) {
return new ShowResultSet(getMetaData(), result);
@@ -296,23 +289,20 @@ public class ShowTableStatsStmt extends ShowStmt
implements NotFallbackInParser
if (indexId == null) {
throw new RuntimeException(String.format("Index %s not exist.",
indexName));
}
- long rowCount =
tableStatistic.getRowCount(olapTable.getIndexIdByName(indexName));
- if (rowCount == -1) {
- return new ShowResultSet(getMetaData(), result);
- }
- long updateTime =
tableStatistic.getRowCountUpdateTime(olapTable.getIndexIdByName(indexName));
+ long rowCount = tableStatistic == null ? -1 :
tableStatistic.getRowCount(olapTable.getIndexIdByName(indexName));
List<String> row = Lists.newArrayList();
row.add(table.getName());
row.add(indexName);
row.add(String.valueOf(rowCount));
- row.add(String.valueOf(updateTime));
+ row.add(String.valueOf(olapTable.getRowCountForIndex(indexId, false)));
+ row.add(String.valueOf(olapTable.getRowCountForIndex(indexId, true)));
result.add(row);
return new ShowResultSet(getMetaData(), result);
}
- public ShowResultSet constructColumnPartitionResultSet(TableStatsMeta
tableStatistic) {
+ public ShowResultSet constructColumnPartitionResultSet(TableStatsMeta
tableStatistic, TableIf table) {
List<List<String>> result = Lists.newArrayList();
- if (!(table instanceof OlapTable)) {
+ if (!(table instanceof OlapTable) || tableStatistic == null) {
return new ShowResultSet(getMetaData(), result);
}
OlapTable olapTable = (OlapTable) table;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
index ff01f736081..0ba2686060e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
@@ -199,6 +199,7 @@ public class CloudTabletStatMgr extends MasterDaemon {
tableRowsetCount += tabletRowsetCount;
tableSegmentCount += tabletSegmentCount;
} // end for tablets
+ index.setRowCountReported(true);
index.setRowCount(indexRowCount);
} // end for indices
} // end for partitions
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
index d4da2175323..6e4f6a2248e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
@@ -73,6 +73,8 @@ public class MaterializedIndex extends MetaObject implements
GsonPostProcessable
@SerializedName(value = "rollupFinishedVersion")
private long rollupFinishedVersion;
+ private boolean rowCountReported = false;
+
public MaterializedIndex() {
this.state = IndexState.NORMAL;
this.idToTablets = new HashMap<>();
@@ -206,6 +208,14 @@ public class MaterializedIndex extends MetaObject
implements GsonPostProcessable
return -1;
}
+ public void setRowCountReported(boolean reported) {
+ this.rowCountReported = reported;
+ }
+
+ public boolean getRowCountReported() {
+ return this.rowCountReported;
+ }
+
@Deprecated
public void readFields(DataInput in) throws IOException {
super.readFields(in);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 4bc05aff206..a60da71b299 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -1564,13 +1564,16 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
@Override
public long fetchRowCount() {
- return getRowCountForIndex(baseIndexId);
+ return getRowCountForIndex(baseIndexId, false);
}
- public long getRowCountForIndex(long indexId) {
+ public long getRowCountForIndex(long indexId, boolean strict) {
long rowCount = 0;
for (Map.Entry<Long, Partition> entry : idToPartition.entrySet()) {
MaterializedIndex index = entry.getValue().getIndex(indexId);
+ if (strict && !index.getRowCountReported()) {
+ return -1;
+ }
rowCount += (index == null || index.getRowCount() == -1) ? 0 :
index.getRowCount();
}
return rowCount;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java
index 0fcbef00743..f7985e57d00 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Replica.java
@@ -177,6 +177,8 @@ public class Replica {
private long userDropTime = -1;
+ private long lastReportVersion = 0;
+
public Replica() {
}
@@ -839,4 +841,12 @@ public class Replica {
return
Env.getCurrentSystemInfo().checkBackendScheduleAvailable(backendId)
&& !isUserDrop();
}
+
+ public void setLastReportVersion(long version) {
+ this.lastReportVersion = version;
+ }
+
+ public long getLastReportVersion() {
+ return lastReportVersion;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
index 1206c6b2f82..faf1fcab94f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
@@ -128,6 +128,7 @@ public class TabletStatMgr extends MasterDaemon {
long version = partition.getVisibleVersion();
for (MaterializedIndex index :
partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
long indexRowCount = 0L;
+ boolean indexReported = true;
for (Tablet tablet : index.getTablets()) {
Long tabletDataSize = 0L;
@@ -135,9 +136,26 @@ public class TabletStatMgr extends MasterDaemon {
Long tabletRowCount = 0L;
+ boolean tabletReported = false;
for (Replica replica : tablet.getReplicas()) {
+ LOG.debug("Table {} replica {} current
version {}, report version {}",
+ olapTable.getName(),
replica.getId(),
+ replica.getVersion(),
replica.getLastReportVersion());
if (replica.checkVersionCatchUp(version,
false)
- && replica.getRowCount() >
tabletRowCount) {
+ && replica.getRowCount() >=
tabletRowCount) {
+ // 1. If replica version and reported
replica version are all equal to
+ // PARTITION_INIT_VERSION, set
tabletReported to true, which indicates this
+ // tablet is empty for sure when
previous report.
+ // 2. If last report version is larger
than PARTITION_INIT_VERSION, set
+ // tabletReported to true as well.
That is, we only guarantee all replicas of
+ // the tablet are reported for the
init version.
+ // e.g. When replica version is 2, but
last reported version is 1,
+ // tabletReported would be false.
+ if (replica.getVersion() ==
Partition.PARTITION_INIT_VERSION
+ &&
replica.getLastReportVersion() == Partition.PARTITION_INIT_VERSION
+ ||
replica.getLastReportVersion() > Partition.PARTITION_INIT_VERSION) {
+ tabletReported = true;
+ }
tabletRowCount = replica.getRowCount();
}
@@ -157,8 +175,14 @@ public class TabletStatMgr extends MasterDaemon {
tableRowCount += tabletRowCount;
indexRowCount += tabletRowCount;
+ // Only when all tablets of this index are
reported, we set indexReported to true.
+ indexReported = indexReported &&
tabletReported;
} // end for tablets
+ index.setRowCountReported(indexReported);
index.setRowCount(indexRowCount);
+ LOG.debug("Table {} index {} all tablets
reported[{}], row count {}",
+ olapTable.getName(),
olapTable.getIndexNameById(index.getId()),
+ indexReported, tableRowCount);
} // end for indices
} // end for partitions
@@ -193,6 +217,9 @@ public class TabletStatMgr extends MasterDaemon {
replica.setTotalVersionCount(stat.getTotalVersionCount());
replica.setVisibleVersionCount(stat.isSetVisibleVersionCount() ?
stat.getVisibleVersionCount()
: stat.getTotalVersionCount());
+ // Older version BE doesn't set visible version. Set
it to max for compatibility.
+
replica.setLastReportVersion(stat.isSetVisibleVersion() ?
stat.getVisibleVersion()
+ : Long.MAX_VALUE);
}
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
index fec744b86b1..9ea58115022 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -366,7 +366,7 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
private Statistics computeOlapScan(OlapScan olapScan) {
OlapTable olapTable = olapScan.getTable();
- double tableRowCount =
olapTable.getRowCountForIndex(olapScan.getSelectedIndexId());
+ double tableRowCount =
olapTable.getRowCountForIndex(olapScan.getSelectedIndexId(), true);
if (tableRowCount <= 0) {
AnalysisManager analysisManager =
Env.getCurrentEnv().getAnalysisManager();
TableStatsMeta tableMeta =
analysisManager.findTableStatsStatus(olapScan.getTable().getId());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index 819c9875991..c2170f90d57 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -2725,14 +2725,7 @@ public class ShowExecutor {
return;
}
TableStatsMeta tableStats =
Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(tableIf.getId());
- /*
- tableStats == null means it's not analyzed, in this case show the
estimated row count.
- */
- if (tableStats == null) {
- resultSet = showTableStatsStmt.constructResultSet(tableIf);
- } else {
- resultSet = showTableStatsStmt.constructResultSet(tableStats,
tableIf);
- }
+ resultSet = showTableStatsStmt.constructResultSet(tableStats, tableIf);
}
private void handleShowColumnStats() throws AnalysisException {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index cc613cb5d8c..77f79bc5768 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -3319,6 +3319,9 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
partitionNames = new PartitionNames(false, new
ArrayList<>(target.partitions));
}
if (target.isTruncate) {
+ if (partitionNames == null || partitionNames.isStar() ||
partitionNames.getPartitionNames() == null) {
+ tableStats.clearIndexesRowCount();
+ }
analysisManager.submitAsyncDropStatsTask(target.catalogId,
target.dbId,
target.tableId, tableStats, partitionNames);
} else {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
index 0e89b5225ab..6ec413821ea 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
@@ -198,7 +198,6 @@ public class AnalysisInfo implements Writable {
public final boolean enablePartition;
public final ConcurrentMap<Long, Long> indexesRowCount = new
ConcurrentHashMap<>();
- public final ConcurrentMap<Long, Long> indexesRowCountUpdateTime = new
ConcurrentHashMap<>();
public AnalysisInfo(long jobId, long taskId, List<Long> taskIds, long
catalogId, long dbId, long tblId,
Set<Pair<String, String>> jobColumns, Set<String> partitionNames,
String colName, Long indexId,
@@ -358,8 +357,4 @@ public class AnalysisInfo implements Writable {
public void addIndexRowCount(long indexId, long rowCount) {
indexesRowCount.put(indexId, rowCount);
}
-
- public void addIndexUpdateRowCountTime(long indexId, long time) {
- indexesRowCountUpdateTime.put(indexId, time);
- }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index 55b0bdc4efd..634f1074cd5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -682,6 +682,9 @@ public class AnalysisManager implements Writable {
long catalogId = table.getDatabase().getCatalog().getId();
long dbId = table.getDatabase().getId();
long tableId = table.getId();
+ if (partitionNames == null || partitionNames.isStar() ||
partitionNames.getPartitionNames() == null) {
+ tableStats.clearIndexesRowCount();
+ }
submitAsyncDropStatsTask(catalogId, dbId, tableId, tableStats,
partitionNames);
// Drop stats ddl is master only operation.
Set<String> partitions = null;
@@ -810,6 +813,8 @@ public class AnalysisManager implements Writable {
}
if (allColumn && allPartition) {
tableStats.removeAllColumn();
+ tableStats.clearIndexesRowCount();
+ removeTableStats(tableId);
}
tableStats.updatedTime = 0;
tableStats.userInjected = false;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
index a16c0275da2..329231f3604 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
@@ -498,7 +498,6 @@ public abstract class BaseAnalysisTask {
try (AutoCloseConnectContext a =
StatisticsUtil.buildConnectContext()) {
stmtExecutor = new StmtExecutor(a.connectContext, sql);
ColStatsData colStatsData = new
ColStatsData(stmtExecutor.executeInternalQuery().get(0));
- long analyzeTimestamp = System.currentTimeMillis();
// Update index row count after analyze.
if (this instanceof OlapAnalysisTask) {
AnalysisInfo jobInfo =
Env.getCurrentEnv().getAnalysisManager().findJobInfo(job.getJobInfo().jobId);
@@ -506,7 +505,6 @@ public abstract class BaseAnalysisTask {
jobInfo = jobInfo == null ? job.jobInfo : jobInfo;
long indexId = info.indexId == -1 ? ((OlapTable)
tbl).getBaseIndexId() : info.indexId;
jobInfo.addIndexRowCount(indexId, colStatsData.count);
- jobInfo.addIndexUpdateRowCountTime(indexId, analyzeTimestamp);
}
Env.getCurrentEnv().getStatisticsCache().syncColStats(colStatsData);
queryId = DebugUtil.printId(stmtExecutor.getContext().queryId());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
index 5f4d812c265..ab81851cf6c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
@@ -101,7 +101,7 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
List<Long> tabletIds = pair.first;
long totalRowCount = info.indexId == -1
? tbl.getRowCount()
- : ((OlapTable) tbl).getRowCountForIndex(info.indexId);
+ : ((OlapTable) tbl).getRowCountForIndex(info.indexId, false);
double scaleFactor = (double) totalRowCount / (double) pair.second;
// might happen if row count in fe metadata hasn't been updated yet
if (Double.isInfinite(scaleFactor) || Double.isNaN(scaleFactor)) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
index 83350d16e04..0d9fa367462 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
@@ -392,7 +392,6 @@ public class StatisticsRepository {
if (objects.table instanceof OlapTable) {
indexId = indexId == -1 ? ((OlapTable)
objects.table).getBaseIndexId() : indexId;
mockedJobInfo.addIndexRowCount(indexId, (long)
Double.parseDouble(rowCount));
- mockedJobInfo.addIndexUpdateRowCountTime(indexId, timestamp);
}
Env.getCurrentEnv().getAnalysisManager().updateTableStatsForAlterStats(mockedJobInfo,
objects.table);
} else {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
index 2d566d71cfc..de0f0eed18d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
@@ -101,10 +101,7 @@ public class TableStatsMeta implements Writable,
GsonPostProcessable {
public ConcurrentMap<Long, Long> partitionUpdateRows = new
ConcurrentHashMap<>();
@SerializedName("irc")
- public ConcurrentMap<Long, Long> indexesRowCount = new
ConcurrentHashMap<>();
-
- @SerializedName("ircut")
- public ConcurrentMap<Long, Long> indexesRowCountUpdateTime = new
ConcurrentHashMap<>();
+ private ConcurrentMap<Long, Long> indexesRowCount = new
ConcurrentHashMap<>();
@VisibleForTesting
public TableStatsMeta() {
@@ -194,8 +191,7 @@ public class TableStatsMeta implements Writable,
GsonPostProcessable {
if (tableIf != null) {
if (tableIf instanceof OlapTable) {
indexesRowCount.putAll(analyzedJob.indexesRowCount);
-
indexesRowCountUpdateTime.putAll(analyzedJob.indexesRowCountUpdateTime);
- clearStaleIndexRowCountAndTime((OlapTable) tableIf);
+ clearStaleIndexRowCount((OlapTable) tableIf);
}
rowCount = analyzedJob.rowCount;
if (rowCount == 0 &&
AnalysisMethod.SAMPLE.equals(analyzedJob.analysisMethod)) {
@@ -227,20 +223,17 @@ public class TableStatsMeta implements Writable,
GsonPostProcessable {
if (colToColStatsMeta == null) {
colToColStatsMeta = new ConcurrentHashMap<>();
}
- if (indexesRowCountUpdateTime == null) {
- indexesRowCountUpdateTime = new ConcurrentHashMap<>();
- }
}
public long getRowCount(long indexId) {
return indexesRowCount.getOrDefault(indexId, -1L);
}
- public long getRowCountUpdateTime(long indexId) {
- return indexesRowCountUpdateTime.getOrDefault(indexId, 0L);
+ public void clearIndexesRowCount() {
+ indexesRowCount.clear();
}
- private void clearStaleIndexRowCountAndTime(OlapTable table) {
+ private void clearStaleIndexRowCount(OlapTable table) {
Iterator<Long> iterator = indexesRowCount.keySet().iterator();
List<Long> indexIds = table.getIndexIds();
while (iterator.hasNext()) {
@@ -249,18 +242,11 @@ public class TableStatsMeta implements Writable,
GsonPostProcessable {
iterator.remove();
}
}
- iterator = indexesRowCountUpdateTime.keySet().iterator();
- while (iterator.hasNext()) {
- long key = iterator.next();
- if (indexIds.contains(key)) {
- iterator.remove();
- }
- }
}
public long getBaseIndexDeltaRowCount(OlapTable table) {
- if (colToColStatsMeta == null) {
- return -1;
+ if (colToColStatsMeta == null || colToColStatsMeta.isEmpty() ||
userInjected) {
+ return 0;
}
long maxUpdateRows = 0;
String baseIndexName = table.getIndexNameById(table.getBaseIndexId());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
index 5e9c22bede7..73c930fb4c6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
@@ -2294,6 +2294,9 @@ public class DatabaseTransactionMgr {
replica.getId(), newVersion,
lastFailedVersion, lastSuccessVersion);
}
replica.updateVersionWithFailed(newVersion,
lastFailedVersion, lastSuccessVersion);
+ if (newVersion == Partition.PARTITION_INIT_VERSION
+ 1) {
+ index.setRowCountReported(false);
+ }
Set<Long> partitionIds =
backendPartitions.get(replica.getBackendId());
if (partitionIds == null) {
partitionIds = Sets.newHashSet();
diff --git a/gensrc/thrift/BackendService.thrift
b/gensrc/thrift/BackendService.thrift
index 058f84aadc9..ed0ae243a1d 100644
--- a/gensrc/thrift/BackendService.thrift
+++ b/gensrc/thrift/BackendService.thrift
@@ -38,6 +38,7 @@ struct TTabletStat {
4: optional i64 total_version_count
5: optional i64 remote_data_size
6: optional i64 visible_version_count
+ 7: optional i64 visible_version
}
struct TTabletStatResult {
diff --git a/regression-test/suites/statistics/analyze_stats.groovy
b/regression-test/suites/statistics/analyze_stats.groovy
index 37f59406d25..fcdb66fcbab 100644
--- a/regression-test/suites/statistics/analyze_stats.groovy
+++ b/regression-test/suites/statistics/analyze_stats.groovy
@@ -2743,7 +2743,7 @@ PARTITION `p599` VALUES IN (599)
assertEquals("true", alter_result[0][7])
sql """drop stats alter_test"""
alter_result = sql """show table stats alter_test"""
- assertEquals("false", alter_result[0][7])
+ assertEquals("", alter_result[0][7])
sql """alter table alter_test modify column id set stats
('row_count'='100', 'ndv'='0', 'num_nulls'='0.0', 'data_size'='2.69975443E8',
'min_value'='1', 'max_value'='2');"""
alter_result = sql """show column stats alter_test(id)"""
logger.info("show column alter_test(id) stats: " + alter_result)
diff --git a/regression-test/suites/statistics/test_analyze_mv.groovy
b/regression-test/suites/statistics/test_analyze_mv.groovy
index 4b9644c4821..d6ee9b4cfc7 100644
--- a/regression-test/suites/statistics/test_analyze_mv.groovy
+++ b/regression-test/suites/statistics/test_analyze_mv.groovy
@@ -127,39 +127,45 @@ suite("test_analyze_mv") {
"replication_num" = "1"
)
"""
+ def result_row
+ if (!isCloudMode()) {
+ // Test row count report and report for nereids
+ result_row = sql """show index stats mvTestDup mvTestDup"""
+ assertEquals(1, result_row.size())
+ assertEquals("mvTestDup", result_row[0][0])
+ assertEquals("mvTestDup", result_row[0][1])
+ assertEquals("0", result_row[0][3])
+ assertEquals("-1", result_row[0][4])
+ }
+
createMV("create materialized view mv1 as select key1 from mvTestDup;")
createMV("create materialized view mv2 as select key2 from mvTestDup;")
createMV("create materialized view mv3 as select key1, key2, sum(value1),
max(value2), min(value3) from mvTestDup group by key1, key2;")
sql """insert into mvTestDup values (1, 2, 3, 4, 5), (1, 2, 3, 4, 5), (10,
20, 30, 40, 50), (10, 20, 30, 40, 50), (100, 200, 300, 400, 500), (1001, 2001,
3001, 4001, 5001);"""
- def timestamp = System.currentTimeMillis();
sql """analyze table mvTestDup with sync;"""
// Test show index row count
- def result_row = sql """show index stats mvTestDup mvTestDup"""
+ result_row = sql """show index stats mvTestDup mvTestDup"""
assertEquals(1, result_row.size())
assertEquals("mvTestDup", result_row[0][0])
assertEquals("mvTestDup", result_row[0][1])
assertEquals("6", result_row[0][2])
- assertTrue(Long.parseLong(result_row[0][3]) >= timestamp)
result_row = sql """show index stats mvTestDup mv1"""
assertEquals(1, result_row.size())
assertEquals("mvTestDup", result_row[0][0])
assertEquals("mv1", result_row[0][1])
assertEquals("6", result_row[0][2])
- assertTrue(Long.parseLong(result_row[0][3]) >= timestamp)
result_row = sql """show index stats mvTestDup mv2"""
assertEquals(1, result_row.size())
assertEquals("mvTestDup", result_row[0][0])
assertEquals("mv2", result_row[0][1])
assertEquals("6", result_row[0][2])
- assertTrue(Long.parseLong(result_row[0][3]) >= timestamp)
result_row = sql """show index stats mvTestDup mv3"""
assertEquals(1, result_row.size())
assertEquals("mvTestDup", result_row[0][0])
assertEquals("mv3", result_row[0][1])
assertEquals("4", result_row[0][2])
- assertTrue(Long.parseLong(result_row[0][3]) >= timestamp)
// Compare show whole table column stats result with show single column.
def result_all = sql """show column stats mvTestDup"""
@@ -438,27 +444,21 @@ suite("test_analyze_mv") {
assertEquals("FULL", result_sample[0][9])
// Test alter table index row count.
- timestamp = System.currentTimeMillis();
sql """alter table mvTestDup modify column `value2` set stats
('row_count'='1.5E8', 'ndv'='3.0', 'num_nulls'='0.0', 'data_size'='1.5E8',
'min_value'='1', 'max_value'='10');"""
result_row = sql """show index stats mvTestDup mvTestDup;"""
assertEquals("mvTestDup", result_row[0][0])
assertEquals("mvTestDup", result_row[0][1])
assertEquals("150000000", result_row[0][2])
- assertTrue(Long.parseLong(result_row[0][3]) >= timestamp)
- timestamp = System.currentTimeMillis();
sql """alter table mvTestDup index mv1 modify column `mv_key1` set stats
('row_count'='3443', 'ndv'='3.0', 'num_nulls'='0.0', 'data_size'='1.5E8',
'min_value'='1', 'max_value'='10');"""
result_row = sql """show index stats mvTestDup mv1;"""
assertEquals("mvTestDup", result_row[0][0])
assertEquals("mv1", result_row[0][1])
assertEquals("3443", result_row[0][2])
- assertTrue(Long.parseLong(result_row[0][3]) >= timestamp)
- timestamp = System.currentTimeMillis();
sql """alter table mvTestDup index mv3 modify column `mva_MAX__``value2```
set stats ('row_count'='234234', 'ndv'='3.0', 'num_nulls'='0.0',
'data_size'='1.5E8', 'min_value'='1', 'max_value'='10');"""
result_row = sql """show index stats mvTestDup mv3;"""
assertEquals("mvTestDup", result_row[0][0])
assertEquals("mv3", result_row[0][1])
assertEquals("234234", result_row[0][2])
- assertTrue(Long.parseLong(result_row[0][3]) >= timestamp)
sql """drop stats mvTestDup"""
result_sample = sql """show column stats mvTestDup"""
@@ -474,6 +474,35 @@ suite("test_analyze_mv") {
logger.info(e.getMessage());
return;
}
+
+ if (!isCloudMode()) {
+ // Test row count report and report for nereids
+ result_row = sql """show index stats mvTestDup mvTestDup"""
+ assertEquals(1, result_row.size())
+ assertEquals("mvTestDup", result_row[0][0])
+ assertEquals("mvTestDup", result_row[0][1])
+ assertEquals("6", result_row[0][3])
+ assertEquals("6", result_row[0][4])
+ result_row = sql """show index stats mvTestDup mv1"""
+ assertEquals(1, result_row.size())
+ assertEquals("mvTestDup", result_row[0][0])
+ assertEquals("mv1", result_row[0][1])
+ assertEquals("6", result_row[0][3])
+ assertEquals("6", result_row[0][4])
+ result_row = sql """show index stats mvTestDup mv2"""
+ assertEquals(1, result_row.size())
+ assertEquals("mvTestDup", result_row[0][0])
+ assertEquals("mv2", result_row[0][1])
+ assertEquals("6", result_row[0][3])
+ assertEquals("6", result_row[0][4])
+ result_row = sql """show index stats mvTestDup mv3"""
+ assertEquals(1, result_row.size())
+ assertEquals("mvTestDup", result_row[0][0])
+ assertEquals("mv3", result_row[0][1])
+ assertEquals("4", result_row[0][3])
+ assertEquals("4", result_row[0][4])
+ }
+
sql """analyze table mvTestDup with sample rows 4000000"""
wait_analyze_finish("mvTestDup")
result_sample = sql """SHOW ANALYZE mvTestDup;"""
@@ -621,6 +650,38 @@ suite("test_analyze_mv") {
verifyTaskStatus(result_sample, "mva_MIN__`value3`", "mv3")
verifyTaskStatus(result_sample, "mva_SUM__CAST(`value1` AS bigint)", "mv3")
+ if (!isCloudMode()) {
+ // Test row count report and report for nereids
+ sql """truncate table mvTestDup"""
+ result_row = sql """show index stats mvTestDup mv3"""
+ assertEquals(1, result_row.size())
+ assertEquals("mvTestDup", result_row[0][0])
+ assertEquals("mv3", result_row[0][1])
+ assertEquals("0", result_row[0][3])
+ assertEquals("-1", result_row[0][4])
+
+ for (int i = 0; i < 120; i++) {
+ result_row = sql """show index stats mvTestDup mv3"""
+ logger.info("mv3 stats: " + result_row)
+ if (result_row[0][4] == "0") {
+ break;
+ }
+ Thread.sleep(5000)
+ }
+ result_row = sql """show index stats mvTestDup mv3"""
+ assertEquals(1, result_row.size())
+ assertEquals("mvTestDup", result_row[0][0])
+ assertEquals("mv3", result_row[0][1])
+ assertEquals("0", result_row[0][3])
+ assertEquals("0", result_row[0][4])
+ sql """insert into mvTestDup values (1, 2, 3, 4, 5), (1, 2, 3, 4, 5),
(10, 20, 30, 40, 50), (10, 20, 30, 40, 50), (100, 200, 300, 400, 500), (1001,
2001, 3001, 4001, 5001);"""
+ result_row = sql """show index stats mvTestDup mv3"""
+ assertEquals(1, result_row.size())
+ assertEquals("mvTestDup", result_row[0][0])
+ assertEquals("mv3", result_row[0][1])
+ assertEquals("-1", result_row[0][4])
+ }
+
// Test alter column stats
sql """drop stats mvTestDup"""
sql """alter table mvTestDup modify column key1 set stats ('ndv'='1',
'num_nulls'='1', 'min_value'='10', 'max_value'='40', 'row_count'='50');"""
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
