This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 27662c3d6288b0d1e3bffaab1edd8b0dafd2246b
Author: Mingyu Chen <morning...@163.com>
AuthorDate: Sat Apr 20 17:37:57 2024 +0800
[fix](row-count-cache) use cached row count for show (#33911)
---
.../main/java/org/apache/doris/catalog/Table.java | 5 +++++
.../java/org/apache/doris/catalog/TableIf.java | 5 +++++
.../doris/datasource/ExternalMetaCacheMgr.java | 2 +-
.../doris/datasource/ExternalRowCountCache.java | 24 ++++++++++++++++++++++
.../org/apache/doris/datasource/ExternalTable.java | 13 ++++++++++++
.../java/org/apache/doris/qe/ShowExecutor.java | 4 ++--
.../apache/doris/service/FrontendServiceImpl.java | 2 +-
7 files changed, 51 insertions(+), 4 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
index 4fc01c07292..8e26c28ee5e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
@@ -633,4 +633,9 @@ public abstract class Table extends MetaObject implements
Writable, TableIf {
public List<Pair<String, String>> getColumnIndexPairs(Set<String> columns)
{
return Lists.newArrayList();
}
+
+ @Override
+ public long getCachedRowCount() {
+ return getRowCount();
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
index 65d810ccd51..b3e27f286c1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
@@ -160,6 +160,11 @@ public interface TableIf {
long getRowCount();
+ // Get the row count from cache,
+ // If miss, just return 0
+ // This is used for external table, because for external table, the
fetching row count may be expensive
+ long getCachedRowCount();
+
long fetchRowCount();
long getDataLength();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
index 6198f1bab9a..88278a1d342 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
@@ -89,7 +89,7 @@ public class ExternalMetaCacheMgr {
public ExternalMetaCacheMgr() {
rowCountRefreshExecutor = ThreadPoolManager.newDaemonFixedThreadPool(
Config.max_external_cache_loader_thread_pool_size,
- Config.max_external_cache_loader_thread_pool_size,
+ Config.max_external_cache_loader_thread_pool_size * 1000,
"RowCountRefreshExecutor", 0, true);
commonRefreshExecutor = ThreadPoolManager.newDaemonFixedThreadPool(
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java
index 4f88d534f92..44aecbca1d5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java
@@ -114,4 +114,28 @@ public class ExternalRowCountCache {
return 0;
}
+ /**
+ * Get cached row count for the given table if present. Return 0 if cached
not loaded.
+ * This method will not trigger async loading if cache is missing.
+ *
+ * @param catalogId
+ * @param dbId
+ * @param tableId
+ * @return
+ */
+ public long getCachedRowCountIfPresent(long catalogId, long dbId, long
tableId) {
+ RowCountKey key = new RowCountKey(catalogId, dbId, tableId);
+ try {
+ CompletableFuture<Optional<Long>> f =
rowCountCache.getIfPresent(key);
+ if (f == null) {
+ return 0;
+ } else if (f.isDone()) {
+ return f.get().orElse(0L);
+ }
+ } catch (Exception e) {
+ LOG.warn("Unexpected exception while returning row count if
present", e);
+ }
+ return 0;
+ }
+
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
index 82390b91656..952b5c64cf8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
@@ -206,6 +206,19 @@ public class ExternalTable implements TableIf, Writable,
GsonPostProcessable {
return
Env.getCurrentEnv().getExtMetaCacheMgr().getRowCountCache().getCachedRowCount(catalog.getId(),
dbId, id);
}
+ @Override
+ public long getCachedRowCount() {
+ // Return 0 if makeSureInitialized throw exception.
+ // For example, init hive table may throw NotSupportedException.
+ try {
+ makeSureInitialized();
+ } catch (Exception e) {
+ LOG.warn("Failed to initialize table {}.{}.{}", catalog.getName(),
dbName, name, e);
+ return 0;
+ }
+ return
Env.getCurrentEnv().getExtMetaCacheMgr().getRowCountCache().getCachedRowCount(catalog.getId(),
dbId, id);
+ }
+
@Override
/**
* Default return 0. Subclass need to implement this interface.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index 58d22b7181b..6a4e5df22d6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -920,7 +920,7 @@ public class ShowExecutor {
// Row_format
row.add(null);
// Rows
- row.add(String.valueOf(table.getRowCount()));
+ row.add(String.valueOf(table.getCachedRowCount()));
// Avg_row_length
row.add(String.valueOf(table.getAvgRowLength()));
// Data_length
@@ -2450,7 +2450,7 @@ public class ShowExecutor {
tableStats == null means it's not analyzed, in this case show the
estimated row count.
*/
if (tableStats == null) {
- resultSet =
showTableStatsStmt.constructResultSet(tableIf.getRowCount());
+ resultSet =
showTableStatsStmt.constructResultSet(tableIf.getCachedRowCount());
} else {
resultSet = showTableStatsStmt.constructResultSet(tableStats);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index 63da3bc12e2..64802093e06 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -612,7 +612,7 @@ public class FrontendServiceImpl implements
FrontendService.Iface {
status.setUpdateTime(table.getUpdateTime() / 1000);
status.setCheckTime(lastCheckTime / 1000);
status.setCollation("utf-8");
- status.setRows(table.getRowCount());
+ status.setRows(table.getCachedRowCount());
status.setDataLength(table.getDataLength());
status.setAvgRowLength(table.getAvgRowLength());
tablesResult.add(status);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
