This is an automated email from the ASF dual-hosted git repository.
lide pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new 1deb9d13b63 [fix](sql cache) Fix enable sql cache lead to FE Full GC
or OOM (#49424)
1deb9d13b63 is described below
commit 1deb9d13b63c6072d5fcefa6c4cdb6cc6d7fabb6
Author: camby <camby...@tencent.com>
AuthorDate: Tue Mar 25 21:39:34 2025 +0800
[fix](sql cache) Fix enable sql cache lead to FE Full GC or OOM (#49424)
pick https://github.com/apache/doris/pull/22645 and
https://github.com/apache/doris/pull/22769 to branch-1.2
Co-authored-by: Xinyi Zou <zouxiny...@gmail.com>
---
.../src/main/java/org/apache/doris/common/Config.java | 10 +++++++++-
.../src/main/java/org/apache/doris/qe/cache/Cache.java | 13 +++++++++++--
.../main/java/org/apache/doris/qe/cache/PartitionCache.java | 3 +++
.../java/org/apache/doris/qe/cache/RowBatchBuilder.java | 12 ++++++++++++
.../src/main/java/org/apache/doris/qe/cache/SqlCache.java | 3 +++
5 files changed, 38 insertions(+), 3 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index a0eaf40f77c..edb01513626 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1386,9 +1386,17 @@ public class Config extends ConfigBase {
/**
* Set the maximum number of rows that can be cached
*/
- @ConfField(mutable = true, masterOnly = false)
+ @ConfField(mutable = true, masterOnly = false, description =
{"SQL/Partition Cache可以缓存的最大行数。",
+ "Maximum number of rows that can be cached in SQL/Partition Cache, is
3000 by default."})
public static int cache_result_max_row_count = 3000;
+ /**
+ * Set the maximum data size that can be cached
+ */
+ @ConfField(mutable = true, masterOnly = false, description =
{"SQL/Partition Cache可以缓存的最大数据大小。",
+ "Maximum data size of rows that can be cached in SQL/Partition Cache,
is 3000 by default."})
+ public static int cache_result_max_data_size = 31457280; // 30M
+
/**
* Used to limit element num of InPredicate in delete statement.
*/
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/Cache.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/Cache.java
index 2efd805fa8a..e68597bec11 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/Cache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/Cache.java
@@ -41,6 +41,7 @@ public abstract class Cache {
protected TUniqueId queryId;
protected SelectStmt selectStmt;
protected RowBatchBuilder rowBatchBuilder;
+ protected boolean disableCache = false;
protected CacheAnalyzer.CacheTable latestTable;
protected CacheProxy proxy;
protected HitRange hitRange;
@@ -75,12 +76,20 @@ public abstract class Cache {
public abstract void updateCache();
protected boolean checkRowLimit() {
- if (rowBatchBuilder == null) {
+ if (disableCache || rowBatchBuilder == null) {
return false;
}
if (rowBatchBuilder.getRowSize() > Config.cache_result_max_row_count) {
- LOG.info("can not be cached. rowbatch size {} is more than {}",
rowBatchBuilder.getRowSize(),
+ LOG.debug("can not be cached. rowbatch size {} is more than {}",
rowBatchBuilder.getRowSize(),
Config.cache_result_max_row_count);
+ rowBatchBuilder.clear();
+ disableCache = true;
+ return false;
+ } else if (rowBatchBuilder.getDataSize() >
Config.cache_result_max_data_size) {
+ LOG.debug("can not be cached. rowbatch data size {} is more than
{}", rowBatchBuilder.getDataSize(),
+ Config.cache_result_max_data_size);
+ rowBatchBuilder.clear();
+ disableCache = true;
return false;
} else {
return true;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java
index 9113bfec82d..d65d67fdbd2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java
@@ -123,6 +123,9 @@ public class PartitionCache extends Cache {
rowBatchBuilder.buildPartitionIndex(selectStmt.getResultExprs(),
selectStmt.getColLabels(),
partColumn, range.buildUpdatePartitionRange());
}
+ if (!super.checkRowLimit()) {
+ return;
+ }
rowBatchBuilder.copyRowData(rowBatch);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/RowBatchBuilder.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/RowBatchBuilder.java
index 6d5a635a30b..2cae79597a3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/RowBatchBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/RowBatchBuilder.java
@@ -55,6 +55,10 @@ public class RowBatchBuilder {
return rowSize;
}
+ public int getDataSize() {
+ return dataSize;
+ }
+
public RowBatchBuilder(CacheAnalyzer.CacheMode model) {
cacheMode = model;
keyIndex = 0;
@@ -99,6 +103,14 @@ public class RowBatchBuilder {
}
}
+ public void clear() {
+ rowList = Lists.newArrayList();
+ cachePartMap = new HashMap<>();
+ batchSize = 0;
+ rowSize = 0;
+ dataSize = 0;
+ }
+
public InternalService.PUpdateCacheRequest buildSqlUpdateRequest(
String sql, long partitionKey, long lastVersion, long lastestTime)
{
if (updateRequest == null) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java
index 9cd6c365dcd..9d53bb47c4d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java
@@ -76,6 +76,9 @@ public class SqlCache extends Cache {
if (rowBatchBuilder == null) {
rowBatchBuilder = new RowBatchBuilder(CacheAnalyzer.CacheMode.Sql);
}
+ if (!super.checkRowLimit()) {
+ return;
+ }
rowBatchBuilder.copyRowData(rowBatch);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
