This is an automated email from the ASF dual-hosted git repository.
freemandealer pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 118f9e661fb [fix](filecache) Disable sync file cache clear in http api
(#64321)
118f9e661fb is described below
commit 118f9e661fb6afd63c875e6c407fd9a2e8b71f15
Author: zhengyu <[email protected]>
AuthorDate: Wed Jun 10 10:32:00 2026 +0800
[fix](filecache) Disable sync file cache clear in http api (#64321)
Problem Summary: The file cache HTTP clear endpoint allowed callers to
request synchronous clear with sync=true. The synchronous clear path is
kept for unit tests and internal callers, but it is not safe to expose
through the HTTP API. This change makes the HTTP op=clear endpoint
always use async clear. The default behavior remains async, and explicit
sync=true returns a warning message while still running async clear.
Segment-specific HTTP clear also uses async remove instead of the
synchronous remove path.
---
be/src/service/http/action/file_cache_action.cpp | 14 +++-
be/test/service/http/file_cache_action_test.cpp | 86 +++++++++++++++++++++++-
2 files changed, 95 insertions(+), 5 deletions(-)
diff --git a/be/src/service/http/action/file_cache_action.cpp
b/be/src/service/http/action/file_cache_action.cpp
index 97f46c219f7..2402f1ad572 100644
--- a/be/src/service/http/action/file_cache_action.cpp
+++ b/be/src/service/http/action/file_cache_action.cpp
@@ -59,6 +59,8 @@ constexpr static std::string_view RELEASED_ELEMENTS =
"released_elements";
constexpr static std::string_view DUMP = "dump";
constexpr static std::string_view VALUE = "value";
constexpr static std::string_view RELOAD = "reload";
+constexpr static std::string_view SYNC_CLEAR_UNSUPPORTED_MSG =
+ "sync clear_file_cache is no longer supported in http api, running
async clear instead";
Status FileCacheAction::_handle_header(HttpRequest* req, std::string*
json_metrics) {
const std::string header_json(HEADER_JSON);
@@ -85,11 +87,17 @@ Status FileCacheAction::_handle_header(HttpRequest* req,
std::string* json_metri
const std::string& sync = req->param(std::string(SYNC));
const std::string& segment_path = req->param(std::string(VALUE));
if (segment_path.empty()) {
- io::FileCacheFactory::instance()->clear_file_caches(to_lower(sync)
== "true");
+ io::FileCacheFactory::instance()->clear_file_caches(false);
} else {
io::UInt128Wrapper hash = io::BlockFileCache::hash(segment_path);
io::BlockFileCache* cache =
io::FileCacheFactory::instance()->get_by_path(hash);
- cache->remove_if_cached(hash);
+ cache->remove_if_cached_async(hash);
+ }
+ if (to_lower(sync) == "true") {
+ EasyJson json;
+ json["status"] = "OK";
+ json["msg"] = std::string(SYNC_CLEAR_UNSUPPORTED_MSG);
+ *json_metrics = json.ToString();
}
} else if (operation == RESET) {
std::string capacity = req->param(std::string(CAPACITY));
@@ -216,4 +224,4 @@ void FileCacheAction::handle(HttpRequest* req) {
}
}
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/test/service/http/file_cache_action_test.cpp
b/be/test/service/http/file_cache_action_test.cpp
index 4c51cb34453..066d07b25e3 100644
--- a/be/test/service/http/file_cache_action_test.cpp
+++ b/be/test/service/http/file_cache_action_test.cpp
@@ -24,6 +24,7 @@
#include <chrono>
#include <filesystem>
#include <memory>
+#include <string_view>
#include <thread>
#include "common/config.h"
@@ -32,6 +33,7 @@
#include "io/cache/file_cache_common.h"
#include "runtime/exec_env.h"
#include "service/http/http_request.h"
+#include "util/slice.h"
namespace doris {
@@ -65,6 +67,7 @@ public:
_action = std::make_unique<FileCacheAction>(nullptr);
ASSERT_NE(doris::ExecEnv::GetInstance()->file_cache_factory(),
nullptr);
+ reset_file_cache_factory();
_base_path = "/tmp/file_cache_action_ut_" + std::to_string(getpid());
(void)std::filesystem::create_directories(_base_path);
@@ -72,6 +75,8 @@ public:
settings.storage = "memory";
settings.capacity = 1024 * 1024; // 1MB
settings.max_file_block_size = 64 * 1024; // 64KB
+ settings.query_queue_size = 1024 * 1024;
+ settings.query_queue_elements = 16;
auto cache = std::make_unique<doris::io::BlockFileCache>(_base_path,
settings);
ASSERT_TRUE(cache->initialize());
for (int i = 0; i < 1000; ++i) {
@@ -82,6 +87,7 @@ public:
auto* factory = doris::io::FileCacheFactory::instance();
factory->_caches.emplace_back(std::move(cache));
factory->_path_to_cache[_base_path] = raw;
+ _cache = raw;
}
void TearDown() override {
@@ -94,7 +100,7 @@ public:
}
if (doris::ExecEnv::GetInstance()->file_cache_factory() != nullptr) {
- doris::io::FileCacheFactory::instance()->clear_file_caches(true);
+ reset_file_cache_factory();
}
}
@@ -103,8 +109,35 @@ protected:
event_base* _event_base = nullptr;
std::unique_ptr<FileCacheAction> _action;
std::string _base_path;
+ doris::io::BlockFileCache* _cache = nullptr;
inline static std::unique_ptr<doris::io::FileCacheFactory> _suite_factory;
+
+ void reset_file_cache_factory() {
+ auto* factory = doris::io::FileCacheFactory::instance();
+ factory->clear_file_caches(true);
+ factory->_caches.clear();
+ factory->_path_to_cache.clear();
+ factory->_capacity = 0;
+ _cache = nullptr;
+ }
+
+ void cache_file_block(const std::string& file_path) {
+ auto hash = doris::io::BlockFileCache::hash(file_path);
+ doris::io::CacheContext context;
+ doris::io::ReadStatistics stats;
+ context.cache_type = doris::io::FileCacheType::NORMAL;
+ context.stats = &stats;
+
+ auto holder = _cache->get_or_set(hash, 0, 4, context);
+ ASSERT_EQ(holder.file_blocks.size(), 1);
+ auto& block = holder.file_blocks.front();
+ ASSERT_EQ(block->get_or_set_downloader(),
doris::io::FileBlock::get_caller_id());
+ std::string data = "data";
+ ASSERT_TRUE(block->append(Slice(data.data(), data.size())).ok());
+ ASSERT_TRUE(block->finalize().ok());
+ ASSERT_EQ(_cache->_cur_cache_size, 4);
+ }
};
TEST_F(FileCacheActionTest, list_base_paths) {
@@ -160,4 +193,53 @@ TEST_F(FileCacheActionTest, check_consistency_ok) {
EXPECT_EQ(json_metrics, "null");
}
-} // namespace doris
\ No newline at end of file
+TEST_F(FileCacheActionTest, clear_defaults_to_async) {
+ cache_file_block("clear_defaults_to_async.dat");
+ HttpRequest req(_evhttp_req);
+ std::string json_metrics;
+
+ req._params["op"] = "clear";
+
+ Status status = _action->_handle_header(&req, &json_metrics);
+
+ EXPECT_TRUE(status.ok());
+ EXPECT_TRUE(json_metrics.empty());
+ EXPECT_EQ(_cache->_cur_cache_size, 0);
+}
+
+TEST_F(FileCacheActionTest, clear_sync_true_runs_async_and_warns) {
+ cache_file_block("clear_sync_true_runs_async_and_warns.dat");
+ HttpRequest req(_evhttp_req);
+ std::string json_metrics;
+
+ req._params["op"] = "clear";
+ req._params["sync"] = "true";
+
+ Status status = _action->_handle_header(&req, &json_metrics);
+
+ EXPECT_TRUE(status.ok());
+ EXPECT_EQ(json_metrics,
+ "{\"status\":\"OK\",\"msg\":\"sync clear_file_cache is no longer
supported in "
+ "http api, running async clear instead\"}");
+ EXPECT_EQ(_cache->_cur_cache_size, 0);
+}
+
+TEST_F(FileCacheActionTest, clear_value_uses_async_remove) {
+ constexpr std::string_view file_path = "clear_value_uses_async_remove.dat";
+ cache_file_block(std::string(file_path));
+ auto hash = doris::io::BlockFileCache::hash(std::string(file_path));
+ HttpRequest req(_evhttp_req);
+ std::string json_metrics;
+
+ req._params["op"] = "clear";
+ req._params["value"] = std::string(file_path);
+
+ Status status = _action->_handle_header(&req, &json_metrics);
+
+ EXPECT_TRUE(status.ok());
+ EXPECT_TRUE(json_metrics.empty());
+ EXPECT_EQ(_cache->_cur_cache_size, 0);
+ EXPECT_TRUE(_cache->get_blocks_by_key(hash).empty());
+}
+
+} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
