This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new 7ffcd73301 [bugfix] Recycle the fds when remove stale rowset or drop
table (#24115)
7ffcd73301 is described below
commit 7ffcd733016195124cd8f184e8561086bdc3dc88
Author: Lightman <31928846+lchangli...@users.noreply.github.com>
AuthorDate: Sat Sep 9 01:00:56 2023 +0800
[bugfix] Recycle the fds when remove stale rowset or drop table (#24115)
Associated with #23081
---
be/src/olap/olap_server.cpp | 3 +--
be/src/olap/rowset/beta_rowset.cpp | 2 ++
be/src/olap/segment_loader.cpp | 4 ++++
be/src/olap/segment_loader.h | 2 ++
be/src/olap/tablet.h | 6 ++++++
be/src/olap/tablet_manager.cpp | 13 +++++++++++++
6 files changed, 28 insertions(+), 2 deletions(-)
diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index 848539d513..6ce4dec3a6 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -170,9 +170,8 @@ void StorageEngine::_fd_cache_clean_callback() {
#ifdef GOOGLE_PROFILER
ProfilerRegisterThread();
#endif
- int32_t interval = 600;
+ int32_t interval = config::cache_clean_interval;
while
(!_stop_background_threads_latch.wait_for(std::chrono::seconds(interval))) {
- interval = config::cache_clean_interval;
if (interval <= 0) {
LOG(WARNING) << "config of file descriptor clean interval is
illegal: [" << interval
<< "], force set to 3600 ";
diff --git a/be/src/olap/rowset/beta_rowset.cpp
b/be/src/olap/rowset/beta_rowset.cpp
index 8f0bda1bc6..1811e9627f 100644
--- a/be/src/olap/rowset/beta_rowset.cpp
+++ b/be/src/olap/rowset/beta_rowset.cpp
@@ -192,6 +192,8 @@ Status BetaRowset::remove() {
if (!fs) {
return Status::Error<INIT_FAILED>();
}
+ // If the rowset was removed, it need to remove the fds in segment cache
directly
+
SegmentLoader::instance()->erase_segments(SegmentLoader::CacheKey(rowset_id()));
bool success = true;
Status st;
for (int i = 0; i < num_segments(); ++i) {
diff --git a/be/src/olap/segment_loader.cpp b/be/src/olap/segment_loader.cpp
index fa1e6c8751..5c7a81ed44 100644
--- a/be/src/olap/segment_loader.cpp
+++ b/be/src/olap/segment_loader.cpp
@@ -104,4 +104,8 @@ Status SegmentLoader::prune() {
return Status::OK();
}
+void SegmentLoader::erase_segments(const SegmentLoader::CacheKey& key) {
+ _cache->erase(key.encode());
+}
+
} // namespace doris
diff --git a/be/src/olap/segment_loader.h b/be/src/olap/segment_loader.h
index 03a54acf55..c0ece39a65 100644
--- a/be/src/olap/segment_loader.h
+++ b/be/src/olap/segment_loader.h
@@ -96,6 +96,8 @@ public:
: ((double)_cache->get_usage() /
_cache->get_total_capacity());
}
+ void erase_segments(const SegmentLoader::CacheKey& key);
+
private:
SegmentLoader();
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index e81f1212e7..ce7bb1cc44 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -385,6 +385,12 @@ public:
int64_t get_table_id() { return _tablet_meta->table_id(); }
+ // MUST hold SHARED `_meta_lock`
+ const auto& rowset_map() const { return _rs_version_map; }
+
+ // MUST hold SHARED `_meta_lock`
+ const auto& stale_rowset_map() const { return _stale_rs_version_map; }
+
private:
Status _init_once_action();
void _print_missed_versions(const std::vector<Version>& missed_versions)
const;
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 5f2bd1b5c0..a523ec620d 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -42,6 +42,7 @@
#include "olap/reader.h"
#include "olap/rowset/rowset_id_generator.h"
#include "olap/schema_change.h"
+#include "olap/segment_loader.h"
#include "olap/tablet.h"
#include "olap/tablet_meta.h"
#include "olap/tablet_meta_manager.h"
@@ -470,6 +471,18 @@ Status TabletManager::_drop_tablet_unlocked(TTabletId
tablet_id, TReplicaId repl
_remove_tablet_from_partition(to_drop_tablet);
tablet_map_t& tablet_map = _get_tablet_map(tablet_id);
tablet_map.erase(tablet_id);
+ {
+ std::shared_lock rlock(to_drop_tablet->get_header_lock());
+ static auto recycle_segment_cache = [](const auto& rowset_map) {
+ for (auto& [_, rowset] : rowset_map) {
+ // If the tablet was deleted, it need to remove all rowsets
fds directly
+ SegmentLoader::instance()->erase_segments(
+ SegmentLoader::CacheKey(rowset->rowset_id()));
+ }
+ };
+ recycle_segment_cache(to_drop_tablet->rowset_map());
+ recycle_segment_cache(to_drop_tablet->stale_rowset_map());
+ }
if (!keep_files) {
// drop tablet will update tablet meta, should lock
std::lock_guard<std::shared_mutex>
wrlock(to_drop_tablet->get_header_lock());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
