This is an automated email from the ASF dual-hosted git repository.
wyxxxcat pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d9d60bee9fe [fix](cloud) Add configurable GCS object deletion (#70119)
d9d60bee9fe is described below
commit d9d60bee9fea41273206052e7eba9b331c7f9959
Author: Yixuan Wang <[email protected]>
AuthorDate: Fri Nov 6 11:46:49 2026 +0800
[fix](cloud) Add configurable GCS object deletion (#70119)
### What problem does this PR solve?
Issue Number: None
Related PR: None
Problem Summary: Recycler previously used the generic S3 accessor for
GCS. Add a GCS accessor that deletes objects individually for file lists
and prefixes while preserving expiration filtering and missing-object
handling.
The enable_gcs_batch_delete_object startup configuration selects the GCS
accessor and is enabled by default. Disabling it retains S3Accessor.
### Release note
GCS recycling uses individual-object deletion by default. Set
enable_gcs_batch_delete_object=false to retain the generic S3 accessor.
### Check List (For Author)
- Test: Not run, per user instruction; no build, formatting checks, or
unit tests
- Behavior changed: Yes, GCS uses GcsAccessor by default with an opt-out
configuration
- Does this need documentation: No
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
cloud/src/common/config.h | 2 +
cloud/src/recycler/s3_accessor.cpp | 88 +++++++++++++++++++++++++++++++++++++-
cloud/src/recycler/s3_accessor.h | 11 +++++
3 files changed, 100 insertions(+), 1 deletion(-)
diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
index 949f4633960..92282d1b4c0 100644
--- a/cloud/src/common/config.h
+++ b/cloud/src/common/config.h
@@ -326,6 +326,8 @@ CONF_String(priority_networks, "");
CONF_Bool(enable_cluster_name_check, "false");
+CONF_Bool(enable_gcs_batch_delete_object, "true");
+
// Default HTTP scheme used by S3Client when the endpoint has no scheme.
CONF_String(s3_client_http_scheme, "https");
CONF_Validator(s3_client_http_scheme, [](const std::string& config) -> bool {
diff --git a/cloud/src/recycler/s3_accessor.cpp
b/cloud/src/recycler/s3_accessor.cpp
index 0413044d3fa..b84a2e4c64a 100644
--- a/cloud/src/recycler/s3_accessor.cpp
+++ b/cloud/src/recycler/s3_accessor.cpp
@@ -329,7 +329,17 @@ std::string S3Accessor::to_uri(const std::string&
relative_path) const {
int S3Accessor::create(S3Conf conf, std::shared_ptr<S3Accessor>* accessor) {
TEST_SYNC_POINT_RETURN_WITH_VALUE("S3Accessor::init.s3_init_failed",
(int)-1);
- *accessor = std::make_shared<S3Accessor>(conf);
+ switch (conf.provider) {
+ case S3Conf::GCS:
+ if (!config::enable_gcs_batch_delete_object) {
+ *accessor = std::make_shared<GcsAccessor>(conf);
+ break;
+ }
+ default:
+ *accessor = std::make_shared<S3Accessor>(conf);
+ break;
+ }
+
return (*accessor)->init();
}
@@ -613,4 +623,80 @@ int S3Accessor::check_versioning() {
return obj_client_->check_versioning(conf_.bucket).status.code;
}
+int GcsAccessor::delete_prefix_impl(const std::string& path_prefix, int64_t
expiration_time) {
+ LOG_INFO("begin delete prefix").tag("uri", to_uri(path_prefix));
+
+ int ret = 0;
+ int cnt = 0;
+ int skip = 0;
+ int64_t del_nonexisted = 0;
+ int del = 0;
+ auto iter = obj_client_->list_objects({.bucket = conf_.bucket, .prefix =
get_key(path_prefix)});
+ for (;;) {
+ auto result = iter->next();
+ if (!result.object.has_value()) {
+ if (result.resp.status.code != ObjStorageStatus::NOT_FOUND &&
+ result.resp.status.code != ObjStorageStatus::OK) {
+ ret = result.resp.status.code;
+ }
+ break;
+ }
+ auto& obj = *result.object;
+ if (!(++cnt % 100)) {
+ LOG_INFO("loop delete prefix")
+ .tag("uri", to_uri(path_prefix))
+ .tag("total_obj_cnt", cnt)
+ .tag("deleted", del)
+ .tag("del_nonexisted", del_nonexisted)
+ .tag("skipped", skip);
+ }
+ if (expiration_time > 0 && obj.mtime_s > expiration_time) {
+ skip++;
+ continue;
+ }
+ del++;
+
+ // FIXME(plat1ko): Delete objects by batch with genuine GCS client
+ int del_ret =
+ obj_client_->delete_object({.bucket = conf_.bucket, .key =
obj.key}).status.code;
+ del_nonexisted += (del_ret == ObjStorageStatus::NOT_FOUND);
+ static_assert(ObjStorageStatus::OK == 0);
+ if (del_ret != ObjStorageStatus::OK && del_ret !=
ObjStorageStatus::NOT_FOUND) {
+ ret = del_ret;
+ }
+ }
+
+ LOG_INFO("finish delete prefix")
+ .tag("uri", to_uri(path_prefix))
+ .tag("total_obj_cnt", cnt)
+ .tag("deleted", del)
+ .tag("del_nonexisted", del_nonexisted)
+ .tag("skipped", skip);
+
+ if (!iter->is_valid()) {
+ return -1;
+ }
+
+ return ret;
+}
+
+int GcsAccessor::delete_files(const std::vector<std::string>& paths) {
+ std::vector<int> delete_rets(paths.size());
+#ifdef USE_LIBCPP
+ std::transform(paths.begin(), paths.end(), delete_rets.begin(),
+#else
+ std::transform(std::execution::par, paths.begin(), paths.end(),
delete_rets.begin(),
+#endif
+ [this](const std::string& path) { return delete_file(path);
});
+
+ int ret = 0;
+ for (int delete_ret : delete_rets) {
+ if (delete_ret != 0) {
+ ret = delete_ret;
+ break;
+ }
+ }
+ return ret;
+}
+
} // namespace doris::cloud
diff --git a/cloud/src/recycler/s3_accessor.h b/cloud/src/recycler/s3_accessor.h
index 5a0b48628a0..adee55e2195 100644
--- a/cloud/src/recycler/s3_accessor.h
+++ b/cloud/src/recycler/s3_accessor.h
@@ -160,5 +160,16 @@ protected:
std::string _ca_cert_file_path;
};
+class GcsAccessor final : public S3Accessor {
+public:
+ explicit GcsAccessor(S3Conf conf) : S3Accessor(std::move(conf)) {}
+ ~GcsAccessor() override = default;
+
+ int delete_files(const std::vector<std::string>& paths) override;
+
+private:
+ int delete_prefix_impl(const std::string& path_prefix, int64_t
expiration_time = 0) override;
+};
+
} // namespace cloud
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
