wyxxxcat commented on code in PR #70119:
URL: https://github.com/apache/doris/pull/70119#discussion_r4488810252
##########
cloud/src/recycler/s3_accessor.cpp:
##########
@@ -613,4 +623,80 @@ int S3Accessor::check_versioning() {
return obj_client_->check_versioning(conf_.bucket).status.code;
}
+int GcsAccessor::delete_prefix_impl(const std::string& path_prefix, int64_t
expiration_time) {
+ LOG_INFO("begin delete prefix").tag("uri", to_uri(path_prefix));
+
+ int ret = 0;
+ int cnt = 0;
+ int skip = 0;
+ int64_t del_nonexisted = 0;
+ int del = 0;
+ auto iter = obj_client_->list_objects({.bucket = conf_.bucket, .prefix =
get_key(path_prefix)});
+ for (;;) {
+ auto result = iter->next();
+ if (!result.object.has_value()) {
+ if (result.resp.status.code != ObjStorageStatus::NOT_FOUND &&
+ result.resp.status.code != ObjStorageStatus::OK) {
+ ret = result.resp.status.code;
+ }
+ break;
+ }
+ auto& obj = *result.object;
+ if (!(++cnt % 100)) {
+ LOG_INFO("loop delete prefix")
+ .tag("uri", to_uri(path_prefix))
+ .tag("total_obj_cnt", cnt)
+ .tag("deleted", del)
+ .tag("del_nonexisted", del_nonexisted)
+ .tag("skipped", skip);
+ }
+ if (expiration_time > 0 && obj.mtime_s > expiration_time) {
+ skip++;
+ continue;
+ }
+ del++;
+
+ // FIXME(plat1ko): Delete objects by batch with genuine GCS client
+ int del_ret =
+ obj_client_->delete_object({.bucket = conf_.bucket, .key =
obj.key}).status.code;
+ del_nonexisted += (del_ret == ObjStorageStatus::NOT_FOUND);
+ static_assert(ObjStorageStatus::OK == 0);
+ if (del_ret != ObjStorageStatus::OK && del_ret !=
ObjStorageStatus::NOT_FOUND) {
+ ret = del_ret;
+ }
+ }
+
+ LOG_INFO("finish delete prefix")
+ .tag("uri", to_uri(path_prefix))
+ .tag("total_obj_cnt", cnt)
+ .tag("deleted", del)
+ .tag("del_nonexisted", del_nonexisted)
+ .tag("skipped", skip);
+
+ if (!iter->is_valid()) {
+ return -1;
+ }
+
+ return ret;
+}
+
+int GcsAccessor::delete_files(const std::vector<std::string>& paths) {
+ std::vector<int> delete_rets(paths.size());
+#ifdef USE_LIBCPP
+ std::transform(paths.begin(), paths.end(), delete_rets.begin(),
+#else
+ std::transform(std::execution::par, paths.begin(), paths.end(),
delete_rets.begin(),
+#endif
+ [this](const std::string& path) { return delete_file(path);
});
+
+ int ret = 0;
+ for (int delete_ret : delete_rets) {
+ if (delete_ret != 0) {
+ ret = delete_ret;
Review Comment:
An error is only reported when the bucket does not exist; this is reasonable.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
