This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new b4f00be528b branch-2.1: [fix](restore) Make the DirMoveTask
idempotent. #47313 (#47584)
b4f00be528b is described below
commit b4f00be528bd039de570f73d50073939024524aa
Author: walter <maoch...@selectdb.com>
AuthorDate: Sat Feb 8 14:28:38 2025 +0800
branch-2.1: [fix](restore) Make the DirMoveTask idempotent. #47313 (#47584)
cherry pick from #47313
---
be/src/runtime/snapshot_loader.cpp | 27 ++++++++++++++++++++++++---
1 file changed, 24 insertions(+), 3 deletions(-)
diff --git a/be/src/runtime/snapshot_loader.cpp
b/be/src/runtime/snapshot_loader.cpp
index 2a6ba9274e2..639701f068c 100644
--- a/be/src/runtime/snapshot_loader.cpp
+++ b/be/src/runtime/snapshot_loader.cpp
@@ -43,6 +43,7 @@
#include "http/http_client.h"
#include "io/fs/broker_file_system.h"
#include "io/fs/file_system.h"
+#include "io/fs/file_writer.h"
#include "io/fs/hdfs_file_system.h"
#include "io/fs/local_file_system.h"
#include "io/fs/path.h"
@@ -61,7 +62,17 @@
#include "util/thrift_rpc_helper.h"
namespace doris {
-namespace {
+
+static std::string get_loaded_tag_path(const std::string& snapshot_path) {
+ return snapshot_path + "/LOADED";
+}
+
+static Status write_loaded_tag(const std::string& snapshot_path, int64_t
tablet_id) {
+ std::unique_ptr<io::FileWriter> writer;
+ std::string file = get_loaded_tag_path(snapshot_path);
+ RETURN_IF_ERROR(io::global_local_filesystem()->create_file(file, &writer));
+ return writer->close();
+}
Status upload_with_checksum(io::RemoteFileSystem& fs, std::string_view
local_path,
std::string_view remote_path, std::string_view
checksum) {
@@ -83,8 +94,6 @@ Status upload_with_checksum(io::RemoteFileSystem& fs,
std::string_view local_pat
return Status::OK();
}
-} // namespace
-
SnapshotLoader::SnapshotLoader(ExecEnv* env, int64_t job_id, int64_t task_id)
: _env(env),
_job_id(job_id),
@@ -756,6 +765,14 @@ Status SnapshotLoader::move(const std::string&
snapshot_path, TabletSharedPtr ta
return Status::InternalError(ss.str());
}
+ std::string loaded_tag_path = get_loaded_tag_path(snapshot_path);
+ bool already_loaded = false;
+ RETURN_IF_ERROR(io::global_local_filesystem()->exists(loaded_tag_path,
&already_loaded));
+ if (already_loaded) {
+ LOG(INFO) << "snapshot path already moved: " << snapshot_path;
+ return Status::OK();
+ }
+
// rename the rowset ids and tabletid info in rowset meta
auto res = SnapshotManager::instance()->convert_rowset_ids(
snapshot_path, tablet_id, tablet->replica_id(),
tablet->partition_id(), schema_hash);
@@ -824,6 +841,10 @@ Status SnapshotLoader::move(const std::string&
snapshot_path, TabletSharedPtr ta
LOG(WARNING) << ss.str();
return Status::InternalError(ss.str());
}
+
+ // mark the snapshot path as loaded
+ RETURN_IF_ERROR(write_loaded_tag(snapshot_path, tablet_id));
+
LOG(INFO) << "finished to reload header of tablet: " << tablet_id;
return status;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
