w41ter commented on code in PR #49233:
URL: https://github.com/apache/doris/pull/49233#discussion_r2011481510
##########
be/src/runtime/snapshot_loader.cpp:
##########
@@ -479,6 +487,73 @@ Status SnapshotHttpDownloader::_link_same_rowset_files() {
}
}
+ for (const auto& remote_rowset_meta : remote_tablet_meta.rs_metas()) {
+ if (remote_rowset_meta.has_resource_id() ||
!remote_rowset_meta.has_source_rowset_id()) {
+ continue;
+ }
+
+ auto local_rowset_meta =
local_rowset_metas.find(remote_rowset_meta.source_rowset_id());
+ if (local_rowset_meta == local_rowset_metas.end()) {
+ continue;
+ }
+
+ const auto& local_rowset_id = local_rowset_meta->first;
+ const auto& local_rowset_meta_pb = local_rowset_meta->second;
+ const auto& remote_rowset_id = remote_rowset_meta.rowset_id_v2();
+ auto local_tablet_id = local_rowset_meta_pb.tablet_id();
+
+ if (remote_rowset_meta.start_version() !=
local_rowset_meta_pb.start_version() ||
+ remote_rowset_meta.end_version() !=
local_rowset_meta_pb.end_version()) {
+ continue;
+ }
+
+ LOG(INFO) << "remote rowset " << remote_rowset_id << " was derived
from local tablet "
+ << local_tablet_id << " rowset " << local_rowset_id
+ << ", skip downloading these files";
+
+ for (const auto& remote_file : _remote_file_list) {
+ if (!remote_file.starts_with(remote_rowset_id)) {
+ continue;
+ }
+
+ std::string local_file = remote_file;
+ local_file.replace(0, remote_rowset_id.size(), local_rowset_id);
+ std::string local_file_path = _local_path + "/" + local_file;
+ std::string remote_file_path = _local_path + "/" + remote_file;
+
+ bool exist = false;
+
RETURN_IF_ERROR(io::global_local_filesystem()->exists(local_file_path, &exist));
+ if (!exist) {
+ continue;
+ }
+
+ LOG(INFO) << "link file from " << local_file_path << " to " <<
remote_file_path;
+ if (!io::global_local_filesystem()->link_file(local_file_path,
remote_file_path)) {
+ std::string msg = fmt::format("link file failed from {} to {},
err: {}",
+ local_file_path,
remote_file_path, strerror(errno));
+ LOG(WARNING) << msg;
+ return Status::InternalError(std::move(msg));
+ } else {
+ auto it = _local_files.find(local_file);
Review Comment:
The local file stats must exist so that you can treat the not-found event as
an error.
##########
be/src/runtime/snapshot_loader.cpp:
##########
@@ -479,6 +487,73 @@ Status SnapshotHttpDownloader::_link_same_rowset_files() {
}
}
+ for (const auto& remote_rowset_meta : remote_tablet_meta.rs_metas()) {
+ if (remote_rowset_meta.has_resource_id() ||
!remote_rowset_meta.has_source_rowset_id()) {
+ continue;
+ }
+
+ auto local_rowset_meta =
local_rowset_metas.find(remote_rowset_meta.source_rowset_id());
+ if (local_rowset_meta == local_rowset_metas.end()) {
+ continue;
+ }
+
+ const auto& local_rowset_id = local_rowset_meta->first;
+ const auto& local_rowset_meta_pb = local_rowset_meta->second;
+ const auto& remote_rowset_id = remote_rowset_meta.rowset_id_v2();
+ auto local_tablet_id = local_rowset_meta_pb.tablet_id();
+
+ if (remote_rowset_meta.start_version() !=
local_rowset_meta_pb.start_version() ||
+ remote_rowset_meta.end_version() !=
local_rowset_meta_pb.end_version()) {
+ continue;
+ }
+
+ LOG(INFO) << "remote rowset " << remote_rowset_id << " was derived
from local tablet "
+ << local_tablet_id << " rowset " << local_rowset_id
+ << ", skip downloading these files";
+
+ for (const auto& remote_file : _remote_file_list) {
+ if (!remote_file.starts_with(remote_rowset_id)) {
+ continue;
+ }
+
+ std::string local_file = remote_file;
+ local_file.replace(0, remote_rowset_id.size(), local_rowset_id);
+ std::string local_file_path = _local_path + "/" + local_file;
+ std::string remote_file_path = _local_path + "/" + remote_file;
+
+ bool exist = false;
+
RETURN_IF_ERROR(io::global_local_filesystem()->exists(local_file_path, &exist));
Review Comment:
```suggestion
RETURN_IF_ERROR(io::global_local_filesystem()->exists(remote_file_path,
&exist));
```
skip if the remote file already exists.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
