This is an automated email from the ASF dual-hosted git repository.
plat1ko pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 97850cf2bbf [fix](cooldown) Fix hdfs path (#33315)
97850cf2bbf is described below
commit 97850cf2bbf6d3fac993094e70731af5966f94f9
Author: plat1ko <platonekos...@gmail.com>
AuthorDate: Tue Apr 9 12:55:53 2024 +0800
[fix](cooldown) Fix hdfs path (#33315)
---
be/src/io/fs/hdfs_file_system.cpp | 7 ++++---
be/src/util/hdfs_util.cpp | 22 ++++++++++++----------
2 files changed, 16 insertions(+), 13 deletions(-)
diff --git a/be/src/io/fs/hdfs_file_system.cpp
b/be/src/io/fs/hdfs_file_system.cpp
index 5b5dbba16b2..22b4784ea61 100644
--- a/be/src/io/fs/hdfs_file_system.cpp
+++ b/be/src/io/fs/hdfs_file_system.cpp
@@ -283,11 +283,12 @@ Status HdfsFileSystem::list_impl(const Path& path, bool
only_file, std::vector<F
if (only_file && file.mKind == kObjectKindDirectory) {
continue;
}
- FileInfo file_info;
- file_info.file_name = file.mName;
+ auto& file_info = files->emplace_back();
+ std::string_view fname(file.mName);
+ fname.remove_prefix(fname.rfind('/') + 1);
+ file_info.file_name = fname;
file_info.file_size = file.mSize;
file_info.is_file = (file.mKind != kObjectKindDirectory);
- files->emplace_back(std::move(file_info));
}
hdfsFreeFileInfo(hdfs_file_info, numEntries);
return Status::OK();
diff --git a/be/src/util/hdfs_util.cpp b/be/src/util/hdfs_util.cpp
index 6e99fdea3d3..794c53f15e4 100644
--- a/be/src/util/hdfs_util.cpp
+++ b/be/src/util/hdfs_util.cpp
@@ -42,17 +42,19 @@ hdfsFS HDFSHandle::create_hdfs_fs(HDFSCommonBuilder&
hdfs_builder) {
}
Path convert_path(const Path& path, const std::string& namenode) {
- Path real_path(path);
- if (path.string().find(namenode) != std::string::npos) {
- std::string real_path_str = path.string().substr(namenode.size());
- if (!real_path_str.starts_with("/")) {
- // The real path must starts with "/"
- // Or the hadoop client will add a prefix like "/user/hadoop".
- real_path_str = "/" + real_path_str;
- }
- real_path = real_path_str;
+ std::string fs_path;
+ if (path.native().starts_with(namenode)) {
+ // `path` is URI format, remove the namenode part in `path`
+ fs_path = path.native().substr(namenode.size());
+ } else {
+ fs_path = path;
}
- return real_path;
+
+ // Always use absolute path (start with '/') in hdfs
+ if (fs_path.empty() || fs_path[0] != '/') {
+ fs_path.insert(fs_path.begin(), '/');
+ }
+ return fs_path;
}
bool is_hdfs(const std::string& path_or_fs) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
