github-actions[bot] commented on code in PR #31214:
URL: https://github.com/apache/doris/pull/31214#discussion_r1500289914
##########
be/src/io/fs/s3_file_system.cpp:
##########
@@ -312,7 +312,73 @@ Status S3FileSystem::file_size_impl(const Path& file,
int64_t* file_size) const
return Status::OK();
}
-Status S3FileSystem::list_impl(const Path& dir, bool only_file,
std::vector<FileInfo>* files,
+class S3FileListIterator final : public FileListIterator {
+public:
+ S3FileListIterator(std::shared_ptr<Aws::S3::S3Client> client,
+ Aws::S3::Model::ListObjectsV2Request request, bool
only_file,
+ std::string_view prefix, std::string full_path)
+ : client(std::move(client)),
+ request(std::move(request)),
+ only_file(only_file),
+ prefix(prefix),
+ full_path(std::move(full_path)) {}
+ ~S3FileListIterator() override = default;
+
+ bool has_next() const override {
+ return iter != outcome.GetResult().GetContents().end() || is_trucated;
+ }
+ Result<FileInfo> next() override {
+ FileInfo file_info;
+ while (true) {
+ for (; iter != outcome.GetResult().GetContents().end(); iter++) {
+ const auto& obj = *iter;
+ std::string key = obj.GetKey();
+ bool is_dir = (key.back() == '/');
+ if (only_file && is_dir) {
+ continue;
+ }
+ file_info.file_name = obj.GetKey().substr(prefix.size());
+ file_info.file_size = obj.GetSize();
+ file_info.is_file = !is_dir;
+ iter++;
+ return file_info;
+ }
+ // Return early to prevent one useless S3 io
+ if (!is_trucated) {
+ break;
+ }
+ if (auto st = get_list_outcome(); !st.ok()) [[unlikely]] {
+ return ResultError(std::move(st));
+ }
+ }
+ return ResultError(Status::InternalError("Out of elements"));
+ }
+
+ Status get_list_outcome() {
Review Comment:
warning: method 'get_list_outcome' can be made static
[readability-convert-member-functions-to-static]
```suggestion
static Status get_list_outcome() {
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
