(doris) branch master updated: [bug](s3) Fix object data is overwritten by empty object Fix wrong empty s3 file (#32252)

[dataroaring]

This is an automated email from the ASF dual-hosted git repository.

dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new 8fdd809a7b8 [bug](s3) Fix object data is overwritten by empty object 
Fix wrong empty s3 file (#32252)
8fdd809a7b8 is described below

commit 8fdd809a7b8a9e8ee7cd2aac618646104f846266
Author: plat1ko <platonekos...@gmail.com>
AuthorDate: Thu Mar 14 21:55:21 2024 +0800

    [bug](s3) Fix object data is overwritten by empty object Fix wrong empty s3 
file (#32252)
---
 be/src/io/fs/err_utils.cpp      |  8 ++++----
 be/src/io/fs/s3_file_writer.cpp | 11 +++++------
 be/src/io/fs/s3_file_writer.h   |  1 -
 3 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/be/src/io/fs/err_utils.cpp b/be/src/io/fs/err_utils.cpp
index 5688b888f44..35f4b79a304 100644
--- a/be/src/io/fs/err_utils.cpp
+++ b/be/src/io/fs/err_utils.cpp
@@ -89,7 +89,7 @@ Status localfs_error(const std::error_code& ec, 
std::string_view msg) {
     } else if (ec == std::errc::permission_denied) {
         return Status::Error<PERMISSION_DENIED, false>(msg);
     } else {
-        return Status::Error<doris::INTERNAL_ERROR, false>("{}: {}", msg, 
ec.message());
+        return Status::Error<ErrorCode::INTERNAL_ERROR, false>("{}: {}", msg, 
ec.message());
     }
 }
 
@@ -106,8 +106,8 @@ Status localfs_error(int posix_errno, std::string_view msg) 
{
     case EACCES:
         return Status::Error<PERMISSION_DENIED, false>(msg);
     default:
-        return Status::Error<doris::INTERNAL_ERROR, false>("{}: {}", msg,
-                                                           
std::strerror(posix_errno));
+        return Status::Error<ErrorCode::INTERNAL_ERROR, false>("{}: {}", msg,
+                                                               
std::strerror(posix_errno));
     }
 }
 
@@ -122,7 +122,7 @@ Status s3fs_error(const Aws::S3::S3Error& err, 
std::string_view msg) {
                                                        err.GetExceptionName(), 
err.GetMessage(),
                                                        err.GetErrorType());
     default:
-        return Status::Error<doris::INTERNAL_ERROR, false>(
+        return Status::Error<ErrorCode::INTERNAL_ERROR, false>(
                 "{}: {} {} code={} type={}", msg, err.GetExceptionName(), 
err.GetMessage(),
                 err.GetResponseCode(), err.GetErrorType());
     }
diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp
index dbe5ce8e70c..5286fbf4a52 100644
--- a/be/src/io/fs/s3_file_writer.cpp
+++ b/be/src/io/fs/s3_file_writer.cpp
@@ -107,9 +107,8 @@ S3FileWriter::~S3FileWriter() {
         // if we don't abort multi part upload, the uploaded part in object
         // store will not automatically reclaim itself, it would cost more 
money
         static_cast<void>(_abort());
-        _bytes_written = 0;
     }
-    s3_bytes_written_total << _bytes_written;
+    s3_bytes_written_total << _bytes_appended;
     s3_file_being_written << -1;
 }
 
@@ -207,8 +206,10 @@ Status S3FileWriter::close() {
             auto* buf = dynamic_cast<UploadFileBuffer*>(_pending_buf.get());
             DCHECK(buf != nullptr);
             buf->set_upload_to_remote([this](UploadFileBuffer& b) { 
_put_object(b); });
-        } else if (_create_empty_file) {
-            // if there is no pending buffer, we need to create an empty file
+        }
+
+        if (_bytes_appended == 0 && _create_empty_file) {
+            // No data written, but need to create an empty file
             auto builder = FileBufferBuilder();
             builder.set_type(BufferType::UPLOAD)
                     .set_upload_callback([this](UploadFileBuffer& buf) { 
_put_object(buf); })
@@ -390,7 +391,6 @@ void S3FileWriter::_upload_one_part(int64_t part_num, 
UploadFileBuffer& buf) {
 
     std::unique_lock<std::mutex> lck {_completed_lock};
     _completed_parts.emplace_back(std::move(completed_part));
-    _bytes_written += buf.get_size();
 }
 
 Status S3FileWriter::_complete() {
@@ -501,7 +501,6 @@ void S3FileWriter::_put_object(UploadFileBuffer& buf) {
         buf.set_status(_st);
         return;
     }
-    _bytes_written += buf.get_size();
     s3_file_created_total << 1;
 }
 
diff --git a/be/src/io/fs/s3_file_writer.h b/be/src/io/fs/s3_file_writer.h
index 8774362f817..fc1d85b345a 100644
--- a/be/src/io/fs/s3_file_writer.h
+++ b/be/src/io/fs/s3_file_writer.h
@@ -79,7 +79,6 @@ private:
 
     std::atomic_bool _failed = false;
     Status _st;
-    size_t _bytes_written = 0;
 
     std::shared_ptr<FileBuffer> _pending_buf;
     int64_t _expiration_time;


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org