This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d7c9e5ec3a0 [fix](file writer) fix value of s3 bytes written bvar
incorrect (#56262)
d7c9e5ec3a0 is described below
commit d7c9e5ec3a044c0421e02dfb30ba75b5d741c116
Author: hui lai <[email protected]>
AuthorDate: Mon Sep 22 12:17:35 2025 +0800
[fix](file writer) fix value of s3 bytes written bvar incorrect (#56262)
### What problem does this PR solve?
Upload small files efficiently using single PUT Object instead of
multipart upload, but it missing record s3 bytes written bvar.
---
be/src/io/fs/s3_file_writer.cpp | 1 +
.../test_s3_bytes_written_metrics.groovy | 80 ++++++++++++++++++++++
2 files changed, 81 insertions(+)
diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp
index 309d4392f6d..79e19967e77 100644
--- a/be/src/io/fs/s3_file_writer.cpp
+++ b/be/src/io/fs/s3_file_writer.cpp
@@ -480,6 +480,7 @@ void S3FileWriter::_put_object(UploadFileBuffer& buf) {
<< " size=" << _bytes_appended << " time=" <<
timer.elapsed_time_milliseconds()
<< "ms";
s3_file_created_total << 1;
+ s3_bytes_written_total << buf.get_size();
}
std::string S3FileWriter::_dump_completed_part() const {
diff --git
a/regression-test/suites/load_p0/stream_load/test_s3_bytes_written_metrics.groovy
b/regression-test/suites/load_p0/stream_load/test_s3_bytes_written_metrics.groovy
new file mode 100644
index 00000000000..88d5461a7ae
--- /dev/null
+++
b/regression-test/suites/load_p0/stream_load/test_s3_bytes_written_metrics.groovy
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_s3_bytes_written_metrics", "p0") {
+ if (!context.config.isCloudMode()) {
+ return
+ }
+
+ def tableName = "test_s3_bytes_written_metrics"
+ def getBrpcMetrics = {ip, port, name ->
+ def url = "http://${ip}:${port}/brpc_metrics";
+ def metrics = new URL(url).text
+ def matcher = metrics =~ ~"${name}\\s+(\\d+)"
+ if (matcher.find()) {
+ def ret = matcher[0][1] as long
+ logger.info("getBrpcMetrics, ${url}, name:${name}, value:${ret}")
+ return ret
+ } else {
+ throw new RuntimeException("${name} not found for ${ip}:${port}")
+ }
+ }
+
+ def getTotalS3BytesWritten = {
+ def backends = sql """SHOW BACKENDS"""
+ def totalBytes = 0L
+ for (def backend : backends) {
+ def ip = backend[1]
+ def httpPort = backend[5]
+ def bytes = getBrpcMetrics(ip, httpPort,
"s3_file_writer_bytes_written")
+ totalBytes += bytes
+ }
+ return totalBytes
+ }
+
+ def initialBytes = getTotalS3BytesWritten()
+ logger.info("before load s3_file_writer_bytes_written: ${initialBytes}")
+
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `k1` int(20) NULL,
+ `k2` string NULL,
+ `v1` date NULL,
+ `v2` string NULL,
+ `v3` datetime NULL,
+ `v4` string NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`k1`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 3
+ PROPERTIES ("replication_allocation" = "tag.location.default: 1");
+ """
+
+ streamLoad {
+ table "${tableName}"
+ set 'column_separator', ','
+
+ file "empty_field_as_null.csv"
+ }
+
+ def afterLoadBytes = getTotalS3BytesWritten()
+ def loadBytes = afterLoadBytes - initialBytes
+ logger.info("after load s3_file_writer_bytes_written: ${afterLoadBytes},
written: ${loadBytes}")
+ assertTrue(loadBytes > 0, "s3_file_writer_bytes_written should increase")
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
