This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 909711db6c1 branch-3.1: [fix](file writer) fix value of s3 bytes
written bvar incorrect #56262 (#56598)
909711db6c1 is described below
commit 909711db6c15ddc59b51317c0785245a026f23b3
Author: hui lai <[email protected]>
AuthorDate: Mon Sep 29 18:10:47 2025 +0800
branch-3.1: [fix](file writer) fix value of s3 bytes written bvar incorrect
#56262 (#56598)
picked from #56262
---
be/src/io/fs/s3_file_writer.cpp | 1 +
.../load_p0/stream_load/empty_field_as_null.csv | 2 +
.../test_s3_bytes_written_metrics.groovy | 80 ++++++++++++++++++++++
3 files changed, 83 insertions(+)
diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp
index 756f83c78da..d4c842ce969 100644
--- a/be/src/io/fs/s3_file_writer.cpp
+++ b/be/src/io/fs/s3_file_writer.cpp
@@ -471,6 +471,7 @@ void S3FileWriter::_put_object(UploadFileBuffer& buf) {
}
s3_file_created_total << 1;
+ s3_bytes_written_total << buf.get_size();
}
std::string S3FileWriter::_dump_completed_part() const {
diff --git a/regression-test/data/load_p0/stream_load/empty_field_as_null.csv
b/regression-test/data/load_p0/stream_load/empty_field_as_null.csv
new file mode 100644
index 00000000000..38a59c4335e
--- /dev/null
+++ b/regression-test/data/load_p0/stream_load/empty_field_as_null.csv
@@ -0,0 +1,2 @@
+9,\N,2023-07-15,def,2023-07-20T05:48:31,ghi
+10,,2023-07-15,def,2023-07-20T05:48:31,ghi
\ No newline at end of file
diff --git
a/regression-test/suites/load_p0/stream_load/test_s3_bytes_written_metrics.groovy
b/regression-test/suites/load_p0/stream_load/test_s3_bytes_written_metrics.groovy
new file mode 100644
index 00000000000..88d5461a7ae
--- /dev/null
+++
b/regression-test/suites/load_p0/stream_load/test_s3_bytes_written_metrics.groovy
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_s3_bytes_written_metrics", "p0") {
+ if (!context.config.isCloudMode()) {
+ return
+ }
+
+ def tableName = "test_s3_bytes_written_metrics"
+ def getBrpcMetrics = {ip, port, name ->
+ def url = "http://${ip}:${port}/brpc_metrics";
+ def metrics = new URL(url).text
+ def matcher = metrics =~ ~"${name}\\s+(\\d+)"
+ if (matcher.find()) {
+ def ret = matcher[0][1] as long
+ logger.info("getBrpcMetrics, ${url}, name:${name}, value:${ret}")
+ return ret
+ } else {
+ throw new RuntimeException("${name} not found for ${ip}:${port}")
+ }
+ }
+
+ def getTotalS3BytesWritten = {
+ def backends = sql """SHOW BACKENDS"""
+ def totalBytes = 0L
+ for (def backend : backends) {
+ def ip = backend[1]
+ def httpPort = backend[5]
+ def bytes = getBrpcMetrics(ip, httpPort,
"s3_file_writer_bytes_written")
+ totalBytes += bytes
+ }
+ return totalBytes
+ }
+
+ def initialBytes = getTotalS3BytesWritten()
+ logger.info("before load s3_file_writer_bytes_written: ${initialBytes}")
+
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `k1` int(20) NULL,
+ `k2` string NULL,
+ `v1` date NULL,
+ `v2` string NULL,
+ `v3` datetime NULL,
+ `v4` string NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`k1`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 3
+ PROPERTIES ("replication_allocation" = "tag.location.default: 1");
+ """
+
+ streamLoad {
+ table "${tableName}"
+ set 'column_separator', ','
+
+ file "empty_field_as_null.csv"
+ }
+
+ def afterLoadBytes = getTotalS3BytesWritten()
+ def loadBytes = afterLoadBytes - initialBytes
+ logger.info("after load s3_file_writer_bytes_written: ${afterLoadBytes},
written: ${loadBytes}")
+ assertTrue(loadBytes > 0, "s3_file_writer_bytes_written should increase")
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
