This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 23fe429cbcd branch-4.0: [chore](cloud) Print new value json when
set_value #57385 (#57459)
23fe429cbcd is described below
commit 23fe429cbcdeb57aeebf3d20b9f1ce2d75e73ebb
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Oct 30 10:47:12 2025 +0800
branch-4.0: [chore](cloud) Print new value json when set_value #57385
(#57459)
Cherry-picked from #57385
Co-authored-by: Gavin Chou <[email protected]>
---
cloud/src/meta-service/http_encode_key.cpp | 45 ++++++++++++++++++------------
cloud/test/http_encode_key_test.cpp | 11 ++++++--
cloud/test/meta_service_http_test.cpp | 7 +++--
3 files changed, 40 insertions(+), 23 deletions(-)
diff --git a/cloud/src/meta-service/http_encode_key.cpp
b/cloud/src/meta-service/http_encode_key.cpp
index cae6fb322f7..4ee76af5c9f 100644
--- a/cloud/src/meta-service/http_encode_key.cpp
+++ b/cloud/src/meta-service/http_encode_key.cpp
@@ -23,6 +23,7 @@
#include <google/protobuf/util/json_util.h>
#include <bit>
+#include <chrono>
#include <fstream>
#include <iomanip>
#include <sstream>
@@ -240,15 +241,16 @@ static std::unordered_map<std::string_view,
};
// clang-format on
-static MetaServiceResponseStatus encode_key(const brpc::URI& uri, std::string&
key) {
+static MetaServiceResponseStatus encode_key(const brpc::URI& uri, std::string*
key,
+ std::string* key_type = nullptr) {
MetaServiceResponseStatus status;
status.set_code(MetaServiceCode::OK);
- std::string_view key_type = http_query(uri, "key_type");
- auto it = param_set.find(key_type);
+ std::string_view kt = http_query(uri, "key_type");
+ if (key_type != nullptr) *key_type = kt;
+ auto it = param_set.find(kt);
if (it == param_set.end()) {
status.set_code(MetaServiceCode::INVALID_ARGUMENT);
- status.set_msg(fmt::format("key_type not supported: {}",
- (key_type.empty() ? "(empty)" : key_type)));
+ status.set_msg(fmt::format("key_type not supported: {}", (kt.empty() ?
"(empty)" : kt)));
return status;
}
auto& key_params = std::get<0>(it->second);
@@ -264,16 +266,18 @@ static MetaServiceResponseStatus encode_key(const
brpc::URI& uri, std::string& k
params.emplace_back(p);
}
auto& key_encoding_function = std::get<1>(it->second);
- key = key_encoding_function(params);
+ *key = key_encoding_function(params);
return status;
}
HttpResponse process_http_get_value(TxnKv* txn_kv, const brpc::URI& uri) {
std::string key;
+ std::string key_type;
if (auto hex_key = http_query(uri, "key"); !hex_key.empty()) {
key = unhex(hex_key);
+ key_type = http_query(uri, "key_type");
} else { // Encode key from params
- auto st = encode_key(uri, key);
+ auto st = encode_key(uri, &key, &key_type);
if (st.code() != MetaServiceCode::OK) {
return http_json_reply(st);
}
@@ -285,7 +289,6 @@ HttpResponse process_http_get_value(TxnKv* txn_kv, const
brpc::URI& uri) {
fmt::format("failed to create txn, err={}",
err));
}
- std::string_view key_type = http_query(uri, "key_type");
auto it = param_set.find(key_type);
if (it == param_set.end()) {
return http_json_reply(MetaServiceCode::INVALID_ARGUMENT,
@@ -330,17 +333,20 @@ HttpResponse process_http_get_value(TxnKv* txn_kv, const
brpc::URI& uri) {
}
std::string handle_kv_output(std::string_view key, std::string_view value,
- std::string_view original_value_json,
+ std::string_view original_value_json,
std::string_view new_value_json,
std::string_view serialized_value_to_save) {
std::stringstream final_output;
- final_output << "original_value_hex=" << hex(value) << "\n"
- << "key_hex=" << hex(key) << "\n"
+ final_output << "key_hex=" << hex(key) << "\n"
<< "original_value_json=" << original_value_json << "\n"
- << "changed_value_hex=" << hex(serialized_value_to_save) <<
"\n";
+ << "new_value_json=" << new_value_json << "\n"
+ << "original_value_hex=" << hex(value) << "\n"
+ << "new_value_hex=" << hex(serialized_value_to_save) << "\n";
std::string final_json_str = final_output.str();
LOG(INFO) << final_json_str;
if (final_json_str.size() > 25000) {
- std::string file_path = fmt::format("/tmp/{}.txt", hex(key));
+ using namespace std::chrono;
+ auto ts =
duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count();
+ std::string file_path = fmt::format("/tmp/{}_{}.txt", hex(key), ts);
LOG(INFO) << "write to file=" << file_path << ", key=" << hex(key)
<< " size=" << final_json_str.size();
try {
@@ -350,7 +356,7 @@ std::string handle_kv_output(std::string_view key,
std::string_view value,
kv_file.close();
}
} catch (...) {
- LOG(INFO) << "write tmp file failed.";
+ LOG(INFO) << "write tmp file failed: " << file_path;
}
}
@@ -363,10 +369,12 @@ HttpResponse process_http_set_value(TxnKv* txn_kv,
brpc::Controller* cntl) {
LOG(INFO) << "set value, body=" << body;
std::string key;
+ std::string key_type;
if (auto hex_key = http_query(uri, "key"); !hex_key.empty()) {
key = unhex(hex_key);
+ key_type = http_query(uri, "key_type");
} else { // Encode key from params
- auto st = encode_key(uri, key);
+ auto st = encode_key(uri, &key, &key_type);
if (st.code() != MetaServiceCode::OK) {
return http_json_reply(st);
}
@@ -378,13 +386,13 @@ HttpResponse process_http_set_value(TxnKv* txn_kv,
brpc::Controller* cntl) {
fmt::format("failed to create txn, err={}",
err));
}
- std::string_view key_type = http_query(uri, "key_type");
auto it = param_set.find(key_type);
if (it == param_set.end()) {
return http_json_reply(MetaServiceCode::INVALID_ARGUMENT,
fmt::format("key_type not supported: {}",
(key_type.empty() ? "(empty)" :
key_type)));
}
+
auto& json_parsing_function = std::get<3>(it->second);
std::shared_ptr<google::protobuf::Message> pb_to_save =
json_parsing_function(body);
if (pb_to_save == nullptr) {
@@ -458,14 +466,15 @@ HttpResponse process_http_set_value(TxnKv* txn_kv,
brpc::Controller* cntl) {
LOG(WARNING) << "set_value saved, key=" << hex(key);
std::string final_json_str =
- handle_kv_output(key, value.value(), original_value_json,
serialized_value_to_save);
+ handle_kv_output(key, value.value(), original_value_json,
proto_to_json(*pb_to_save),
+ serialized_value_to_save);
return http_text_reply(MetaServiceCode::OK, "", final_json_str);
}
HttpResponse process_http_encode_key(const brpc::URI& uri) {
std::string key;
- auto st = encode_key(uri, key);
+ auto st = encode_key(uri, &key);
if (st.code() != MetaServiceCode::OK) {
return http_json_reply(st);
}
diff --git a/cloud/test/http_encode_key_test.cpp
b/cloud/test/http_encode_key_test.cpp
index d38351b6ec6..de1ffe42ffe 100644
--- a/cloud/test/http_encode_key_test.cpp
+++ b/cloud/test/http_encode_key_test.cpp
@@ -116,6 +116,13 @@ struct Input {
std::vector<std::string> key;
std::function<std::vector<std::string>()> gen_value;
std::string_view value;
+
+ std::string to_string() {
+ std::stringstream ss;
+ ss << "key_type=" << key_type << " param=" << param << " key=" <<
key[0]
+ << " value=" << value;
+ return ss.str();
+ }
};
// clang-format off
@@ -618,7 +625,7 @@ TEST(HttpGetValueTest,
process_http_get_value_test_cover_all_template) {
// std::cout << url << std::endl;
ASSERT_EQ(uri.SetHttpURL(url), 0); // clear and set query string
auto http_res = process_http_get_value(txn_kv.get(), uri);
- EXPECT_EQ(http_res.status_code, 200);
+ EXPECT_EQ(http_res.status_code, 200) << url << " " <<
input.to_string();
// std::cout << http_res.body << std::endl;
EXPECT_EQ(http_res.body, input.value);
// Key mode
@@ -626,7 +633,7 @@ TEST(HttpGetValueTest,
process_http_get_value_test_cover_all_template) {
// std::cout << url << std::endl;
ASSERT_EQ(uri.SetHttpURL(url), 0); // clear and set query string
http_res = process_http_get_value(txn_kv.get(), uri);
- EXPECT_EQ(http_res.status_code, 200);
+ EXPECT_EQ(http_res.status_code, 200) << url << " " <<
input.to_string();
// std::cout << http_res.body << std::endl;
EXPECT_EQ(http_res.body, input.value);
}
diff --git a/cloud/test/meta_service_http_test.cpp
b/cloud/test/meta_service_http_test.cpp
index 302dccf7b60..8ae106bc861 100644
--- a/cloud/test/meta_service_http_test.cpp
+++ b/cloud/test/meta_service_http_test.cpp
@@ -2096,10 +2096,11 @@ TEST(HttpEncodeKeyTest, ProcessHttpSetValue) {
auto response = process_http_set_value(txn_kv.get(), &cntl);
EXPECT_EQ(response.status_code, 200) << response.msg;
std::stringstream final_json;
- final_json << "original_value_hex=" <<
hex(initial_rowset_meta.SerializeAsString()) << "\n"
- << "key_hex=" << hex(initial_key) << "\n"
+ final_json << "key_hex=" << hex(initial_key) << "\n"
<< "original_value_json=" << proto_to_json(initial_rowset_meta)
<< "\n"
- << "changed_value_hex=" <<
hex(new_rowset_meta.SerializeAsString()) << "\n";
+ << "new_value_json=" << proto_to_json(new_rowset_meta) << "\n"
+ << "original_value_hex=" <<
hex(initial_rowset_meta.SerializeAsString()) << "\n"
+ << "new_value_hex=" << hex(new_rowset_meta.SerializeAsString())
<< "\n";
// std::cout << "xxx " << final_json.str() << std::endl;
EXPECT_EQ(response.body, final_json.str());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
