This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 07d2425157b branch-3.1: [chore](sk) Encrypt `secret key` and hide
`access key` for log #55241 (#55619)
07d2425157b is described below
commit 07d2425157bae51ccd3603e3a90bc01a9e22135b
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sat Sep 6 00:36:22 2025 +0800
branch-3.1: [chore](sk) Encrypt `secret key` and hide `access key` for log
#55241 (#55619)
Cherry-picked from #55241
Co-authored-by: Uniqueyou <[email protected]>
---
be/src/cloud/cloud_meta_mgr.cpp | 9 ++
be/src/util/s3_util.cpp | 26 +++++
be/src/util/s3_util.h | 8 +-
be/test/util/s3_util_test.cpp | 79 ++++++++++++++
cloud/src/meta-service/meta_service.cpp | 26 +++++
cloud/src/meta-service/meta_service_helper.h | 84 +++++++++++----
cloud/src/meta-service/meta_service_http.cpp | 11 +-
cloud/src/meta-service/meta_service_resource.cpp | 13 ++-
cloud/src/resource-manager/resource_manager.cpp | 6 +-
cloud/test/meta_service_http_test.cpp | 118 +++++++++++++++++++++
.../doris/cloud/system/CloudSystemInfoService.java | 57 ++++++++++
11 files changed, 405 insertions(+), 32 deletions(-)
diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp
index 7e8f07d26cb..a05d9de402a 100644
--- a/be/src/cloud/cloud_meta_mgr.cpp
+++ b/be/src/cloud/cloud_meta_mgr.cpp
@@ -1271,6 +1271,15 @@ Status
CloudMetaMgr::get_storage_vault_info(StorageVaultInfos* vault_infos, bool
j->mutable_obj_info()->set_sk(j->obj_info().sk().substr(0, 2) + "xxx");
}
+ for (int i = 0; i < resp.obj_info_size(); ++i) {
+
resp.mutable_obj_info(i)->set_ak(hide_access_key(resp.obj_info(i).sk()));
+ }
+ for (int i = 0; i < resp.storage_vault_size(); ++i) {
+ auto* j = resp.mutable_storage_vault(i);
+ if (!j->has_obj_info()) continue;
+ j->mutable_obj_info()->set_sk(hide_access_key(j->obj_info().sk()));
+ }
+
LOG(INFO) << "get storage vault, enable_storage_vault=" << *is_vault_mode
<< " response=" << resp.ShortDebugString();
return Status::OK();
diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp
index 174b6f8a5e6..d4c4e4e3d34 100644
--- a/be/src/util/s3_util.cpp
+++ b/be/src/util/s3_util.cpp
@@ -574,4 +574,30 @@ S3Conf S3Conf::get_s3_conf(const TS3StorageParam& param) {
return ret;
}
+std::string hide_access_key(const std::string& ak) {
+ std::string key = ak;
+ size_t key_len = key.length();
+ size_t reserved_count;
+ if (key_len > 7) {
+ reserved_count = 6;
+ } else if (key_len > 2) {
+ reserved_count = key_len - 2;
+ } else {
+ reserved_count = 0;
+ }
+
+ size_t x_count = key_len - reserved_count;
+ size_t left_x_count = (x_count + 1) / 2;
+
+ if (left_x_count > 0) {
+ key.replace(0, left_x_count, left_x_count, 'x');
+ }
+
+ if (x_count - left_x_count > 0) {
+ key.replace(key_len - (x_count - left_x_count), x_count - left_x_count,
+ x_count - left_x_count, 'x');
+ }
+ return key;
+}
+
} // end namespace doris
diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h
index 8b96fb0776b..7f72bd3af26 100644
--- a/be/src/util/s3_util.h
+++ b/be/src/util/s3_util.h
@@ -61,6 +61,8 @@ extern bvar::LatencyRecorder s3_get_bucket_version_latency;
extern bvar::LatencyRecorder s3_copy_object_latency;
}; // namespace s3_bvar
+std::string hide_access_key(const std::string& ak);
+
class S3URI;
struct S3ClientConf {
std::string endpoint;
@@ -107,9 +109,9 @@ struct S3ClientConf {
"(ak={}, token={}, endpoint={}, region={}, bucket={},
max_connections={}, "
"request_timeout_ms={}, connect_timeout_ms={},
use_virtual_addressing={}, "
"cred_provider_type={},role_arn={}, external_id={}",
- ak, token, endpoint, region, bucket, max_connections,
request_timeout_ms,
- connect_timeout_ms, use_virtual_addressing,
cred_provider_type, role_arn,
- external_id);
+ hide_access_key(ak), token, endpoint, region, bucket,
max_connections,
+ request_timeout_ms, connect_timeout_ms,
use_virtual_addressing, cred_provider_type,
+ role_arn, external_id);
}
};
diff --git a/be/test/util/s3_util_test.cpp b/be/test/util/s3_util_test.cpp
new file mode 100644
index 00000000000..cd8eeb5cb68
--- /dev/null
+++ b/be/test/util/s3_util_test.cpp
@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "util/s3_util.h"
+
+#include <gtest/gtest-test-part.h>
+
+#include <string>
+
+#include "gtest/gtest_pred_impl.h"
+#include "util/s3_uri.h"
+
+namespace doris {
+
+class S3UTILTest : public testing::Test {
+public:
+ S3UTILTest() = default;
+ ~S3UTILTest() = default;
+}; // end class S3UTILTest
+
+TEST_F(S3UTILTest, hide_access_key_empty) {
+ EXPECT_EQ("", hide_access_key(""));
+}
+
+TEST_F(S3UTILTest, hide_access_key_single_char) {
+ EXPECT_EQ("x", hide_access_key("A"));
+}
+
+TEST_F(S3UTILTest, hide_access_key_two_chars) {
+ EXPECT_EQ("xx", hide_access_key("AB"));
+}
+
+TEST_F(S3UTILTest, hide_access_key_three_chars) {
+ EXPECT_EQ("xBx", hide_access_key("ABC"));
+}
+
+TEST_F(S3UTILTest, hide_access_key_four_chars) {
+ EXPECT_EQ("xBCx", hide_access_key("ABCD"));
+}
+
+TEST_F(S3UTILTest, hide_access_key_six_chars) {
+ EXPECT_EQ("xBCDEx", hide_access_key("ABCDEF"));
+}
+
+TEST_F(S3UTILTest, hide_access_key_seven_chars) {
+ EXPECT_EQ("xBCDEFx", hide_access_key("ABCDEFG"));
+}
+
+TEST_F(S3UTILTest, hide_access_key_normal_length) {
+ EXPECT_EQ("xxxDEFGHIxxx", hide_access_key("ABCDEFGHIJKL"));
+}
+
+TEST_F(S3UTILTest, hide_access_key_long_key) {
+ std::string long_key = "ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
+ std::string result = hide_access_key(long_key);
+ EXPECT_EQ("xxxxxxxxxxxxxxxPQRSTUxxxxxxxxxxxxxxx", result);
+}
+
+TEST_F(S3UTILTest, hide_access_key_typical_aws_key) {
+ std::string aws_key = "AKIAIOSFODNN7EXAMPLE";
+ std::string result = hide_access_key(aws_key);
+ EXPECT_EQ("xxxxxxxFODNN7xxxxxxx", result);
+}
+
+} // end namespace doris
diff --git a/cloud/src/meta-service/meta_service.cpp
b/cloud/src/meta-service/meta_service.cpp
index c54c1e30b95..aeb3e46c7da 100644
--- a/cloud/src/meta-service/meta_service.cpp
+++ b/cloud/src/meta-service/meta_service.cpp
@@ -4401,4 +4401,30 @@ void
MetaServiceImpl::get_schema_dict(::google::protobuf::RpcController* control
response->mutable_schema_dict()->Swap(&schema_dict);
}
+std::string hide_access_key(const std::string& ak) {
+ std::string key = ak;
+ size_t key_len = key.length();
+ size_t reserved_count;
+ if (key_len > 6) {
+ reserved_count = 6;
+ } else if (key_len > 2) {
+ reserved_count = key_len - 2;
+ } else {
+ reserved_count = 0;
+ }
+
+ size_t x_count = key_len - reserved_count;
+ size_t left_x_count = x_count / 2;
+
+ if (left_x_count > 0) {
+ key.replace(0, left_x_count, left_x_count, 'x');
+ }
+
+ if (x_count - left_x_count > 0) {
+ key.replace(key_len - (x_count - left_x_count), x_count - left_x_count,
+ x_count - left_x_count, 'x');
+ }
+ return key;
+}
+
} // namespace doris::cloud
diff --git a/cloud/src/meta-service/meta_service_helper.h
b/cloud/src/meta-service/meta_service_helper.h
index c35873ed949..38f32b2d4f8 100644
--- a/cloud/src/meta-service/meta_service_helper.h
+++ b/cloud/src/meta-service/meta_service_helper.h
@@ -54,6 +54,45 @@ inline std::string md5(const std::string& str) {
return ss.str();
}
+std::string hide_access_key(const std::string& ak);
+
+// is_handle_sk: true for encrypting sk, false for hiding ak
+inline void process_ak_sk_pattern(std::string& str, const std::string&
pattern, bool is_handle_sk) {
+ size_t pos = 0;
+ while ((pos = str.find(pattern, pos)) != std::string::npos) {
+ size_t colon_pos = str.find(':', pos);
+ if (colon_pos == std::string::npos) {
+ pos += pattern.length();
+ continue;
+ }
+
+ size_t quote_pos = str.find('\"', colon_pos);
+ if (quote_pos == std::string::npos) {
+ pos += pattern.length();
+ continue;
+ }
+
+ size_t value_start = quote_pos + 1;
+ size_t value_end = str.find('\"', value_start);
+ if (value_end == std::string::npos) {
+ pos = value_start;
+ continue;
+ }
+
+ std::string key_value = str.substr(value_start, value_end -
value_start);
+
+ if (is_handle_sk) {
+ key_value = "md5: " + md5(key_value);
+ } else {
+ key_value = hide_access_key(key_value);
+ }
+
+ str.replace(value_start, value_end - value_start, key_value);
+
+ pos = value_end + (key_value.length() - key_value.length());
+ }
+};
+
/**
* Encrypts all "sk" values in the given debug string with MD5 hashes.
*
@@ -61,36 +100,26 @@ inline std::string md5(const std::string& str) {
* - Input string contains one or more occurrences of "sk: " followed by a
value in double quotes.
* - An md5() function exists that takes a std::string and returns its MD5
hash as a string.
*
- * @param debug_string Input string containing "sk: " fields to be encrypted.
+ * @param debug_string Input string containing "sk: " or ""sk": " fields to be
encrypted.
* @return A new string with all "sk" values replaced by their MD5 hashes.
*
- * Behavior:
+ * Behavior for "sk: " format:
* 1. Searches for all occurrences of "sk: " in the input string.
* 2. For each occurrence, extracts the value between double quotes.
* 3. Replaces the original value with "md5: " followed by its MD5 hash.
* 4. Returns the modified string with all "sk" values encrypted.
*/
inline std::string encryt_sk(std::string debug_string) {
- // Start position for searching "sk" fields
- size_t pos = 0;
- // Iterate through the string and find all occurrences of "sk: "
- while ((pos = debug_string.find("sk: ", pos)) != std::string::npos) {
- // Find the start and end of the "sk" value (assumed to be within
quotes)
- // Start after the quote
- size_t sk_value_start = debug_string.find('\"', pos) + 1;
- // End at the next quote
- size_t sk_value_end = debug_string.find('\"', sk_value_start);
-
- // Extract the "sk" value
- std::string sk_value = debug_string.substr(sk_value_start,
sk_value_end - sk_value_start);
- // Encrypt the "sk" value with MD5
- std::string encrypted_sk = "md5: " + md5(sk_value);
-
- // Replace the original "sk" value with the encrypted MD5 value
- debug_string.replace(sk_value_start, sk_value_end - sk_value_start,
encrypted_sk);
- // Move the position to the end of the current "sk" field and continue
searching
- pos = sk_value_end;
- }
+ process_ak_sk_pattern(debug_string, "sk: ", true);
+ process_ak_sk_pattern(debug_string, "\"sk\"", true);
+
+ return debug_string;
+}
+
+inline std::string hide_ak(std::string debug_string) {
+ process_ak_sk_pattern(debug_string, "ak: ", false);
+ process_ak_sk_pattern(debug_string, "\"ak\"", false);
+
return debug_string;
}
@@ -134,6 +163,13 @@ void begin_rpc(std::string_view func_name,
brpc::Controller* ctrl, const Request
<< " lock_id=" << req->lock_id() << " initiator=" <<
req->initiator()
<< " expiration=" << req->expiration()
<< " require_compaction_stats=" <<
req->require_compaction_stats();
+ } else if constexpr (std::is_same_v<Request, CreateInstanceRequest> ||
+ std::is_same_v<Request, CreateStageRequest>) {
+ std::string debug_string = encryt_sk(req->ShortDebugString());
+ debug_string = hide_ak(debug_string);
+ TEST_SYNC_POINT_CALLBACK("sk_begin_rpc", &debug_string);
+ LOG(INFO) << "begin " << func_name << " remote_caller=" <<
ctrl->remote_side()
+ << " original_client_ip=" << req->request_ip() << "
request=" << debug_string;
} else {
LOG(INFO) << "begin " << func_name << " remote_caller=" <<
ctrl->remote_side()
<< " original_client_ip=" << req->request_ip()
@@ -179,8 +215,10 @@ void finish_rpc(std::string_view func_name,
brpc::Controller* ctrl, const Reques
<< " original_client_ip=" << req->request_ip()
<< " status=" << res->status().ShortDebugString();
} else if constexpr (std::is_same_v<Response, GetObjStoreInfoResponse> ||
- std::is_same_v<Response, GetStageResponse>) {
+ std::is_same_v<Response, GetStageResponse> ||
+ std::is_same_v<Response, GetInstanceResponse>) {
std::string debug_string = encryt_sk(res->DebugString());
+ debug_string = hide_ak(debug_string);
TEST_SYNC_POINT_CALLBACK("sk_finish_rpc", &debug_string);
LOG(INFO) << "finish " << func_name << " remote_caller=" <<
ctrl->remote_side()
<< " original_client_ip=" << req->request_ip() << "
response=" << debug_string;
diff --git a/cloud/src/meta-service/meta_service_http.cpp
b/cloud/src/meta-service/meta_service_http.cpp
index 8d7f6bba6e4..be4c643f7e5 100644
--- a/cloud/src/meta-service/meta_service_http.cpp
+++ b/cloud/src/meta-service/meta_service_http.cpp
@@ -48,6 +48,7 @@
#include "common/configbase.h"
#include "common/logging.h"
#include "common/string_util.h"
+#include "meta-service/meta_service_helper.h"
#include "meta-store/keys.h"
#include "meta-store/txn_kv.h"
#include "meta-store/txn_kv_error.h"
@@ -63,7 +64,7 @@ namespace doris::cloud {
auto st = parse_json_message(unresolved_path, body, &req);
\
if (!st.ok()) {
\
std::string msg = "parse http request '" + unresolved_path + "': "
+ st.ToString(); \
- LOG_WARNING(msg).tag("body", body);
\
+ LOG_WARNING(msg).tag("body", encryt_sk(body));
\
return http_json_reply(MetaServiceCode::PROTOBUF_PARSE_ERR, msg);
\
}
\
} while (0)
@@ -86,7 +87,7 @@ static google::protobuf::util::Status
parse_json_message(const std::string& unre
if (!st.ok()) {
std::string msg = "failed to strictly parse http request for '" +
unresolved_path +
"' error: " + st.ToString();
- LOG_WARNING(msg).tag("body", body);
+ LOG_WARNING(msg).tag("body", encryt_sk(hide_access_key(body)));
// ignore unknown fields
google::protobuf::util::JsonParseOptions json_parse_options;
@@ -776,6 +777,8 @@ void
MetaServiceImpl::http(::google::protobuf::RpcController* controller,
LOG(INFO) << "rpc from " << cntl->remote_side()
<< " request: " << cntl->http_request().uri().path();
std::string http_request = format_http_request(cntl);
+ std::string http_request_for_log = encryt_sk(http_request);
+ http_request_for_log = hide_ak(http_request_for_log);
// Auth
auto token = http_query(cntl->http_request().uri(), "token");
@@ -786,7 +789,7 @@ void
MetaServiceImpl::http(::google::protobuf::RpcController* controller,
cntl->response_attachment().append(body);
cntl->response_attachment().append("\n");
LOG(WARNING) << "failed to handle http from " << cntl->remote_side()
- << " request: " << http_request << " msg: " << body;
+ << " request: " << http_request_for_log << " msg: " <<
body;
return;
}
@@ -806,7 +809,7 @@ void
MetaServiceImpl::http(::google::protobuf::RpcController* controller,
int ret = cntl->http_response().status_code();
LOG(INFO) << (ret == 200 ? "succ to " : "failed to ") <<
__PRETTY_FUNCTION__ << " "
<< cntl->remote_side() << " request=\n"
- << http_request << "\n ret=" << ret << " msg=" << msg;
+ << http_request_for_log << "\n ret=" << ret << " msg=" << msg;
}
} // namespace doris::cloud
diff --git a/cloud/src/meta-service/meta_service_resource.cpp
b/cloud/src/meta-service/meta_service_resource.cpp
index 9a45c7f6e9d..67bd81c10a6 100644
--- a/cloud/src/meta-service/meta_service_resource.cpp
+++ b/cloud/src/meta-service/meta_service_resource.cpp
@@ -76,7 +76,7 @@ static int encrypt_ak_sk_helper(const std::string plain_ak,
const std::string pl
MetaServiceCode& code, std::string& msg) {
std::string key;
int64_t key_id;
- LOG_INFO("enter encrypt_ak_sk_helper, plain_ak {}", plain_ak);
+ LOG_INFO("enter encrypt_ak_sk_helper, plain_ak {}",
hide_access_key(plain_ak));
int ret = get_newest_encryption_key_for_ak_sk(&key_id, &key);
TEST_SYNC_POINT_CALLBACK("encrypt_ak_sk:get_encryption_key", &ret, &key,
&key_id);
if (ret != 0) {
@@ -1661,7 +1661,10 @@ void
MetaServiceImpl::create_instance(google::protobuf::RpcController* controlle
const CreateInstanceRequest* request,
CreateInstanceResponse* response,
::google::protobuf::Closure* done) {
+ TEST_SYNC_POINT_CALLBACK("create_instance_sk_request",
+ const_cast<CreateInstanceRequest**>(&request));
RPC_PREPROCESS(create_instance, get, put);
+ TEST_SYNC_POINT_RETURN_WITH_VOID("create_instance_sk_request_return");
if (request->has_ram_user()) {
auto& ram_user = request->ram_user();
std::string ram_user_id = ram_user.has_user_id() ? ram_user.user_id()
: "";
@@ -1739,6 +1742,10 @@ void
MetaServiceImpl::create_instance(google::protobuf::RpcController* controlle
return;
}
+ for (auto& obj_info : *instance.mutable_obj_info()) {
+ obj_info.set_ak(hide_access_key(obj_info.ak()));
+ }
+
LOG(INFO) << "xxx instance json=" << proto_to_json(instance);
// Check existence before proceeding
@@ -1978,6 +1985,8 @@ void
MetaServiceImpl::get_instance(google::protobuf::RpcController* controller,
const GetInstanceRequest* request,
GetInstanceResponse* response,
::google::protobuf::Closure* done) {
RPC_PREPROCESS(get_instance, get);
+ TEST_SYNC_POINT_CALLBACK("get_instance_sk_response", &response);
+ TEST_SYNC_POINT_RETURN_WITH_VOID("get_instance_sk_response_return");
std::string cloud_unique_id = request->has_cloud_unique_id() ?
request->cloud_unique_id() : "";
if (cloud_unique_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
@@ -2763,7 +2772,9 @@ void
MetaServiceImpl::get_cluster(google::protobuf::RpcController* controller,
void MetaServiceImpl::create_stage(::google::protobuf::RpcController*
controller,
const CreateStageRequest* request,
CreateStageResponse* response,
::google::protobuf::Closure* done) {
+ TEST_SYNC_POINT_CALLBACK("create_stage_sk_request",
const_cast<CreateStageRequest**>(&request));
RPC_PREPROCESS(create_stage, get, put);
+ TEST_SYNC_POINT_RETURN_WITH_VOID("create_stage_sk_request_return");
std::string cloud_unique_id = request->has_cloud_unique_id() ?
request->cloud_unique_id() : "";
if (cloud_unique_id.empty()) {
code = MetaServiceCode::INVALID_ARGUMENT;
diff --git a/cloud/src/resource-manager/resource_manager.cpp
b/cloud/src/resource-manager/resource_manager.cpp
index 7c5a95b4267..3aa7fcb99b3 100644
--- a/cloud/src/resource-manager/resource_manager.cpp
+++ b/cloud/src/resource-manager/resource_manager.cpp
@@ -519,8 +519,12 @@ std::pair<MetaServiceCode, std::string>
ResourceManager::add_cluster(const std::
}
auto& req_cluster = cluster.cluster;
+ InstanceInfoPB instance_for_log {instance};
+ for (auto& obj_info : *instance_for_log.mutable_obj_info()) {
+ obj_info.set_ak(hide_access_key(obj_info.ak()));
+ }
LOG(INFO) << "cluster to add json=" << proto_to_json(req_cluster);
- LOG(INFO) << "json=" << proto_to_json(instance);
+ LOG(INFO) << "json=" << proto_to_json(instance_for_log);
// Check id and name, they need to be unique
// One cluster id per name, name is alias of cluster id
diff --git a/cloud/test/meta_service_http_test.cpp
b/cloud/test/meta_service_http_test.cpp
index 082a7e3d974..9fb7abb04f3 100644
--- a/cloud/test/meta_service_http_test.cpp
+++ b/cloud/test/meta_service_http_test.cpp
@@ -1603,6 +1603,124 @@ TEST(MetaServiceHttpTest,
get_obj_store_info_response_sk) {
ms->get_obj_store_info(&cntl, &req1, &res1, nullptr);
}
+TEST(MetaServiceHttpTest, get_instance_response_sk) {
+ auto sp = SyncPoint::get_instance();
+ sp->enable_processing();
+ DORIS_CLOUD_DEFER {
+ sp->disable_processing();
+ };
+
+ GetInstanceResponse res;
+ auto* obj_info = res.mutable_instance()->add_obj_info();
+ obj_info->set_ak("instance-ak");
+ obj_info->set_sk("instance-sk");
+ auto foo = [res](auto args) {
+ (*(try_any_cast<GetInstanceResponse**>(args[0])))->CopyFrom(res);
+ };
+ sp->set_call_back("get_instance_sk_response", foo);
+ sp->set_call_back("get_instance_sk_response_return",
+ [](auto&& args) { *try_any_cast<bool*>(args.back()) =
true; });
+
+ auto rate_limiter = std::make_shared<cloud::RateLimiter>();
+
+ auto ms = std::make_unique<cloud::MetaServiceImpl>(nullptr, nullptr,
rate_limiter);
+
+ auto bar = [](auto args) {
+ std::cout << *try_any_cast<std::string*>(args[0]);
+
+ EXPECT_TRUE((*try_any_cast<std::string*>(args[0])).find("instance-sk")
==
+ std::string::npos);
+ EXPECT_TRUE((*try_any_cast<std::string*>(args[0]))
+ .find("md5: 79d1924d669b7412019edc42db31bd92") !=
std::string::npos);
+ };
+ sp->set_call_back("sk_finish_rpc", bar);
+
+ GetInstanceResponse res1;
+ GetInstanceRequest req1;
+ brpc::Controller cntl;
+ ms->get_instance(&cntl, &req1, &res1, nullptr);
+}
+
+TEST(MetaServiceHttpTest, create_instance_request_sk) {
+ auto sp = SyncPoint::get_instance();
+ sp->enable_processing();
+ DORIS_CLOUD_DEFER {
+ sp->disable_processing();
+ };
+
+ CreateInstanceRequest req;
+ req.set_instance_id("get_value_instance_id");
+ req.set_user_id("test_user");
+ req.set_name("test_name");
+ ObjectStoreInfoPB obj;
+ obj.set_ak("instance-ak");
+ obj.set_sk("instance-sk");
+ req.mutable_obj_info()->CopyFrom(obj);
+
+ auto foo = [req](auto args) {
+ (*(try_any_cast<CreateInstanceRequest**>(args[0])))->CopyFrom(req);
+ };
+ sp->set_call_back("create_instance_sk_request", foo);
+ sp->set_call_back("create_instance_sk_request_return",
+ [](auto&& args) { *try_any_cast<bool*>(args.back()) =
true; });
+
+ auto rate_limiter = std::make_shared<cloud::RateLimiter>();
+
+ auto ms = std::make_unique<cloud::MetaServiceImpl>(nullptr, nullptr,
rate_limiter);
+
+ auto bar = [](auto args) {
+ std::cout << *try_any_cast<std::string*>(args[0]) << '\n';
+
+ EXPECT_TRUE((*try_any_cast<std::string*>(args[0])).find("instance-sk")
==
+ std::string::npos);
+ EXPECT_TRUE((*try_any_cast<std::string*>(args[0]))
+ .find("md5: 79d1924d669b7412019edc42db31bd92") !=
std::string::npos);
+ };
+ sp->set_call_back("sk_begin_rpc", bar);
+
+ CreateInstanceResponse res1;
+ CreateInstanceRequest req1;
+ brpc::Controller cntl;
+ ms->create_instance(&cntl, &req1, &res1, nullptr);
+}
+
+TEST(MetaServiceHttpTest, create_stage_request_sk) {
+ auto sp = SyncPoint::get_instance();
+ sp->enable_processing();
+ DORIS_CLOUD_DEFER {
+ sp->disable_processing();
+ };
+
+ CreateStageRequest req;
+ req.mutable_stage()->mutable_obj_info()->set_ak("stage-ak");
+ req.mutable_stage()->mutable_obj_info()->set_sk("stage-sk");
+
+ auto foo = [req](auto args) {
+ (*(try_any_cast<CreateStageRequest**>(args[0])))->CopyFrom(req);
+ };
+ sp->set_call_back("create_stage_sk_request", foo);
+ sp->set_call_back("create_stage_sk_request_return",
+ [](auto&& args) { *try_any_cast<bool*>(args.back()) =
true; });
+
+ auto rate_limiter = std::make_shared<cloud::RateLimiter>();
+
+ auto ms = std::make_unique<cloud::MetaServiceImpl>(nullptr, nullptr,
rate_limiter);
+
+ auto bar = [](auto args) {
+ std::cout << *try_any_cast<std::string*>(args[0]) << '\n';
+
+ EXPECT_TRUE((*try_any_cast<std::string*>(args[0])).find("stage-sk") ==
std::string::npos);
+ EXPECT_TRUE((*try_any_cast<std::string*>(args[0]))
+ .find("md5: f497d053066fa4b7d3b1f6564597d233") !=
std::string::npos);
+ };
+ sp->set_call_back("sk_begin_rpc", bar);
+
+ CreateStageResponse res1;
+ CreateStageRequest req1;
+ brpc::Controller cntl;
+ ms->create_stage(&cntl, &req1, &res1, nullptr);
+}
+
TEST(MetaServiceHttpTest, AdjustRateLimit) {
HttpContext ctx;
{
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java
index 81765e18029..10ad5c7ab66 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/system/CloudSystemInfoService.java
@@ -26,6 +26,7 @@ import org.apache.doris.cloud.catalog.ComputeGroup;
import org.apache.doris.cloud.proto.Cloud;
import org.apache.doris.cloud.proto.Cloud.ClusterPB;
import org.apache.doris.cloud.proto.Cloud.InstanceInfoPB;
+import org.apache.doris.cloud.proto.Cloud.ObjectStoreInfoPB;
import org.apache.doris.cloud.qe.ComputeGroupException;
import org.apache.doris.cloud.rpc.MetaServiceProxy;
import org.apache.doris.common.AnalysisException;
@@ -1525,6 +1526,7 @@ public class CloudSystemInfoService extends
SystemInfoService {
try {
Cloud.GetInstanceRequest request = builder.build();
response = MetaServiceProxy.getInstance().getInstance(request);
+ response = hideAkSkForStorageVault(response.toBuilder());
LOG.info("get instance info, request: {}, response: {}", request,
response);
if (response.getStatus().getCode() != Cloud.MetaServiceCode.OK) {
LOG.warn("Failed to get instance info, response: {}",
response);
@@ -1537,6 +1539,61 @@ public class CloudSystemInfoService extends
SystemInfoService {
}
}
+ private String hideKey(String key) {
+ if (key == null || key.isEmpty()) {
+ return "";
+ }
+
+ int keyLen = key.length();
+ int reservedCount = (keyLen > 6) ? 6 : (keyLen > 2 ? keyLen - 2 : 0);
+ int xCount = keyLen - reservedCount;
+
+ int leftXCount = xCount / 2;
+ StringBuilder result = new StringBuilder();
+
+ for (int i = 0; i < leftXCount; i++) {
+ result.append('x');
+ }
+
+ int startIndex = leftXCount;
+ int endIndex = startIndex + reservedCount;
+ result.append(key.substring(startIndex, endIndex));
+
+ int rightXCount = xCount - leftXCount;
+ for (int i = 0; i < rightXCount; i++) {
+ result.append('x');
+ }
+ return result.toString();
+ }
+
+ public Cloud.GetInstanceResponse
hideAkSkForStorageVault(Cloud.GetInstanceResponse.Builder resp) {
+ if (resp == null) {
+ return null;
+ }
+
+ if (resp.getInstance().getObjInfoCount() == 0) {
+ return resp.build();
+ }
+
+ for (int i = 0; i < resp.getInstance().getObjInfoCount(); i++) {
+ ObjectStoreInfoPB objInfo = resp.getInstance().getObjInfo(i);
+ if (objInfo == null) {
+ continue;
+ }
+ if (objInfo.hasAk()) {
+ String ak = objInfo.getAk();
+ String hiddenAk = hideKey(ak);
+ resp.getInstanceBuilder().getObjInfoBuilder(i).setAk(hiddenAk);
+ }
+ if (objInfo.hasSk()) {
+ String sk = objInfo.getSk();
+ String hiddenSk = hideKey(sk);
+ resp.getInstanceBuilder().getObjInfoBuilder(i).setSk(hiddenSk);
+ }
+ }
+ return resp.build();
+ }
+
public void renameComputeGroup(String originalName, String newGroupName)
throws UserException {
String cloudInstanceId = ((CloudEnv)
Env.getCurrentEnv()).getCloudInstanceId();
if (Strings.isNullOrEmpty(cloudInstanceId)) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
