gavinchou commented on code in PR #38685:
URL: https://github.com/apache/doris/pull/38685#discussion_r1708671056
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java:
##########
@@ -56,6 +58,11 @@ public class HdfsStorageVault extends StorageVault {
public static final String HDFS_PREFIX = "hdfs:";
public static final String HDFS_FILE_PREFIX = "hdfs://";
+ public static final HashSet<String> ALTER_CHECK_PROPERTIES = new
HashSet<>(Arrays.asList(
Review Comment:
for hdfs, only name and passwd can be altered
##########
cloud/src/meta-service/meta_service_resource.cpp:
##########
@@ -510,13 +510,115 @@ static void set_default_vault_log_helper(const
InstanceInfoPB& instance,
LOG(INFO) << vault_msg;
}
+static int alter_hdfs_storage_vault(InstanceInfoPB& instance,
std::unique_ptr<Transaction> txn,
+ const StorageVaultPB& vault,
MetaServiceCode& code,
+ std::string& msg) {
+ if (!vault.has_hdfs_info()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ std::stringstream ss;
+ ss << "There is no hdfs vault provided";
+ msg = ss.str();
+ return -1;
+ }
+ const auto& hdfs_info = vault.hdfs_info();
+ if (hdfs_info.has_prefix() || !hdfs_info.has_build_conf() ||
+ hdfs_info.build_conf().has_fs_name()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ std::stringstream ss;
+ ss << "You can not alter prefix or fs name";
Review Comment:
for what reason, pls describe what is wrong here
##########
cloud/src/meta-service/meta_service_resource.cpp:
##########
@@ -510,13 +510,115 @@ static void set_default_vault_log_helper(const
InstanceInfoPB& instance,
LOG(INFO) << vault_msg;
}
+static int alter_hdfs_storage_vault(InstanceInfoPB& instance,
std::unique_ptr<Transaction> txn,
+ const StorageVaultPB& vault,
MetaServiceCode& code,
+ std::string& msg) {
+ if (!vault.has_hdfs_info()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ std::stringstream ss;
+ ss << "There is no hdfs vault provided";
+ msg = ss.str();
+ return -1;
+ }
+ const auto& hdfs_info = vault.hdfs_info();
+ if (hdfs_info.has_prefix() || !hdfs_info.has_build_conf() ||
+ hdfs_info.build_conf().has_fs_name()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ std::stringstream ss;
+ ss << "You can not alter prefix or fs name";
+ msg = ss.str();
+ return -1;
+ }
+ const auto& name = vault.name();
+ // Here we try to get mutable iter since we might need to alter the vault
name
+ auto name_itr =
std::find_if(instance.mutable_storage_vault_names()->begin(),
+ instance.mutable_storage_vault_names()->end(),
+ [&](const auto& vault_name) { return name ==
vault_name; });
+ if (name_itr == instance.storage_vault_names().end()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ std::stringstream ss;
+ ss << "invalid storage vault name, not found, name =" << name;
+ msg = ss.str();
+ return -1;
+ }
+ auto pos = name_itr - instance.storage_vault_names().begin();
+ auto id_itr = instance.resource_ids().begin() + pos;
+ auto vault_key = storage_vault_key({instance.instance_id(), *id_itr});
+ std::string val;
+
+ auto err = txn->get(vault_key, &val);
+ LOG(INFO) << "get instance_key=" << hex(vault_key);
Review Comment:
typo in log content, vault key?
##########
cloud/src/meta-service/meta_service_resource.cpp:
##########
@@ -510,13 +510,115 @@ static void set_default_vault_log_helper(const
InstanceInfoPB& instance,
LOG(INFO) << vault_msg;
}
+static int alter_hdfs_storage_vault(InstanceInfoPB& instance,
std::unique_ptr<Transaction> txn,
+ const StorageVaultPB& vault,
MetaServiceCode& code,
+ std::string& msg) {
+ if (!vault.has_hdfs_info()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ std::stringstream ss;
+ ss << "There is no hdfs vault provided";
+ msg = ss.str();
+ return -1;
+ }
+ const auto& hdfs_info = vault.hdfs_info();
+ if (hdfs_info.has_prefix() || !hdfs_info.has_build_conf() ||
+ hdfs_info.build_conf().has_fs_name()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ std::stringstream ss;
+ ss << "You can not alter prefix or fs name";
+ msg = ss.str();
+ return -1;
+ }
+ const auto& name = vault.name();
+ // Here we try to get mutable iter since we might need to alter the vault
name
+ auto name_itr =
std::find_if(instance.mutable_storage_vault_names()->begin(),
+ instance.mutable_storage_vault_names()->end(),
+ [&](const auto& vault_name) { return name ==
vault_name; });
+ if (name_itr == instance.storage_vault_names().end()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ std::stringstream ss;
+ ss << "invalid storage vault name, not found, name =" << name;
+ msg = ss.str();
+ return -1;
+ }
+ auto pos = name_itr - instance.storage_vault_names().begin();
+ auto id_itr = instance.resource_ids().begin() + pos;
+ auto vault_key = storage_vault_key({instance.instance_id(), *id_itr});
+ std::string val;
+
+ auto err = txn->get(vault_key, &val);
+ LOG(INFO) << "get instance_key=" << hex(vault_key);
+
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::READ>(err);
+ std::stringstream ss;
+ ss << "failed to get storage vault, vault_id=" << *name_itr << ",
vault_name="
+ << "" << name << " err=" << err;
+ msg = ss.str();
+ return -1;
+ }
+ StorageVaultPB alter;
+ alter.ParseFromString(val);
+ if (vault.has_alter_name()) {
+ alter.set_name(vault.alter_name());
Review Comment:
Do a name check, a vault name should satisfy `[a-zA-Z][0-9a-zA-Z_]`
make it strict
##########
cloud/src/meta-service/meta_service_resource.cpp:
##########
@@ -510,13 +510,115 @@ static void set_default_vault_log_helper(const
InstanceInfoPB& instance,
LOG(INFO) << vault_msg;
}
+static int alter_hdfs_storage_vault(InstanceInfoPB& instance,
std::unique_ptr<Transaction> txn,
+ const StorageVaultPB& vault,
MetaServiceCode& code,
+ std::string& msg) {
+ if (!vault.has_hdfs_info()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ std::stringstream ss;
+ ss << "There is no hdfs vault provided";
+ msg = ss.str();
+ return -1;
+ }
+ const auto& hdfs_info = vault.hdfs_info();
+ if (hdfs_info.has_prefix() || !hdfs_info.has_build_conf() ||
+ hdfs_info.build_conf().has_fs_name()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ std::stringstream ss;
+ ss << "You can not alter prefix or fs name";
+ msg = ss.str();
+ return -1;
+ }
+ const auto& name = vault.name();
+ // Here we try to get mutable iter since we might need to alter the vault
name
+ auto name_itr =
std::find_if(instance.mutable_storage_vault_names()->begin(),
+ instance.mutable_storage_vault_names()->end(),
+ [&](const auto& vault_name) { return name ==
vault_name; });
+ if (name_itr == instance.storage_vault_names().end()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ std::stringstream ss;
+ ss << "invalid storage vault name, not found, name =" << name;
+ msg = ss.str();
+ return -1;
+ }
+ auto pos = name_itr - instance.storage_vault_names().begin();
+ auto id_itr = instance.resource_ids().begin() + pos;
+ auto vault_key = storage_vault_key({instance.instance_id(), *id_itr});
+ std::string val;
+
+ auto err = txn->get(vault_key, &val);
+ LOG(INFO) << "get instance_key=" << hex(vault_key);
+
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::READ>(err);
+ std::stringstream ss;
+ ss << "failed to get storage vault, vault_id=" << *name_itr << ",
vault_name="
+ << "" << name << " err=" << err;
+ msg = ss.str();
+ return -1;
+ }
+ StorageVaultPB alter;
+ alter.ParseFromString(val);
+ if (vault.has_alter_name()) {
+ alter.set_name(vault.alter_name());
+ *name_itr = vault.alter_name();
+ }
+ std::stringstream log_msg;
+ auto* alter_hdfs_info = alter.mutable_hdfs_info();
+ if (hdfs_info.build_conf().has_hdfs_kerberos_keytab()) {
+ log_msg << " set kerberos key tab from "
+ << alter_hdfs_info->build_conf().hdfs_kerberos_keytab() << "
to "
+ << hdfs_info.build_conf().hdfs_kerberos_keytab();
+ alter_hdfs_info->mutable_build_conf()->set_hdfs_kerberos_keytab(
+ hdfs_info.build_conf().hdfs_kerberos_keytab());
+ }
+ if (hdfs_info.build_conf().has_hdfs_kerberos_principal()) {
+ log_msg << " set kerberos principal from "
+ << alter_hdfs_info->build_conf().hdfs_kerberos_principal() <<
" to "
+ << hdfs_info.build_conf().hdfs_kerberos_principal();
+ alter_hdfs_info->mutable_build_conf()->set_hdfs_kerberos_principal(
+ hdfs_info.build_conf().hdfs_kerberos_principal());
+ }
+ if (hdfs_info.build_conf().has_user()) {
+ log_msg << " set user from " << alter_hdfs_info->build_conf().user()
<< " to "
+ << hdfs_info.build_conf().user();
+
alter_hdfs_info->mutable_build_conf()->set_user(hdfs_info.build_conf().user());
+ }
+ if (0 != hdfs_info.build_conf().hdfs_confs_size()) {
+ for (const auto& conf : hdfs_info.build_conf().hdfs_confs()) {
+ log_msg << " add hdfs conf key " << conf.key() << " value " <<
conf.value();
+ }
+ alter_hdfs_info->mutable_build_conf()->mutable_hdfs_confs()->Add(
+ hdfs_info.build_conf().hdfs_confs().begin(),
+ hdfs_info.build_conf().hdfs_confs().end());
+ }
+
+ val = alter.SerializeAsString();
+ if (val.empty()) {
+ msg = "failed to serialize";
+ code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR;
+ return -1;
+ }
+
+ txn->put(vault_key, val);
Review Comment:
print the entire new vault info with LOG here before committing.
##########
cloud/src/meta-service/meta_service_resource.cpp:
##########
@@ -510,13 +510,115 @@ static void set_default_vault_log_helper(const
InstanceInfoPB& instance,
LOG(INFO) << vault_msg;
}
+static int alter_hdfs_storage_vault(InstanceInfoPB& instance,
std::unique_ptr<Transaction> txn,
+ const StorageVaultPB& vault,
MetaServiceCode& code,
+ std::string& msg) {
+ if (!vault.has_hdfs_info()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ std::stringstream ss;
+ ss << "There is no hdfs vault provided";
+ msg = ss.str();
+ return -1;
+ }
+ const auto& hdfs_info = vault.hdfs_info();
+ if (hdfs_info.has_prefix() || !hdfs_info.has_build_conf() ||
+ hdfs_info.build_conf().has_fs_name()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ std::stringstream ss;
+ ss << "You can not alter prefix or fs name";
+ msg = ss.str();
+ return -1;
+ }
+ const auto& name = vault.name();
+ // Here we try to get mutable iter since we might need to alter the vault
name
+ auto name_itr =
std::find_if(instance.mutable_storage_vault_names()->begin(),
+ instance.mutable_storage_vault_names()->end(),
+ [&](const auto& vault_name) { return name ==
vault_name; });
+ if (name_itr == instance.storage_vault_names().end()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ std::stringstream ss;
+ ss << "invalid storage vault name, not found, name =" << name;
+ msg = ss.str();
+ return -1;
+ }
+ auto pos = name_itr - instance.storage_vault_names().begin();
+ auto id_itr = instance.resource_ids().begin() + pos;
Review Comment:
just make a copy of it to make the code more clear. e.g.
std::string vault_id = instance.resource_ids().begin()[pos];
do not manipulate the pointer/iterator thingy, it saves nothing and brings
extra complexity and hard to read.
##########
cloud/src/meta-service/meta_service_resource.cpp:
##########
@@ -510,13 +510,115 @@ static void set_default_vault_log_helper(const
InstanceInfoPB& instance,
LOG(INFO) << vault_msg;
}
+static int alter_hdfs_storage_vault(InstanceInfoPB& instance,
std::unique_ptr<Transaction> txn,
+ const StorageVaultPB& vault,
MetaServiceCode& code,
+ std::string& msg) {
+ if (!vault.has_hdfs_info()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ std::stringstream ss;
+ ss << "There is no hdfs vault provided";
+ msg = ss.str();
+ return -1;
+ }
+ const auto& hdfs_info = vault.hdfs_info();
+ if (hdfs_info.has_prefix() || !hdfs_info.has_build_conf() ||
+ hdfs_info.build_conf().has_fs_name()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ std::stringstream ss;
+ ss << "You can not alter prefix or fs name";
+ msg = ss.str();
+ return -1;
+ }
+ const auto& name = vault.name();
+ // Here we try to get mutable iter since we might need to alter the vault
name
+ auto name_itr =
std::find_if(instance.mutable_storage_vault_names()->begin(),
+ instance.mutable_storage_vault_names()->end(),
+ [&](const auto& vault_name) { return name ==
vault_name; });
+ if (name_itr == instance.storage_vault_names().end()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ std::stringstream ss;
+ ss << "invalid storage vault name, not found, name =" << name;
+ msg = ss.str();
+ return -1;
+ }
+ auto pos = name_itr - instance.storage_vault_names().begin();
+ auto id_itr = instance.resource_ids().begin() + pos;
+ auto vault_key = storage_vault_key({instance.instance_id(), *id_itr});
+ std::string val;
+
+ auto err = txn->get(vault_key, &val);
+ LOG(INFO) << "get instance_key=" << hex(vault_key);
+
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::READ>(err);
+ std::stringstream ss;
+ ss << "failed to get storage vault, vault_id=" << *name_itr << ",
vault_name="
+ << "" << name << " err=" << err;
+ msg = ss.str();
+ return -1;
+ }
+ StorageVaultPB alter;
Review Comment:
naming `new_vault` instead of `alter`.
and print the entire vault original info with LOG after parsing.
##########
cloud/src/meta-service/meta_service_resource.cpp:
##########
Review Comment:
We need UT to for this file
##########
cloud/src/meta-service/meta_service_resource.cpp:
##########
@@ -510,13 +510,115 @@ static void set_default_vault_log_helper(const
InstanceInfoPB& instance,
LOG(INFO) << vault_msg;
}
+static int alter_hdfs_storage_vault(InstanceInfoPB& instance,
std::unique_ptr<Transaction> txn,
+ const StorageVaultPB& vault,
MetaServiceCode& code,
+ std::string& msg) {
+ if (!vault.has_hdfs_info()) {
Review Comment:
we only support changing user and password/auth
reject the rest altering request here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
