This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new dd6e3fc0 fix(core): resolve storage config aliases to canonical keys
(#540)
dd6e3fc0 is described below
commit dd6e3fc09402b4c9afd819490ccd4b28aeda3e8b
Author: Wuhen- Li <[email protected]>
AuthorDate: Mon Jul 20 11:07:27 2026 +0800
fix(core): resolve storage config aliases to canonical keys (#540)
---
crates/paimon/src/io/storage_azdls.rs | 72 ++++++++++++++++++++++++++--------
crates/paimon/src/io/storage_config.rs | 32 +++++----------
crates/paimon/src/io/storage_cos.rs | 6 +--
crates/paimon/src/io/storage_gcs.rs | 45 ++++++++++-----------
crates/paimon/src/io/storage_obs.rs | 4 +-
crates/paimon/src/io/storage_s3.rs | 23 +++++------
6 files changed, 102 insertions(+), 80 deletions(-)
diff --git a/crates/paimon/src/io/storage_azdls.rs
b/crates/paimon/src/io/storage_azdls.rs
index 8a71cdd4..3c2a36af 100644
--- a/crates/paimon/src/io/storage_azdls.rs
+++ b/crates/paimon/src/io/storage_azdls.rs
@@ -32,21 +32,23 @@ const AZURE_ACCOUNT_KEY: &str = "azure.account-key";
const AZURE_SAS_TOKEN: &str = "azure.sas-token";
const CONFIG_PREFIXES: &[&str] = &["fs.azure.", "fs.abfs.", "abfs.", "abfss.",
"azure."];
-const MIRRORED_KEYS: &[(&str, &str)] = &[
- ("azure.account-name", "azure.account.name"),
- ("azure.account_name", "azure.account.name"),
- ("azure.account-key", "azure.account.key"),
- ("azure.account_key", "azure.account.key"),
- ("azure.sas-token", "azure.sas.token"),
- ("azure.sas_token", "azure.sas.token"),
- ("azure.client-id", "azure.client.id"),
- ("azure.client_id", "azure.client.id"),
- ("azure.client-secret", "azure.client.secret"),
- ("azure.client_secret", "azure.client.secret"),
- ("azure.tenant-id", "azure.tenant.id"),
- ("azure.tenant_id", "azure.tenant.id"),
- ("azure.authority-host", "azure.authority.host"),
- ("azure.authority_host", "azure.authority.host"),
+// Aliases for each canonical key are ordered from highest to lowest priority.
+// An explicitly supplied canonical key always takes precedence over its
aliases.
+const KEY_ALIASES: &[(&str, &str)] = &[
+ ("azure.account.name", "azure.account-name"),
+ ("azure.account_name", "azure.account-name"),
+ ("azure.account.key", "azure.account-key"),
+ ("azure.account_key", "azure.account-key"),
+ ("azure.sas.token", "azure.sas-token"),
+ ("azure.sas_token", "azure.sas-token"),
+ ("azure.client.id", "azure.client-id"),
+ ("azure.client_id", "azure.client-id"),
+ ("azure.client.secret", "azure.client-secret"),
+ ("azure.client_secret", "azure.client-secret"),
+ ("azure.tenant.id", "azure.tenant-id"),
+ ("azure.tenant_id", "azure.tenant-id"),
+ ("azure.authority.host", "azure.authority-host"),
+ ("azure.authority_host", "azure.authority-host"),
];
#[derive(Debug, Clone)]
@@ -56,7 +58,7 @@ pub struct AzdlsStorageConfig {
}
pub(crate) fn azdls_config_parse(props: HashMap<String, String>) ->
Result<AzdlsStorageConfig> {
- let normalized = normalize_storage_config(props, CONFIG_PREFIXES,
"azure.", MIRRORED_KEYS);
+ let normalized = normalize_storage_config(props, CONFIG_PREFIXES,
"azure.", KEY_ALIASES);
let config = config_from_normalized(&normalized);
Ok(AzdlsStorageConfig { config, normalized })
@@ -318,6 +320,44 @@ mod tests {
assert_eq!(cfg.config.tenant_id.as_deref(), Some("tenant"));
}
+ #[test]
+ fn test_azdls_config_parse_underscore_aliases() {
+ type ConfigValue = for<'a> fn(&'a AzdlsConfig) -> Option<&'a str>;
+
+ let cases: &[(&str, ConfigValue)] = &[
+ ("azure.account_name", |cfg| cfg.account_name.as_deref()),
+ ("azure.account_key", |cfg| cfg.account_key.as_deref()),
+ ("azure.sas_token", |cfg| cfg.sas_token.as_deref()),
+ ("azure.client_id", |cfg| cfg.client_id.as_deref()),
+ ("azure.client_secret", |cfg| cfg.client_secret.as_deref()),
+ ("azure.tenant_id", |cfg| cfg.tenant_id.as_deref()),
+ ("azure.authority_host", |cfg| cfg.authority_host.as_deref()),
+ ];
+
+ for (alias, config_value) in cases {
+ let cfg = azdls_config_parse(make_props(&[(alias,
"alias-value")])).unwrap();
+ assert_eq!(config_value(&cfg.config), Some("alias-value"),
"{alias}");
+ }
+ }
+
+ #[test]
+ fn test_azdls_config_alias_priority() {
+ let cfg = azdls_config_parse(make_props(&[
+ ("azure.account-name", "canonical"),
+ ("azure.account.name", "dotted"),
+ ("azure.account_name", "underscore"),
+ ]))
+ .unwrap();
+ assert_eq!(cfg.config.account_name.as_deref(), Some("canonical"));
+
+ let cfg = azdls_config_parse(make_props(&[
+ ("azure.account.name", "dotted"),
+ ("azure.account_name", "underscore"),
+ ]))
+ .unwrap();
+ assert_eq!(cfg.config.account_name.as_deref(), Some("dotted"));
+ }
+
#[test]
fn test_azdls_config_uses_account_scoped_hadoop_key() {
let cfg = azdls_config_parse(make_props(&[(
diff --git a/crates/paimon/src/io/storage_config.rs
b/crates/paimon/src/io/storage_config.rs
index 90206db0..d1a35646 100644
--- a/crates/paimon/src/io/storage_config.rs
+++ b/crates/paimon/src/io/storage_config.rs
@@ -21,7 +21,7 @@ pub(super) fn normalize_storage_config(
props: HashMap<String, String>,
config_prefixes: &[&str],
canonical_prefix: &str,
- mirrored_keys: &[(&str, &str)],
+ key_aliases: &[(&str, &str)],
) -> HashMap<String, String> {
let mut result = HashMap::new();
@@ -33,27 +33,15 @@ pub(super) fn normalize_storage_config(
}
}
- let mirrored_additions: Vec<(String, String)> = mirrored_keys
- .iter()
- .flat_map(|(a, b)| {
- let mut pairs = Vec::new();
-
- if !result.contains_key(*b) {
- if let Some(v) = result.get(*a) {
- pairs.push((b.to_string(), v.clone()));
- }
- }
- if !result.contains_key(*a) {
- if let Some(v) = result.get(*b) {
- pairs.push((a.to_string(), v.clone()));
- }
- }
- pairs
- })
- .collect();
-
- for (k, v) in mirrored_additions {
- result.insert(k, v);
+ // Canonical keys always win. Aliases for the same canonical key are
+ // checked in declaration order, so callers can define their priority.
+ for (alias, canonical) in key_aliases {
+ if result.contains_key(*canonical) {
+ continue;
+ }
+ if let Some(value) = result.get(*alias).cloned() {
+ result.insert(canonical.to_string(), value);
+ }
}
result
diff --git a/crates/paimon/src/io/storage_cos.rs
b/crates/paimon/src/io/storage_cos.rs
index b8bb3ea4..ff7b11a6 100644
--- a/crates/paimon/src/io/storage_cos.rs
+++ b/crates/paimon/src/io/storage_cos.rs
@@ -31,8 +31,8 @@ const COS_SECRET_ID: &str = "fs.cosn.userinfo.secretId";
const COS_SECRET_KEY: &str = "fs.cosn.userinfo.secretKey";
const CONFIG_PREFIXES: &[&str] = &["fs.cosn.", "cosn.", "cos."];
-const MIRRORED_KEYS: &[(&str, &str)] = &[
- ("fs.cosn.endpoint", "fs.cosn.userinfo.endpoint"),
+const KEY_ALIASES: &[(&str, &str)] = &[
+ ("fs.cosn.userinfo.endpoint", "fs.cosn.endpoint"),
("fs.cosn.secret_id", "fs.cosn.userinfo.secretId"),
("fs.cosn.secret-id", "fs.cosn.userinfo.secretId"),
("fs.cosn.secret_key", "fs.cosn.userinfo.secretKey"),
@@ -40,7 +40,7 @@ const MIRRORED_KEYS: &[(&str, &str)] = &[
];
pub(crate) fn cos_config_parse(props: HashMap<String, String>) ->
Result<CosConfig> {
- let normalized = normalize_storage_config(props, CONFIG_PREFIXES,
"fs.cosn.", MIRRORED_KEYS);
+ let normalized = normalize_storage_config(props, CONFIG_PREFIXES,
"fs.cosn.", KEY_ALIASES);
let cfg = CosConfig {
endpoint: normalized.get(COS_ENDPOINT).cloned(),
diff --git a/crates/paimon/src/io/storage_gcs.rs
b/crates/paimon/src/io/storage_gcs.rs
index 575dae3f..dd24cb69 100644
--- a/crates/paimon/src/io/storage_gcs.rs
+++ b/crates/paimon/src/io/storage_gcs.rs
@@ -33,34 +33,31 @@ const GCS_SERVICE_ACCOUNT: &str = "gcs.service-account";
const GCS_ALLOW_ANONYMOUS: &str = "gcs.allow-anonymous";
const CONFIG_PREFIXES: &[&str] = &["fs.gs.", "fs.gcs.", "gs.", "gcs."];
-const MIRRORED_KEYS: &[(&str, &str)] = &[
- ("gcs.credential-path", "gcs.google_application_credentials"),
- ("gcs.credential-path", "gcs.google-application-credentials"),
- ("gcs.credential-path", "gcs.application-credentials"),
- ("gcs.credential", "gcs.google_service_account_key"),
- ("gcs.credential", "gcs.google-service-account-key"),
- ("gcs.credential", "gcs.service-account-key"),
- ("gcs.credential", "gcs.service_account_key"),
- ("gcs.service-account", "gcs.google_service_account"),
- ("gcs.service-account", "gcs.google-service-account"),
- ("gcs.service-account", "gcs.service_account"),
- ("gcs.predefined-acl", "gcs.predefined_acl"),
- ("gcs.default-storage-class", "gcs.default_storage_class"),
- ("gcs.allow-anonymous", "gcs.google_skip_signature"),
- ("gcs.allow-anonymous", "gcs.google-skip-signature"),
- ("gcs.allow_anonymous", "gcs.google_skip_signature"),
- ("gcs.allow-anonymous", "gcs.allow_anonymous"),
- ("gcs.allow-anonymous", "gcs.skip-signature"),
- ("gcs.allow-anonymous", "gcs.skip_signature"),
- ("gcs.skip-signature", "gcs.google_skip_signature"),
- ("gcs.skip_signature", "gcs.google_skip_signature"),
- ("gcs.disable-vm-metadata", "gcs.disable_vm_metadata"),
- ("gcs.disable-config-load", "gcs.disable_config_load"),
+const KEY_ALIASES: &[(&str, &str)] = &[
+ ("gcs.google_application_credentials", "gcs.credential-path"),
+ ("gcs.google-application-credentials", "gcs.credential-path"),
+ ("gcs.application-credentials", "gcs.credential-path"),
+ ("gcs.google_service_account_key", "gcs.credential"),
+ ("gcs.google-service-account-key", "gcs.credential"),
+ ("gcs.service-account-key", "gcs.credential"),
+ ("gcs.service_account_key", "gcs.credential"),
+ ("gcs.google_service_account", "gcs.service-account"),
+ ("gcs.google-service-account", "gcs.service-account"),
+ ("gcs.service_account", "gcs.service-account"),
+ ("gcs.predefined_acl", "gcs.predefined-acl"),
+ ("gcs.default_storage_class", "gcs.default-storage-class"),
+ ("gcs.google_skip_signature", "gcs.allow-anonymous"),
+ ("gcs.google-skip-signature", "gcs.allow-anonymous"),
+ ("gcs.allow_anonymous", "gcs.allow-anonymous"),
+ ("gcs.skip-signature", "gcs.allow-anonymous"),
+ ("gcs.skip_signature", "gcs.allow-anonymous"),
+ ("gcs.disable_vm_metadata", "gcs.disable-vm-metadata"),
+ ("gcs.disable_config_load", "gcs.disable-config-load"),
];
#[allow(clippy::field_reassign_with_default)]
pub(crate) fn gcs_config_parse(props: HashMap<String, String>) ->
Result<GcsConfig> {
- let normalized = normalize_storage_config(props, CONFIG_PREFIXES, "gcs.",
MIRRORED_KEYS);
+ let normalized = normalize_storage_config(props, CONFIG_PREFIXES, "gcs.",
KEY_ALIASES);
let mut cfg = GcsConfig::default();
cfg.endpoint = normalized.get(GCS_ENDPOINT).cloned();
diff --git a/crates/paimon/src/io/storage_obs.rs
b/crates/paimon/src/io/storage_obs.rs
index 2fde849e..4e7def74 100644
--- a/crates/paimon/src/io/storage_obs.rs
+++ b/crates/paimon/src/io/storage_obs.rs
@@ -31,7 +31,7 @@ const OBS_ACCESS_KEY_ID: &str = "fs.obs.access.key";
const OBS_SECRET_ACCESS_KEY: &str = "fs.obs.secret.key";
const CONFIG_PREFIXES: &[&str] = &["fs.obs.", "obs."];
-const MIRRORED_KEYS: &[(&str, &str)] = &[
+const KEY_ALIASES: &[(&str, &str)] = &[
("fs.obs.access-key-id", "fs.obs.access.key"),
("fs.obs.access_key_id", "fs.obs.access.key"),
("fs.obs.secret-access-key", "fs.obs.secret.key"),
@@ -40,7 +40,7 @@ const MIRRORED_KEYS: &[(&str, &str)] = &[
#[allow(clippy::field_reassign_with_default)]
pub(crate) fn obs_config_parse(props: HashMap<String, String>) ->
Result<ObsConfig> {
- let normalized = normalize_storage_config(props, CONFIG_PREFIXES,
"fs.obs.", MIRRORED_KEYS);
+ let normalized = normalize_storage_config(props, CONFIG_PREFIXES,
"fs.obs.", KEY_ALIASES);
let mut cfg = ObsConfig::default();
cfg.endpoint = normalized.get(OBS_ENDPOINT).cloned();
diff --git a/crates/paimon/src/io/storage_s3.rs
b/crates/paimon/src/io/storage_s3.rs
index 6fe20b87..19134342 100644
--- a/crates/paimon/src/io/storage_s3.rs
+++ b/crates/paimon/src/io/storage_s3.rs
@@ -63,14 +63,13 @@ const S3_REGION: &str = "s3.region";
/// Reference: `S3FileIO.CONFIG_PREFIXES` in Java Paimon.
const JAVA_CONFIG_PREFIXES: &[&str] = &["fs.s3a.", "s3a.", "s3."];
-/// Mirrored config keys — Java Paimon maps these interchangeably.
-/// Both directions are applied so users can use either form.
+/// External aliases mapped to the canonical keys read by [`S3Config`].
///
/// Reference: `S3FileIO.MIRRORED_CONFIG_KEYS` in Java Paimon.
-const MIRRORED_KEYS: &[(&str, &str)] = &[
- ("s3.access-key", "s3.access.key"),
- ("s3.secret-key", "s3.secret.key"),
- ("s3.path-style-access", "s3.path.style.access"),
+const KEY_ALIASES: &[(&str, &str)] = &[
+ ("s3.access.key", "s3.access-key"),
+ ("s3.secret.key", "s3.secret-key"),
+ ("s3.path.style.access", "s3.path-style-access"),
];
/// Parse paimon catalog options into an [`S3Config`].
@@ -83,7 +82,7 @@ const MIRRORED_KEYS: &[(&str, &str)] = &[
/// to path-style for S3-compatible stores like MinIO.
#[allow(clippy::field_reassign_with_default)]
pub(crate) fn s3_config_parse(props: HashMap<String, String>) ->
Result<S3Config> {
- let normalized = normalize_storage_config(props, JAVA_CONFIG_PREFIXES,
"s3.", MIRRORED_KEYS);
+ let normalized = normalize_storage_config(props, JAVA_CONFIG_PREFIXES,
"s3.", KEY_ALIASES);
let mut cfg = S3Config::default();
@@ -186,7 +185,7 @@ mod tests {
cfg.endpoint.as_deref(),
Some("https://s3.eu-west-1.amazonaws.com";)
);
- // `fs.s3a.access.key` → `s3.access.key`, then mirrored →
`s3.access-key`
+ // `fs.s3a.access.key` → `s3.access.key`, then aliased →
`s3.access-key`
assert_eq!(cfg.access_key_id.as_deref(), Some("AKID2"));
assert_eq!(cfg.secret_access_key.as_deref(), Some("SECRET2"));
}
@@ -340,11 +339,9 @@ mod tests {
}
#[test]
- fn test_mirrored_keys() {
- // `s3.access.key` (dot form) should be mirrored from `s3.access-key`
(dash form)
- let props = make_props(&[("s3.access-key", "AKID")]);
- let normalized =
- normalize_storage_config(props, JAVA_CONFIG_PREFIXES, "s3.",
MIRRORED_KEYS);
+ fn test_key_aliases() {
+ let props = make_props(&[("s3.access.key", "AKID")]);
+ let normalized = normalize_storage_config(props, JAVA_CONFIG_PREFIXES,
"s3.", KEY_ALIASES);
assert_eq!(
normalized.get("s3.access.key").map(|s| s.as_str()),
Some("AKID")
