This is an automated email from the ASF dual-hosted git repository.
zhoujinsong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new e8fde484d [AMORO-4359][AMS] Support AWS Secrets Manager as a
ConfigShade (#4360)
e8fde484d is described below
commit e8fde484d57c945a95e10dce06728521387fa335
Author: wangxianghu <[email protected]>
AuthorDate: Thu Sep 10 07:19:31 2026 +0400
[AMORO-4359][AMS] Support AWS Secrets Manager as a ConfigShade (#4360)
---
LICENSE-binary | 1 +
amoro-ams/pom.xml | 16 ++
.../config/shade/AwsSecretsManagerClient.java | 42 ++++
.../config/shade/AwsSecretsManagerConfigShade.java | 186 +++++++++++++++++
.../shade/DefaultAwsSecretsManagerClient.java | 112 +++++++++++
.../amoro/server/config/shade/SecretReference.java | 136 +++++++++++++
.../org.apache.amoro.config.shade.ConfigShade | 19 ++
.../shade/TestAwsSecretsManagerConfigShade.java | 222 +++++++++++++++++++++
.../server/config/shade/TestSecretReference.java | 120 +++++++++++
dev/deps/dependencies-hadoop-2-spark-3.3 | 1 +
dev/deps/dependencies-hadoop-3-spark-3.5 | 1 +
pom.xml | 6 +
12 files changed, 862 insertions(+)
diff --git a/LICENSE-binary b/LICENSE-binary
index 5a57a2f3a..72192356f 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -328,6 +328,7 @@ software.amazon.awssdk:protocol-core
software.amazon.awssdk:regions
software.amazon.awssdk:s3
software.amazon.awssdk:sdk-core
+software.amazon.awssdk:secretsmanager
software.amazon.awssdk:sts
software.amazon.awssdk:third-party-jackson-core
software.amazon.awssdk:url-connection-client
diff --git a/amoro-ams/pom.xml b/amoro-ams/pom.xml
index 47d4a4823..8cfc07e40 100644
--- a/amoro-ams/pom.xml
+++ b/amoro-ams/pom.xml
@@ -336,6 +336,22 @@
<artifactId>url-connection-client</artifactId>
</dependency>
+ <!-- AWS Secrets Manager config shade (uses url-connection-client
above) -->
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>secretsmanager</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>netty-nio-client</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>apache-client</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>s3-transfer-manager</artifactId>
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/config/shade/AwsSecretsManagerClient.java
b/amoro-ams/src/main/java/org/apache/amoro/server/config/shade/AwsSecretsManagerClient.java
new file mode 100644
index 000000000..2df819c96
--- /dev/null
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/config/shade/AwsSecretsManagerClient.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.server.config.shade;
+
+/**
+ * A minimal abstraction over AWS Secrets Manager. It only exposes a single
method for fetching a
+ * secret string, which makes it easy to stub in unit tests.
+ *
+ * <p>Implementations are expected to cache within the process — AMS only
calls {@link
+ * #getSecretString(String)} during startup, but the same secret may be
referenced in several places
+ * (for example, the username/password of the same DB living in the same JSON).
+ */
+interface AwsSecretsManagerClient extends AutoCloseable {
+
+ /**
+ * Fetches the {@code SecretString} content of the secret.
+ *
+ * @param secretId the secret name or ARN
+ * @return the plaintext of the secret (either a plain string or JSON)
+ * @throws RuntimeException if the fetch fails; the caller decides whether
to fail-fast
+ */
+ String getSecretString(String secretId);
+
+ @Override
+ void close();
+}
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/config/shade/AwsSecretsManagerConfigShade.java
b/amoro-ams/src/main/java/org/apache/amoro/server/config/shade/AwsSecretsManagerConfigShade.java
new file mode 100644
index 000000000..3297160ab
--- /dev/null
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/config/shade/AwsSecretsManagerConfigShade.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.server.config.shade;
+
+import org.apache.amoro.config.Configurations;
+import org.apache.amoro.config.shade.ConfigShade;
+import org.apache.amoro.shade.jackson2.com.fasterxml.jackson.databind.JsonNode;
+import
org.apache.amoro.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.function.Function;
+
+/**
+ * Delegates AMS's sensitive configs (db.username / db.password, etc.) to AWS
Secrets Manager
+ * through Amoro's {@link ConfigShade} SPI.
+ *
+ * <p>Usage (in {@code $AMORO_HOME/conf/config.yaml}):
+ *
+ * <pre>{@code
+ * ams:
+ * shade:
+ * identifier: aws-sm
+ * sensitive-keywords: database.username;database.password
+ * database:
+ * # both point to the same JSON secret; '#<field>' selects the value out
of it
+ * username:
arn:aws:secretsmanager:ap-northeast-1:123456789012:secret:prod-amoro-db-mOhyOp#db.username
+ * password:
arn:aws:secretsmanager:ap-northeast-1:123456789012:secret:prod-amoro-db-mOhyOp#db.password
+ * }</pre>
+ *
+ * <p>The trailing {@code -mOhyOp} is the six random characters AWS appends to
every secret ARN; use
+ * the full ARN as shown. The optional {@code #<field>} suffix selects one
field when the secret
+ * value is JSON, e.g. {@code {"db.username": "amoro", "db.password": "..."}}
— the field name is
+ * whatever key you stored, taken as-is (dots are part of the key, not a
nested path). Omit the
+ * suffix when the secret value is the plaintext itself.
+ *
+ * <p>No explicit region configuration is required — it is resolved from the
ARN. All secrets shaded
+ * for one AMS instance are expected to live in the same region, so a single
client is created
+ * (lazily, memoized within the process); a reference resolving to a different
region is treated as
+ * a misconfiguration and fails fast.
+ *
+ * <p>AMS startup flow: {@code AmoroServiceContainer.initServiceConfig} →
{@code
+ * ConfigShadeUtils.decryptConfig} → find this SPI instance with
identifier=aws-sm → call {@link
+ * #decrypt(String)} for each sensitive key.
+ *
+ * <p>Failure strategy: always fail-fast (malformed ARN, fetch failure,
missing JSON field) so that
+ * AMS startup blows up immediately, instead of continuing with a wrong
password — errors that
+ * surface after the DB receives a wrong password are far from the root cause.
+ */
+public class AwsSecretsManagerConfigShade implements ConfigShade {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(AwsSecretsManagerConfigShade.class);
+
+ public static final String IDENTIFIER = "aws-sm";
+
+ private final ObjectMapper jsonMapper;
+
+ /**
+ * A single client, lazily initialized on the first decrypt() call. All
secrets shaded for one AMS
+ * instance (admin-password, database.password, ...) live in the same
region, so one client is
+ * enough. Guarded by {@code this} for the double-checked lazy init; the
region it was bound to is
+ * validated against every subsequent reference so that a misconfigured
cross-region ARN fails
+ * fast instead of being fetched with the wrong client.
+ */
+ private volatile AwsSecretsManagerClient client;
+
+ private volatile String boundRegion;
+
+ /**
+ * The factory that creates a client; {@link
DefaultAwsSecretsManagerClient#create} for
+ * production, a mock injected in unit tests.
+ */
+ private final Function<String, AwsSecretsManagerClient> clientFactory;
+
+ /** No-arg SPI constructor. */
+ public AwsSecretsManagerConfigShade() {
+ this(new ObjectMapper(), DefaultAwsSecretsManagerClient::create);
+ }
+
+ /** For injecting a mock client factory in unit tests. */
+ AwsSecretsManagerConfigShade(
+ ObjectMapper jsonMapper, Function<String, AwsSecretsManagerClient>
clientFactory) {
+ this.jsonMapper = Objects.requireNonNull(jsonMapper, "jsonMapper");
+ this.clientFactory = Objects.requireNonNull(clientFactory,
"clientFactory");
+ }
+
+ @Override
+ public String getIdentifier() {
+ return IDENTIFIER;
+ }
+
+ @Override
+ public void initialize(Configurations serviceConfig) {
+ // No SPI-level config to read — region is resolved per ARN; credentials
use the AWS default
+ // provider chain.
+ LOG.info("AwsSecretsManagerConfigShade initialized (region resolved
per-secret from ARN)");
+ }
+
+ @Override
+ public String decrypt(String content) {
+ SecretReference ref = SecretReference.parse(content);
+ String raw =
getOrCreateClient(ref.region()).getSecretString(ref.secretArn());
+ return ref.jsonKey().map(key -> extractJsonField(raw, key,
ref.secretArn())).orElse(raw);
+ }
+
+ /**
+ * Returns the shared client, creating it on the first call. All shaded
secrets are expected to
+ * live in a single region; if a later reference resolves to a different
region it is almost
+ * certainly a misconfiguration, so we fail fast rather than fetch it with
the wrong client.
+ */
+ private AwsSecretsManagerClient getOrCreateClient(String region) {
+ AwsSecretsManagerClient existing = client;
+ if (existing == null) {
+ synchronized (this) {
+ existing = client;
+ if (existing == null) {
+ existing = clientFactory.apply(region);
+ boundRegion = region;
+ client = existing;
+ }
+ }
+ }
+ if (!Objects.equals(region, boundRegion)) {
+ throw new IllegalStateException(
+ "all AWS Secrets Manager references must be in the same region;
expected '"
+ + boundRegion
+ + "' but got '"
+ + region
+ + "'");
+ }
+ return existing;
+ }
+
+ private String extractJsonField(String json, String key, String secretArn) {
+ JsonNode root;
+ try {
+ root = jsonMapper.readTree(json);
+ } catch (IOException e) {
+ // Do not log the raw JSON; it may contain other sensitive fields.
+ throw new IllegalStateException(
+ "secret '" + secretArn + "' is not valid JSON; cannot extract field
'" + key + "'", e);
+ }
+ if (root == null || !root.isObject()) {
+ throw new IllegalStateException(
+ "secret '" + secretArn + "' is not a JSON object; cannot extract
field '" + key + "'");
+ }
+ JsonNode value = root.get(key);
+ if (value == null || value.isNull()) {
+ throw new IllegalStateException("secret '" + secretArn + "' has no field
'" + key + "'");
+ }
+ // A config value must be a scalar. For an object/array node asText()
silently returns "", which
+ // would then be handed to the DB as e.g. an empty password — the
far-from-root-cause failure
+ // this class fails fast to avoid. So reject non-scalar values explicitly.
+ if (!value.isValueNode()) {
+ throw new IllegalStateException(
+ "secret '"
+ + secretArn
+ + "' field '"
+ + key
+ + "' must be a scalar value, but is a "
+ + value.getNodeType()
+ + "; a structured JSON value cannot be used as a config value");
+ }
+ // asText() returns strings as-is and converts numbers/booleans to strings
— a good fit for the
+ // password scenario.
+ return value.asText();
+ }
+}
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/config/shade/DefaultAwsSecretsManagerClient.java
b/amoro-ams/src/main/java/org/apache/amoro/server/config/shade/DefaultAwsSecretsManagerClient.java
new file mode 100644
index 000000000..b8a601749
--- /dev/null
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/config/shade/DefaultAwsSecretsManagerClient.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.server.config.shade;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
+import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;
+import
software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest;
+import
software.amazon.awssdk.services.secretsmanager.model.GetSecretValueResponse;
+
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+/**
+ * The default {@link AwsSecretsManagerClient} implementation, bound to a
single region.
+ *
+ * <ul>
+ * <li>Credentials: {@link DefaultCredentialsProvider} — tries env / Java
system props / Web
+ * Identity (EKS IRSA) / ECS / EC2 InstanceProfile / {@code
~/.aws/credentials} in order.
+ * <li>Region: extracted from the ARN by the caller and passed in at
construction time.
+ * <li>HTTP: {@link UrlConnectionHttpClient} — no Netty dependency,
minimizing the shade artifact.
+ * <li>Cache: an in-process {@link ConcurrentHashMap}; each secret is
fetched only once (username
+ * and password usually live in the same JSON and are referenced twice).
+ * </ul>
+ *
+ * <p>On failure it throws {@link IllegalStateException} to make AMS startup
fail-fast, avoiding
+ * starting the service with a wrong password.
+ */
+final class DefaultAwsSecretsManagerClient implements AwsSecretsManagerClient {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(DefaultAwsSecretsManagerClient.class);
+
+ private final SecretsManagerClient delegate;
+ private final String region;
+ private final ConcurrentMap<String, String> cache = new
ConcurrentHashMap<>();
+
+ private DefaultAwsSecretsManagerClient(SecretsManagerClient delegate, String
region) {
+ this.delegate = Objects.requireNonNull(delegate, "delegate");
+ this.region = region;
+ }
+
+ static DefaultAwsSecretsManagerClient create(String region) {
+ Objects.requireNonNull(region, "region");
+ SecretsManagerClient sdkClient =
+ SecretsManagerClient.builder()
+ .region(Region.of(region))
+ .credentialsProvider(DefaultCredentialsProvider.create())
+ .httpClient(UrlConnectionHttpClient.create())
+ .build();
+ LOG.info("AWS Secrets Manager client initialized (region={})", region);
+ return new DefaultAwsSecretsManagerClient(sdkClient, region);
+ }
+
+ @Override
+ public String getSecretString(String secretId) {
+ return cache.computeIfAbsent(secretId, this::fetch);
+ }
+
+ private String fetch(String secretId) {
+ try {
+ GetSecretValueResponse response =
+
delegate.getSecretValue(GetSecretValueRequest.builder().secretId(secretId).build());
+ String value = response.secretString();
+ if (value == null) {
+ // Binary secrets are rarely used for passwords; fail clearly if one
is encountered.
+ throw new IllegalStateException(
+ "secret '" + secretId + "' has no SecretString (binary secrets are
not supported)");
+ }
+ LOG.info("Fetched secret '{}' from AWS Secrets Manager (region={})",
secretId, region);
+ return value;
+ } catch (RuntimeException e) {
+ // Mask the secret value itself, but keep secretId / region for
operational troubleshooting.
+ throw new IllegalStateException(
+ "Failed to fetch secret '"
+ + secretId
+ + "' from AWS Secrets Manager (region="
+ + region
+ + "): "
+ + e.getMessage(),
+ e);
+ }
+ }
+
+ @Override
+ public void close() {
+ try {
+ delegate.close();
+ } catch (RuntimeException e) {
+ LOG.warn("Error closing SecretsManagerClient (region={})", region, e);
+ }
+ }
+}
diff --git
a/amoro-ams/src/main/java/org/apache/amoro/server/config/shade/SecretReference.java
b/amoro-ams/src/main/java/org/apache/amoro/server/config/shade/SecretReference.java
new file mode 100644
index 000000000..cf40454d6
--- /dev/null
+++
b/amoro-ams/src/main/java/org/apache/amoro/server/config/shade/SecretReference.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.server.config.shade;
+
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * Parses an AWS Secrets Manager reference written in {@code config.yaml}.
+ *
+ * <p>Only a full ARN is accepted (short names are not supported); the region
is extracted directly
+ * from the ARN to avoid additional configuration.
+ *
+ * <p>Syntax:
+ *
+ * <pre>{@code
+ * arn:aws:secretsmanager:<region>:<account>:secret:<name>[-<suffix>]
+ * arn:aws:secretsmanager:<region>:<account>:secret:<name>[-<suffix>]#<jsonKey>
+ * }</pre>
+ *
+ * <p>Examples:
+ *
+ * <ul>
+ * <li>{@code
arn:aws:secretsmanager:ap-northeast-1:123456789012:secret:prod/amoro/admin-AbCdEf}
—
+ * the whole secret is the password string.
+ * <li>{@code
+ *
arn:aws:secretsmanager:ap-northeast-1:123456789012:secret:prod/amoro/rds-AbCdEf#password}
—
+ * the secret is JSON, and the {@code password} field is extracted.
+ * </ul>
+ *
+ * <p>The delimiter is fixed to {@code #} to avoid the {@code :} that fills
ARNs and the {@code /}
+ * commonly found in secret names.
+ */
+final class SecretReference {
+
+ private static final char KEY_DELIMITER = '#';
+
+ /** ARN fixed format: {@code
arn:aws:secretsmanager:REGION:ACCOUNT:secret:NAME} — 7 parts. */
+ private static final int ARN_PARTS = 7;
+
+ private static final String ARN_PREFIX = "arn:";
+ private static final String ARN_SERVICE = "secretsmanager";
+ private static final String ARN_RESOURCE_TYPE = "secret";
+
+ private final String secretArn;
+ private final String region;
+ private final String jsonKey; // nullable
+
+ private SecretReference(String secretArn, String region, String jsonKey) {
+ this.secretArn = secretArn;
+ this.region = region;
+ this.jsonKey = jsonKey;
+ }
+
+ static SecretReference parse(String content) {
+ Objects.requireNonNull(content, "secret reference content is null");
+ String trimmed = content.trim();
+ if (trimmed.isEmpty()) {
+ throw new IllegalArgumentException("secret reference is blank");
+ }
+
+ // Split off the optional '#jsonKey' suffix first.
+ String arnPart;
+ String jsonKey;
+ int hashIdx = trimmed.lastIndexOf(KEY_DELIMITER);
+ if (hashIdx < 0) {
+ arnPart = trimmed;
+ jsonKey = null;
+ } else {
+ arnPart = trimmed.substring(0, hashIdx).trim();
+ String tail = trimmed.substring(hashIdx + 1).trim();
+ jsonKey = tail.isEmpty() ? null : tail;
+ }
+
+ validateArn(arnPart);
+ String region = extractRegion(arnPart);
+ return new SecretReference(arnPart, region, jsonKey);
+ }
+
+ private static void validateArn(String arn) {
+ if (!arn.startsWith(ARN_PREFIX)) {
+ throw new IllegalArgumentException(
+ "secret reference must be a full ARN
(arn:aws:secretsmanager:<region>:<account>:secret:<name>)");
+ }
+ // limit=ARN_PARTS ensures that an illegal ':' inside the secret name is
not over-split.
+ String[] parts = arn.split(":", ARN_PARTS);
+ if (parts.length != ARN_PARTS
+ || !ARN_SERVICE.equals(parts[2])
+ || !ARN_RESOURCE_TYPE.equals(parts[5])
+ || parts[3].isEmpty()
+ || parts[6].isEmpty()) {
+ throw new IllegalArgumentException(
+ "malformed AWS Secrets Manager ARN (expected
arn:aws:secretsmanager:<region>:<account>:secret:<name>)");
+ }
+ }
+
+ private static String extractRegion(String arn) {
+ // At this point the ARN has passed validateArn and is guaranteed to have
7 parts.
+ return arn.split(":", ARN_PARTS)[3];
+ }
+
+ /** The full ARN — pass it directly to {@code
GetSecretValueRequest#secretId}. */
+ String secretArn() {
+ return secretArn;
+ }
+
+ /** The region extracted from the ARN; used to select/create the {@code
SecretsManagerClient}. */
+ String region() {
+ return region;
+ }
+
+ Optional<String> jsonKey() {
+ return Optional.ofNullable(jsonKey);
+ }
+
+ @Override
+ public String toString() {
+ return jsonKey == null ? secretArn : secretArn + KEY_DELIMITER + jsonKey;
+ }
+}
diff --git
a/amoro-ams/src/main/resources/META-INF/services/org.apache.amoro.config.shade.ConfigShade
b/amoro-ams/src/main/resources/META-INF/services/org.apache.amoro.config.shade.ConfigShade
new file mode 100644
index 000000000..8e58374e3
--- /dev/null
+++
b/amoro-ams/src/main/resources/META-INF/services/org.apache.amoro.config.shade.ConfigShade
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+# *
+# http://www.apache.org/licenses/LICENSE-2.0
+# *
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.amoro.server.config.shade.AwsSecretsManagerConfigShade
diff --git
a/amoro-ams/src/test/java/org/apache/amoro/server/config/shade/TestAwsSecretsManagerConfigShade.java
b/amoro-ams/src/test/java/org/apache/amoro/server/config/shade/TestAwsSecretsManagerConfigShade.java
new file mode 100644
index 000000000..67a68deb0
--- /dev/null
+++
b/amoro-ams/src/test/java/org/apache/amoro/server/config/shade/TestAwsSecretsManagerConfigShade.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.server.config.shade;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import
org.apache.amoro.shade.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.jupiter.api.Test;
+
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+
+/** Unit tests for {@link AwsSecretsManagerConfigShade} decryption logic with
a mocked client. */
+public class TestAwsSecretsManagerConfigShade {
+
+ /**
+ * The common case: one secret holds a JSON blob with many key-value pairs,
and each config value
+ * selects a field with {@code #<field>}. Most tests use this ARN.
+ */
+ private static final String DB_SECRET_ARN =
+
"arn:aws:secretsmanager:ap-northeast-1:123456789012:secret:prod-amoro-db-mOhyOp";
+
+ /**
+ * A secret whose value is a single plaintext string (no {@code #});
exercises the non-JSON
+ * branch.
+ */
+ private static final String PLAINTEXT_SECRET_ARN =
+
"arn:aws:secretsmanager:ap-northeast-1:123456789012:secret:prod-amoro-admin-AbCdEf";
+
+ /** A secret in a DIFFERENT region, used only to drive the cross-region
fail-fast test. */
+ private static final String OTHER_REGION_SECRET_ARN =
+
"arn:aws:secretsmanager:us-east-1:123456789012:secret:prod-amoro-other-XyZ123";
+
+ @Test
+ public void testIdentifier() {
+ assertEquals("aws-sm", new AwsSecretsManagerConfigShade().getIdentifier());
+ }
+
+ @Test
+ public void testDecryptPlainSecret() {
+ AwsSecretsManagerClient client = mock(AwsSecretsManagerClient.class);
+
when(client.getSecretString(PLAINTEXT_SECRET_ARN)).thenReturn("super-secret-password");
+
+ AwsSecretsManagerConfigShade shade =
+ new AwsSecretsManagerConfigShade(new ObjectMapper(), region -> client);
+
+ assertEquals("super-secret-password", shade.decrypt(PLAINTEXT_SECRET_ARN));
+ verify(client).getSecretString(PLAINTEXT_SECRET_ARN);
+ }
+
+ @Test
+ public void testDecryptJsonField() {
+ // Username and password live in the same JSON secret, selected by dotted
field names.
+ AwsSecretsManagerClient client = mock(AwsSecretsManagerClient.class);
+ when(client.getSecretString(DB_SECRET_ARN))
+ .thenReturn("{\"db.username\":\"amoro\",\"db.password\":\"p@ss\"}");
+
+ AwsSecretsManagerConfigShade shade =
+ new AwsSecretsManagerConfigShade(new ObjectMapper(), region -> client);
+
+ assertEquals("amoro", shade.decrypt(DB_SECRET_ARN + "#db.username"));
+ assertEquals("p@ss", shade.decrypt(DB_SECRET_ARN + "#db.password"));
+ }
+
+ @Test
+ public void testDottedFieldNameIsTakenAsIsNotNestedPath() {
+ // A '.' in the field name is part of the key, not a nested-path
separator: a flat key with dots
+ // resolves, while the same dots interpreted as nesting must not.
+ AwsSecretsManagerClient client = mock(AwsSecretsManagerClient.class);
+ when(client.getSecretString(DB_SECRET_ARN))
+
.thenReturn("{\"db.password\":\"flat\",\"db\":{\"password\":\"nested\"}}");
+
+ AwsSecretsManagerConfigShade shade =
+ new AwsSecretsManagerConfigShade(new ObjectMapper(), region -> client);
+
+ // Must match the flat key literally, never drill into the nested object.
+ assertEquals("flat", shade.decrypt(DB_SECRET_ARN + "#db.password"));
+ }
+
+ @Test
+ public void testDecryptJsonFieldConvertsNonStringValue() {
+ AwsSecretsManagerClient client = mock(AwsSecretsManagerClient.class);
+
when(client.getSecretString(DB_SECRET_ARN)).thenReturn("{\"db.port\":5432}");
+
+ AwsSecretsManagerConfigShade shade =
+ new AwsSecretsManagerConfigShade(new ObjectMapper(), region -> client);
+
+ assertEquals("5432", shade.decrypt(DB_SECRET_ARN + "#db.port"));
+ }
+
+ @Test
+ public void testClientIsMemoized() {
+ AtomicInteger created = new AtomicInteger();
+ AwsSecretsManagerClient client = mock(AwsSecretsManagerClient.class);
+
when(client.getSecretString(DB_SECRET_ARN)).thenReturn("{\"db.password\":\"p@ss\"}");
+ Function<String, AwsSecretsManagerClient> factory =
+ region -> {
+ created.incrementAndGet();
+ return client;
+ };
+
+ AwsSecretsManagerConfigShade shade =
+ new AwsSecretsManagerConfigShade(new ObjectMapper(), factory);
+
+ shade.decrypt(DB_SECRET_ARN + "#db.password");
+ shade.decrypt(DB_SECRET_ARN + "#db.password");
+ // Multiple decrypt calls must reuse a single lazily-created client.
+ assertEquals(1, created.get());
+ verify(client, times(2)).getSecretString(DB_SECRET_ARN);
+ }
+
+ @Test
+ public void testCrossRegionReferenceFailsFast() {
+ AtomicInteger created = new AtomicInteger();
+ AwsSecretsManagerClient client = mock(AwsSecretsManagerClient.class);
+ when(client.getSecretString(PLAINTEXT_SECRET_ARN)).thenReturn("v1");
+ Function<String, AwsSecretsManagerClient> factory =
+ region -> {
+ created.incrementAndGet();
+ return client;
+ };
+
+ AwsSecretsManagerConfigShade shade =
+ new AwsSecretsManagerConfigShade(new ObjectMapper(), factory);
+
+ shade.decrypt(PLAINTEXT_SECRET_ARN); // ap-northeast-1 binds the client
+ // A second reference in a different region is treated as a
misconfiguration.
+ assertThrows(
+ IllegalStateException.class, () ->
shade.decrypt(OTHER_REGION_SECRET_ARN)); // us-east-1
+ // The client is created only once and never rebound to the second region.
+ assertEquals(1, created.get());
+ }
+
+ @Test
+ public void testMissingJsonFieldFailsFast() {
+ AwsSecretsManagerClient client = mock(AwsSecretsManagerClient.class);
+
when(client.getSecretString(DB_SECRET_ARN)).thenReturn("{\"db.username\":\"amoro\"}");
+
+ AwsSecretsManagerConfigShade shade =
+ new AwsSecretsManagerConfigShade(new ObjectMapper(), region -> client);
+
+ assertThrows(IllegalStateException.class, () ->
shade.decrypt(DB_SECRET_ARN + "#db.password"));
+ }
+
+ @Test
+ public void testJsonFieldPointingToObjectFailsFast() {
+ // '#db' selects an object node. Jackson's asText() would silently return
"" for it, so without
+ // the scalar check the DB would receive an empty value; the shade must
fail fast instead.
+ AwsSecretsManagerClient client = mock(AwsSecretsManagerClient.class);
+ when(client.getSecretString(DB_SECRET_ARN))
+ .thenReturn("{\"db\":{\"username\":\"amoro\",\"password\":\"p@ss\"}}");
+
+ AwsSecretsManagerConfigShade shade =
+ new AwsSecretsManagerConfigShade(new ObjectMapper(), region -> client);
+
+ assertThrows(IllegalStateException.class, () ->
shade.decrypt(DB_SECRET_ARN + "#db"));
+ }
+
+ @Test
+ public void testJsonFieldPointingToArrayFailsFast() {
+ // Same trap as an object node: '#hosts' selects an array, for which
asText() returns "".
+ AwsSecretsManagerClient client = mock(AwsSecretsManagerClient.class);
+
when(client.getSecretString(DB_SECRET_ARN)).thenReturn("{\"hosts\":[\"a\",\"b\"]}");
+
+ AwsSecretsManagerConfigShade shade =
+ new AwsSecretsManagerConfigShade(new ObjectMapper(), region -> client);
+
+ assertThrows(IllegalStateException.class, () ->
shade.decrypt(DB_SECRET_ARN + "#hosts"));
+ }
+
+ @Test
+ public void testInvalidJsonFailsFast() {
+ AwsSecretsManagerClient client = mock(AwsSecretsManagerClient.class);
+ when(client.getSecretString(DB_SECRET_ARN)).thenReturn("not-a-json");
+
+ AwsSecretsManagerConfigShade shade =
+ new AwsSecretsManagerConfigShade(new ObjectMapper(), region -> client);
+
+ assertThrows(IllegalStateException.class, () ->
shade.decrypt(DB_SECRET_ARN + "#db.password"));
+ }
+
+ @Test
+ public void testNonObjectJsonFailsFast() {
+ AwsSecretsManagerClient client = mock(AwsSecretsManagerClient.class);
+ when(client.getSecretString(DB_SECRET_ARN)).thenReturn("[1,2,3]");
+
+ AwsSecretsManagerConfigShade shade =
+ new AwsSecretsManagerConfigShade(new ObjectMapper(), region -> client);
+
+ assertThrows(IllegalStateException.class, () ->
shade.decrypt(DB_SECRET_ARN + "#db.password"));
+ }
+
+ @Test
+ public void testMalformedReferenceFailsFast() {
+ AwsSecretsManagerConfigShade shade =
+ new AwsSecretsManagerConfigShade(
+ new ObjectMapper(), region -> mock(AwsSecretsManagerClient.class));
+
+ assertThrows(IllegalArgumentException.class, () ->
shade.decrypt("not-an-arn"));
+ }
+}
diff --git
a/amoro-ams/src/test/java/org/apache/amoro/server/config/shade/TestSecretReference.java
b/amoro-ams/src/test/java/org/apache/amoro/server/config/shade/TestSecretReference.java
new file mode 100644
index 000000000..6c7382d40
--- /dev/null
+++
b/amoro-ams/src/test/java/org/apache/amoro/server/config/shade/TestSecretReference.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.amoro.server.config.shade;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.junit.jupiter.api.Test;
+
+/** Unit tests for {@link SecretReference} ARN parsing and validation. */
+public class TestSecretReference {
+
+ private static final String DB_SECRET_ARN =
+
"arn:aws:secretsmanager:ap-northeast-1:123456789012:secret:prod-amoro-db-mOhyOp";
+
+ @Test
+ public void testParsePlainArn() {
+ SecretReference ref = SecretReference.parse(DB_SECRET_ARN);
+ assertEquals(DB_SECRET_ARN, ref.secretArn());
+ assertEquals("ap-northeast-1", ref.region());
+ assertFalse(ref.jsonKey().isPresent());
+ }
+
+ @Test
+ public void testParseArnWithJsonKey() {
+ SecretReference ref = SecretReference.parse(DB_SECRET_ARN +
"#db.password");
+ // The '#jsonKey' suffix must be stripped from the ARN passed to the SDK.
+ assertEquals(DB_SECRET_ARN, ref.secretArn());
+ assertEquals("ap-northeast-1", ref.region());
+ assertTrue(ref.jsonKey().isPresent());
+ // A dotted field name is kept intact: '#' is split on lastIndexOf, dots
are part of the key.
+ assertEquals("db.password", ref.jsonKey().get());
+ }
+
+ @Test
+ public void testParseTrimsSurroundingWhitespace() {
+ SecretReference ref = SecretReference.parse(" " + DB_SECRET_ARN + " #
db.password ");
+ assertEquals(DB_SECRET_ARN, ref.secretArn());
+ assertEquals("db.password", ref.jsonKey().get());
+ }
+
+ @Test
+ public void testEmptyJsonKeyTreatedAsAbsent() {
+ // A trailing '#' with no key should behave like a plain ARN reference.
+ SecretReference ref = SecretReference.parse(DB_SECRET_ARN + "#");
+ assertEquals(DB_SECRET_ARN, ref.secretArn());
+ assertFalse(ref.jsonKey().isPresent());
+ }
+
+ @Test
+ public void testToStringRoundTrip() {
+ assertEquals(DB_SECRET_ARN,
SecretReference.parse(DB_SECRET_ARN).toString());
+ assertEquals(
+ DB_SECRET_ARN + "#db.password",
+ SecretReference.parse(DB_SECRET_ARN + "#db.password").toString());
+ }
+
+ @Test
+ public void testNullContentRejected() {
+ assertThrows(NullPointerException.class, () ->
SecretReference.parse(null));
+ }
+
+ @Test
+ public void testBlankContentRejected() {
+ assertThrows(IllegalArgumentException.class, () -> SecretReference.parse("
"));
+ }
+
+ @Test
+ public void testNonArnRejected() {
+ assertThrows(
+ IllegalArgumentException.class, () ->
SecretReference.parse("just-a-plain-secret-name"));
+ }
+
+ @Test
+ public void testWrongServiceRejected() {
+ // Correct ARN shape but not a secretsmanager resource.
+ assertThrows(
+ IllegalArgumentException.class,
+ () ->
SecretReference.parse("arn:aws:s3:ap-northeast-1:123456789012:secret:foo"));
+ }
+
+ @Test
+ public void testWrongResourceTypeRejected() {
+ assertThrows(
+ IllegalArgumentException.class,
+ () ->
SecretReference.parse("arn:aws:secretsmanager:ap-northeast-1:123456789012:key:foo"));
+ }
+
+ @Test
+ public void testMissingRegionRejected() {
+ assertThrows(
+ IllegalArgumentException.class,
+ () ->
SecretReference.parse("arn:aws:secretsmanager::123456789012:secret:foo"));
+ }
+
+ @Test
+ public void testMissingNameRejected() {
+ assertThrows(
+ IllegalArgumentException.class,
+ () ->
SecretReference.parse("arn:aws:secretsmanager:ap-northeast-1:123456789012:secret:"));
+ }
+}
diff --git a/dev/deps/dependencies-hadoop-2-spark-3.3
b/dev/deps/dependencies-hadoop-2-spark-3.3
index d1fa96bf9..9703bcca0 100644
--- a/dev/deps/dependencies-hadoop-2-spark-3.3
+++ b/dev/deps/dependencies-hadoop-2-spark-3.3
@@ -382,6 +382,7 @@
scala-parser-combinators_2.12/1.1.2//scala-parser-combinators_2.12-1.1.2.jar
scala-reflect/2.12.15//scala-reflect-2.12.15.jar
scala-xml_2.12/1.2.0//scala-xml_2.12-1.2.0.jar
sdk-core/2.24.12//sdk-core-2.24.12.jar
+secretsmanager/2.24.12//secretsmanager-2.24.12.jar
simpleclient/0.16.0//simpleclient-0.16.0.jar
simpleclient_common/0.16.0//simpleclient_common-0.16.0.jar
simpleclient_httpserver/0.16.0//simpleclient_httpserver-0.16.0.jar
diff --git a/dev/deps/dependencies-hadoop-3-spark-3.5
b/dev/deps/dependencies-hadoop-3-spark-3.5
index c5f4fe97d..4f45caf2e 100644
--- a/dev/deps/dependencies-hadoop-3-spark-3.5
+++ b/dev/deps/dependencies-hadoop-3-spark-3.5
@@ -349,6 +349,7 @@
scala-parser-combinators_2.12/2.3.0//scala-parser-combinators_2.12-2.3.0.jar
scala-reflect/2.12.18//scala-reflect-2.12.18.jar
scala-xml_2.12/2.1.0//scala-xml_2.12-2.1.0.jar
sdk-core/2.24.12//sdk-core-2.24.12.jar
+secretsmanager/2.24.12//secretsmanager-2.24.12.jar
simpleclient/0.16.0//simpleclient-0.16.0.jar
simpleclient_common/0.16.0//simpleclient_common-0.16.0.jar
simpleclient_httpserver/0.16.0//simpleclient_httpserver-0.16.0.jar
diff --git a/pom.xml b/pom.xml
index 2855d1e3a..fab56dfdb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -741,6 +741,12 @@
<version>${awssdk.version}</version>
</dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>secretsmanager</artifactId>
+ <version>${awssdk.version}</version>
+ </dependency>
+
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>url-connection-client</artifactId>
