This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new e41543b6f2 [filesystem][s3] Remove access key and secret from required
options in S3Loader (#8679)
e41543b6f2 is described below
commit e41543b6f270a7f688581801be3a3d9c5feb0813
Author: Qingsheng Ren <[email protected]>
AuthorDate: Thu Jul 16 07:16:19 2026 +0200
[filesystem][s3] Remove access key and secret from required options in
S3Loader (#8679)
This pull request adds comprehensive integration tests for the S3 file
system implementation, improves test infrastructure for S3
authentication scenarios, and simplifies the S3 plugin loader to better
support credential-less authentication.
---
paimon-filesystems/paimon-s3-impl/pom.xml | 14 ++
.../org/apache/paimon/s3/MinioTestContainer.java | 186 +++++++++++++++++++++
.../java/org/apache/paimon/s3/S3FileIOTest.java | 140 ++++++++++++++++
.../main/java/org/apache/paimon/s3/S3Loader.java | 11 --
.../java/org/apache/paimon/s3/S3LoaderTest.java | 43 +++++
5 files changed, 383 insertions(+), 11 deletions(-)
diff --git a/paimon-filesystems/paimon-s3-impl/pom.xml
b/paimon-filesystems/paimon-s3-impl/pom.xml
index 5f69dec498..886af231f4 100644
--- a/paimon-filesystems/paimon-s3-impl/pom.xml
+++ b/paimon-filesystems/paimon-s3-impl/pom.xml
@@ -186,6 +186,20 @@
<!-- packaged as an optional dependency that is only accessible on
Java 11+ -->
<scope>provided</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-common</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-test-utils</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git
a/paimon-filesystems/paimon-s3-impl/src/test/java/org/apache/paimon/s3/MinioTestContainer.java
b/paimon-filesystems/paimon-s3-impl/src/test/java/org/apache/paimon/s3/MinioTestContainer.java
new file mode 100644
index 0000000000..23936149ab
--- /dev/null
+++
b/paimon-filesystems/paimon-s3-impl/src/test/java/org/apache/paimon/s3/MinioTestContainer.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.s3;
+
+import org.apache.paimon.testutils.junit.DockerImageVersions;
+import org.apache.paimon.utils.Preconditions;
+
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.client.builder.AwsClientBuilder;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.securitytoken.AWSSecurityTokenService;
+import
com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder;
+import com.amazonaws.services.securitytoken.model.AssumeRoleRequest;
+import com.amazonaws.services.securitytoken.model.Credentials;
+import com.github.dockerjava.api.command.InspectContainerResponse;
+import org.junit.jupiter.api.extension.AfterAllCallback;
+import org.junit.jupiter.api.extension.BeforeAllCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.wait.strategy.HttpWaitStrategy;
+import org.testcontainers.utility.Base58;
+
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+
+/**
+ * {@code MinioTestContainer} provides a {@code Minio} test instance for
{@code paimon-s3-impl}.
+ *
+ * <p>This is a copy of the fixture in {@code paimon-s3}; it is duplicated
here because {@code
+ * paimon-s3-impl} is an upstream module and cannot depend on {@code
paimon-s3}'s test-jar.
+ */
+public class MinioTestContainer extends GenericContainer<MinioTestContainer>
+ implements BeforeAllCallback, AfterAllCallback {
+
+ private static final String PAIMON_CONFIG_S3_ENDPOINT = "s3.endpoint";
+
+ private static final int DEFAULT_PORT = 9000;
+
+ private static final String MINIO_ACCESS_KEY = "MINIO_ROOT_USER";
+ private static final String MINIO_SECRET_KEY = "MINIO_ROOT_PASSWORD";
+
+ private static final String DEFAULT_STORAGE_DIRECTORY = "/data";
+ private static final String HEALTH_ENDPOINT = "/minio/health/ready";
+
+ private final String accessKey;
+ private final String secretKey;
+ private final String defaultBucketName;
+
+ public MinioTestContainer() {
+ this(randomString("bucket", 6));
+ }
+
+ public MinioTestContainer(String defaultBucketName) {
+ super(DockerImageVersions.MINIO);
+
+ this.accessKey = randomString("accessKey", 10);
+ // secrets must have at least 8 characters
+ this.secretKey = randomString("secret", 10);
+ this.defaultBucketName = Preconditions.checkNotNull(defaultBucketName);
+
+ withNetworkAliases(randomString("minio", 6));
+ addExposedPort(DEFAULT_PORT);
+ withEnv(MINIO_ACCESS_KEY, this.accessKey);
+ withEnv(MINIO_SECRET_KEY, this.secretKey);
+ withCommand("server", DEFAULT_STORAGE_DIRECTORY);
+ setWaitStrategy(
+ new HttpWaitStrategy()
+ .forPort(DEFAULT_PORT)
+ .forPath(HEALTH_ENDPOINT)
+ .withStartupTimeout(Duration.ofMinutes(2)));
+ }
+
+ @Override
+ protected void containerIsStarted(InspectContainerResponse containerInfo) {
+ super.containerIsStarted(containerInfo);
+ createDefaultBucket();
+ }
+
+ private static String randomString(String prefix, int length) {
+ return String.format("%s-%s", prefix,
Base58.randomString(length).toLowerCase(Locale.ROOT));
+ }
+
+ /** Creates {@link AmazonS3} client for accessing the {@code Minio}
instance. */
+ private AmazonS3 getClient() {
+ return AmazonS3Client.builder()
+ .withCredentials(
+ new AWSStaticCredentialsProvider(
+ new BasicAWSCredentials(accessKey, secretKey)))
+ .withPathStyleAccessEnabled(true)
+ .withEndpointConfiguration(
+ new AwsClientBuilder.EndpointConfiguration(
+ getHttpEndpoint(), "unused-region"))
+ .build();
+ }
+
+ private String getHttpEndpoint() {
+ return String.format("http://%s:%s";, getHost(),
getMappedPort(DEFAULT_PORT));
+ }
+
+ /** Paimon S3 options wired to this container, including the root
access/secret key. */
+ public Map<String, String> getS3ConfigOptions() {
+ Map<String, String> config = new HashMap<>();
+ config.put(PAIMON_CONFIG_S3_ENDPOINT, getHttpEndpoint());
+ config.put("s3.path.style.access", "true");
+ config.put("s3.access.key", accessKey);
+ config.put("s3.secret.key", secretKey);
+ return config;
+ }
+
+ private void createDefaultBucket() {
+ getClient().createBucket(defaultBucketName);
+ }
+
+ /**
+ * Paimon S3 options carrying temporary credentials (access key, secret
key and session token)
+ * issued by MinIO's STS endpoint via {@code AssumeRole}. Intended to
exercise the S3A {@code
+ * TemporaryAWSCredentialsProvider}.
+ */
+ public Map<String, String> getS3ConfigOptionsWithSessionToken() {
+ AWSSecurityTokenService sts =
+ AWSSecurityTokenServiceClientBuilder.standard()
+ .withCredentials(
+ new AWSStaticCredentialsProvider(
+ new BasicAWSCredentials(accessKey,
secretKey)))
+ .withEndpointConfiguration(
+ new AwsClientBuilder.EndpointConfiguration(
+ getHttpEndpoint(), "unused-region"))
+ .build();
+ try {
+ Credentials credentials =
+ sts.assumeRole(
+ new AssumeRoleRequest()
+
.withRoleArn("arn:aws:iam::123456789012:role/paimon")
+ .withRoleSessionName("paimon-test")
+ .withDurationSeconds(900))
+ .getCredentials();
+ Map<String, String> config = new HashMap<>();
+ config.put(PAIMON_CONFIG_S3_ENDPOINT, getHttpEndpoint());
+ config.put("s3.path.style.access", "true");
+ config.put("s3.access.key", credentials.getAccessKeyId());
+ config.put("s3.secret.key", credentials.getSecretAccessKey());
+ config.put("s3.session.token", credentials.getSessionToken());
+ return config;
+ } finally {
+ sts.shutdown();
+ }
+ }
+
+ /**
+ * Returns the S3 URI for the default bucket. This can be used to create
the HA storage
+ * directory path.
+ */
+ public String getS3UriForDefaultBucket() {
+ return "s3://" + defaultBucketName;
+ }
+
+ @Override
+ public void afterAll(ExtensionContext extensionContext) throws Exception {
+ super.close();
+ }
+
+ @Override
+ public void beforeAll(ExtensionContext extensionContext) throws Exception {
+ super.start();
+ }
+}
diff --git
a/paimon-filesystems/paimon-s3-impl/src/test/java/org/apache/paimon/s3/S3FileIOTest.java
b/paimon-filesystems/paimon-s3-impl/src/test/java/org/apache/paimon/s3/S3FileIOTest.java
new file mode 100644
index 0000000000..51e6f0c6f7
--- /dev/null
+++
b/paimon-filesystems/paimon-s3-impl/src/test/java/org/apache/paimon/s3/S3FileIOTest.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.s3;
+
+import org.apache.paimon.catalog.CatalogContext;
+import org.apache.paimon.fs.FileIO;
+import org.apache.paimon.fs.FileIOBehaviorTestBase;
+import org.apache.paimon.fs.Path;
+import org.apache.paimon.options.Options;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Behavior tests for {@link S3FileIO}, backed by a MinIO container. Exercises
the file system
+ * contract with credentials and, separately, credential-less (anonymous)
access.
+ */
+class S3FileIOTest extends FileIOBehaviorTestBase {
+
+ private static final String TEMPORARY_PROVIDER =
+ "org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider";
+
+ private static final String DEFAULT_PROVIDER =
+
"software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider";
+
+ @RegisterExtension
+ private static final MinioTestContainer MINIO_CONTAINER = new
MinioTestContainer();
+
+ @Override
+ protected FileIO getFileSystem() {
+ return createFileIO(MINIO_CONTAINER.getS3ConfigOptions());
+ }
+
+ @Override
+ protected Path getBasePath() {
+ return new Path(MINIO_CONTAINER.getS3UriForDefaultBucket() + "/test");
+ }
+
+ private static S3FileIO createFileIO(Map<String, String> options) {
+ S3FileIO fileIO = new S3FileIO();
+ fileIO.configure(CatalogContext.create(Options.fromMap(options)));
+ return fileIO;
+ }
+
+ /** Basic round-trip using long-lived access key / secret key credentials.
*/
+ @Test
+ void testAccessKeyAndSecretKey() throws Exception {
+ FileIO fileIO = createFileIO(MINIO_CONTAINER.getS3ConfigOptions());
+ assertThat(fileIO.isObjectStore()).isTrue();
+
+ Path file = new Path(getBasePath(), "keyed-" + randomName());
+ fileIO.writeFile(file, "keyed-payload", true);
+
+ assertThat(fileIO.exists(file)).isTrue();
+ assertThat(fileIO.readFileUtf8(file)).isEqualTo("keyed-payload");
+
+ fileIO.delete(file, false);
+ assertThat(fileIO.exists(file)).isFalse();
+ }
+
+ /**
+ * Round-trip using temporary credentials (access key, secret key and
session token) issued by
+ * MinIO's STS endpoint, resolved through the S3A {@code
TemporaryAWSCredentialsProvider}.
+ */
+ @Test
+ void testSessionTokenAuth() throws Exception {
+ Map<String, String> options =
+ new
HashMap<>(MINIO_CONTAINER.getS3ConfigOptionsWithSessionToken());
+ options.put("s3.aws.credentials.provider", TEMPORARY_PROVIDER);
+
+ FileIO fileIO = createFileIO(options);
+ assertThat(fileIO.isObjectStore()).isTrue();
+
+ Path file = new Path(getBasePath(), "sts-" + randomName());
+ fileIO.writeFile(file, "sts-payload", true);
+
+ assertThat(fileIO.exists(file)).isTrue();
+ assertThat(fileIO.readFileUtf8(file)).isEqualTo("sts-payload");
+
+ fileIO.delete(file, false);
+ assertThat(fileIO.exists(file)).isFalse();
+ }
+
+ /**
+ * S3 must work without any credentials in the Paimon options, discovering
them through the AWS
+ * default credential provider chain. Production supplies them via
environment variables; a unit
+ * test cannot mutate the JVM environment, so the equivalent JVM system
properties are used
+ * instead — the default chain reads both.
+ */
+ @Test
+ void testDefaultCredentialsProviderChain() throws Exception {
+ Map<String, String> base = MINIO_CONTAINER.getS3ConfigOptions();
+
+ System.setProperty("aws.accessKeyId", base.get("s3.access.key"));
+ System.setProperty("aws.secretAccessKey", base.get("s3.secret.key"));
+ try {
+ Map<String, String> options = new HashMap<>();
+ options.put("s3.endpoint", base.get("s3.endpoint"));
+ options.put("s3.path.style.access", "true");
+ // No access/secret key here: let the AWS default chain discover
the credentials.
+ options.put("s3.aws.credentials.provider", DEFAULT_PROVIDER);
+
+ FileIO fileIO = createFileIO(options);
+ assertThat(fileIO.isObjectStore()).isTrue();
+
+ Path file = new Path(getBasePath(), "default-chain-" +
randomName());
+ fileIO.writeFile(file, "default-chain-payload", true);
+
+ assertThat(fileIO.exists(file)).isTrue();
+
assertThat(fileIO.readFileUtf8(file)).isEqualTo("default-chain-payload");
+
+ fileIO.delete(file, false);
+ assertThat(fileIO.exists(file)).isFalse();
+ } finally {
+ System.clearProperty("aws.accessKeyId");
+ System.clearProperty("aws.secretAccessKey");
+ }
+ }
+}
diff --git
a/paimon-filesystems/paimon-s3/src/main/java/org/apache/paimon/s3/S3Loader.java
b/paimon-filesystems/paimon-s3/src/main/java/org/apache/paimon/s3/S3Loader.java
index 71d76d684b..c93dd2702f 100644
---
a/paimon-filesystems/paimon-s3/src/main/java/org/apache/paimon/s3/S3Loader.java
+++
b/paimon-filesystems/paimon-s3/src/main/java/org/apache/paimon/s3/S3Loader.java
@@ -25,9 +25,6 @@ import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.PluginFileIO;
import org.apache.paimon.plugin.PluginLoader;
-import java.util.ArrayList;
-import java.util.List;
-
/** A {@link PluginLoader} to load s3. */
public class S3Loader implements FileIOLoader {
@@ -54,14 +51,6 @@ public class S3Loader implements FileIOLoader {
return "s3";
}
- @Override
- public List<String[]> requiredOptions() {
- List<String[]> options = new ArrayList<>();
- options.add(new String[] {"s3.access-key", "s3.access.key"});
- options.add(new String[] {"s3.secret-key", "s3.secret.key"});
- return options;
- }
-
@Override
public FileIO load(Path path) {
return new S3PluginFileIO();
diff --git
a/paimon-filesystems/paimon-s3/src/test/java/org/apache/paimon/s3/S3LoaderTest.java
b/paimon-filesystems/paimon-s3/src/test/java/org/apache/paimon/s3/S3LoaderTest.java
new file mode 100644
index 0000000000..6a768d35c1
--- /dev/null
+++
b/paimon-filesystems/paimon-s3/src/test/java/org/apache/paimon/s3/S3LoaderTest.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.s3;
+
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for {@link S3Loader}. */
+public class S3LoaderTest {
+
+ @Test
+ public void testScheme() {
+ assertThat(new S3Loader().getScheme()).isEqualTo("s3");
+ }
+
+ /**
+ * The S3 loader must not declare any required options. Access key /
secret key are optional so
+ * that credential-less authentication (IAM instance profile, environment
variables, ...) routes
+ * through the bundled S3 plugin instead of being gated out and falling
back to another {@code
+ * FileIO}.
+ */
+ @Test
+ public void testRequiresNoOptions() {
+ assertThat(new S3Loader().requiredOptions()).isEmpty();
+ }
+}
