This is an automated email from the ASF dual-hosted git repository.
MisterRaindrop pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/cloudberry-pxf.git
The following commit(s) were added to refs/heads/main by this push:
new 934f9995 Replace MinIO S3 sidecar with RustFS (#163)
934f9995 is described below
commit 934f999593af96752ede3c25c440d56eb0f3742e
Author: Nikolay Antonov <[email protected]>
AuthorDate: Wed Sep 30 17:48:06 2026 +0500
Replace MinIO S3 sidecar with RustFS (#163)
- pin the RustFS 1.0.0 image by digest
- update S3 testcontainer configuration, fixtures, and error expectations
- add distinct aliases for unused S3 Select projection columns
---
automation/Makefile | 2 +-
.../expected/query01.ans | 6 ++---
.../sql/query01.sql | 4 ++--
.../expected/query01.ans | 4 ++--
.../sql/query01.sql | 2 +-
.../expected/query01.ans | 6 ++---
.../sql/query01.sql | 4 ++--
.../pxf/automation/applications/S3Application.java | 10 ++++----
.../{MinIOContainer.java => RustFSContainer.java} | 28 +++++++++++-----------
.../pxf-cbdb/servers/s3-invalid/s3-site.xml | 4 ++--
.../testcontainers/pxf-cbdb/servers/s3/s3-site.xml | 6 ++---
.../automation/features/cloud/CloudAccessTest.java | 28 +++++++++++-----------
.../automation/features/cloud/S3SelectTest.java | 18 +++++++-------
ci/docker/pxf-cbdb-dev/common/script/run_tests.sh | 2 +-
.../pxf/plugins/s3/S3SelectQueryBuilder.java | 4 +++-
.../pxf/plugins/s3/S3SelectQueryBuilderTest.java | 11 +++++++++
16 files changed, 77 insertions(+), 62 deletions(-)
diff --git a/automation/Makefile b/automation/Makefile
index 0ea85cdf..137f6d82 100755
--- a/automation/Makefile
+++ b/automation/Makefile
@@ -233,7 +233,7 @@ endif
# Usage:
# make test-tc => run all testcontainers tests
(Ubuntu)
# make test-tc TC_GROUP=pxf-jdbc => run only pxf-jdbc group
-# make test-tc TC_GROUP=pxf-s3 => run S3 Select + CloudAccess tests
(MinIO sidecar)
+# make test-tc TC_GROUP=pxf-s3 => run S3 Select + CloudAccess tests
(RustFS sidecar)
# make test-tc DISTRO=rocky9 => run with Rocky Linux 9 base image
.PHONY: test-tc
test-tc: check-env symlink_pxf_jars pxf_regress
diff --git
a/automation/sqlrepo/features/cloud_access/server_no_credentials_invalid_config/expected/query01.ans
b/automation/sqlrepo/features/cloud_access/server_no_credentials_invalid_config/expected/query01.ans
index 4d5dcfb2..daf33c53 100644
---
a/automation/sqlrepo/features/cloud_access/server_no_credentials_invalid_config/expected/query01.ans
+++
b/automation/sqlrepo/features/cloud_access/server_no_credentials_invalid_config/expected/query01.ans
@@ -6,8 +6,8 @@
--
-- # create a match/subs
--
--- m/PXF server error.*(com.amazonaws.services.s3.model.AmazonS3Exception:
Forbidden).*/
--- s/PXF server error.*/PXF server error :
com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden/
+-- m/PXF server error.*(com.amazonaws.services.s3.model.AmazonS3Exception:
(Forbidden|The Access Key Id you provided does not exist in our records)).*/
+-- s/PXF server error.*/PXF server error :
com.amazonaws.services.s3.model.AmazonS3Exception: Invalid credentials/
--
-- m/DETAIL/
-- s/DETAIL/CONTEXT/
@@ -23,7 +23,7 @@
--
-- end_matchsubs
SELECT * FROM cloudaccess_server_no_credentials_invalid_config;
-ERROR: PXF server error : com.amazonaws.services.s3.model.AmazonS3Exception:
Forbidden
+ERROR: PXF server error : com.amazonaws.services.s3.model.AmazonS3Exception:
Invalid credentials
-- start_ignore
HINT: Check the PXF logs located in the 'logs-dir' directory on host 'mdw' or
'set client_min_messages=LOG' for additional details.
-- end_ignore
diff --git
a/automation/sqlrepo/features/cloud_access/server_no_credentials_invalid_config/sql/query01.sql
b/automation/sqlrepo/features/cloud_access/server_no_credentials_invalid_config/sql/query01.sql
index 1ff81a67..0debe73b 100644
---
a/automation/sqlrepo/features/cloud_access/server_no_credentials_invalid_config/sql/query01.sql
+++
b/automation/sqlrepo/features/cloud_access/server_no_credentials_invalid_config/sql/query01.sql
@@ -6,8 +6,8 @@
--
-- # create a match/subs
--
--- m/PXF server error.*(com.amazonaws.services.s3.model.AmazonS3Exception:
Forbidden).*/
--- s/PXF server error.*/PXF server error :
com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden/
+-- m/PXF server error.*(com.amazonaws.services.s3.model.AmazonS3Exception:
(Forbidden|The Access Key Id you provided does not exist in our records)).*/
+-- s/PXF server error.*/PXF server error :
com.amazonaws.services.s3.model.AmazonS3Exception: Invalid credentials/
--
-- m/DETAIL/
-- s/DETAIL/CONTEXT/
diff --git
a/automation/sqlrepo/features/cloud_access/server_no_credentials_invalid_config_with_hdfs/expected/query01.ans
b/automation/sqlrepo/features/cloud_access/server_no_credentials_invalid_config_with_hdfs/expected/query01.ans
index 76e02e02..2cca85ce 100644
---
a/automation/sqlrepo/features/cloud_access/server_no_credentials_invalid_config_with_hdfs/expected/query01.ans
+++
b/automation/sqlrepo/features/cloud_access/server_no_credentials_invalid_config_with_hdfs/expected/query01.ans
@@ -7,7 +7,7 @@
-- # create a match/subs
--
-- m/PXF server error.*(doesBucketExist|com.amazonaws).*/
--- s/PXF server error.*/PXF server error :
com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden/
+-- s/PXF server error.*/PXF server error :
com.amazonaws.services.s3.model.AmazonS3Exception: Invalid credentials/
--
-- m/DETAIL/
-- s/DETAIL/CONTEXT/
@@ -29,7 +29,7 @@
--
-- end_matchsubs
SELECT * FROM cloudaccess_server_no_credentials_invalid_config_with_hdfs;
-ERROR: PXF server error : com.amazonaws.services.s3.model.AmazonS3Exception:
Forbidden
+ERROR: PXF server error : com.amazonaws.services.s3.model.AmazonS3Exception:
Invalid credentials
-- start_ignore
HINT: Check the PXF logs located in the 'logs-dir' directory on host 'mdw' or
'set client_min_messages=LOG' for additional details.
-- end_ignore
diff --git
a/automation/sqlrepo/features/cloud_access/server_no_credentials_invalid_config_with_hdfs/sql/query01.sql
b/automation/sqlrepo/features/cloud_access/server_no_credentials_invalid_config_with_hdfs/sql/query01.sql
index 92309109..c4134416 100644
---
a/automation/sqlrepo/features/cloud_access/server_no_credentials_invalid_config_with_hdfs/sql/query01.sql
+++
b/automation/sqlrepo/features/cloud_access/server_no_credentials_invalid_config_with_hdfs/sql/query01.sql
@@ -7,7 +7,7 @@
-- # create a match/subs
--
-- m/PXF server error.*(doesBucketExist|com.amazonaws).*/
--- s/PXF server error.*/PXF server error :
com.amazonaws.services.s3.model.AmazonS3Exception: Forbidden/
+-- s/PXF server error.*/PXF server error :
com.amazonaws.services.s3.model.AmazonS3Exception: Invalid credentials/
--
-- m/DETAIL/
-- s/DETAIL/CONTEXT/
diff --git
a/automation/sqlrepo/features/s3_select/errors/csv_use_headers_with_wrong_col_names/expected/query01.ans
b/automation/sqlrepo/features/s3_select/errors/csv_use_headers_with_wrong_col_names/expected/query01.ans
index d945310d..7cdbb9d1 100644
---
a/automation/sqlrepo/features/s3_select/errors/csv_use_headers_with_wrong_col_names/expected/query01.ans
+++
b/automation/sqlrepo/features/s3_select/errors/csv_use_headers_with_wrong_col_names/expected/query01.ans
@@ -12,8 +12,8 @@
-- m/file:.*;/
-- s/file:.*; lineNumber: \d+; columnNumber: \d+;/SOME_ERROR_LOCATION/g
--
--- m/Some headers in the query are missing from the file.*/
--- s/Some headers in the query are missing from the file.*/Some headers in the
query are missing from the file. Please check the file and try again./
+-- m/PXF server error.*(A column name or a path provided does not exist in the
SQL expression|S3 returned an error: column .* not found).*/
+-- s/PXF server error.*/PXF server error : S3 Select column not found/
--
-- m/DETAIL/
-- s/DETAIL/CONTEXT/
@@ -26,7 +26,7 @@
--
-- end_matchsubs
SELECT * FROM s3select_csv_use_headers_with_wrong_col_names;
-ERROR: PXF server error : S3 returned an error: column invalid_orderkey not
found (InternalError)
+ERROR: PXF server error : S3 Select column not found
-- start_ignore
HINT: Check the PXF logs located in the '/home/gpadmin/pxf-base/logs'
directory on host '0.0.0.0' or 'set client_min_messages=LOG' for additional
details.
-- end_ignore
diff --git
a/automation/sqlrepo/features/s3_select/errors/csv_use_headers_with_wrong_col_names/sql/query01.sql
b/automation/sqlrepo/features/s3_select/errors/csv_use_headers_with_wrong_col_names/sql/query01.sql
index b16c563d..a7e8530f 100644
---
a/automation/sqlrepo/features/s3_select/errors/csv_use_headers_with_wrong_col_names/sql/query01.sql
+++
b/automation/sqlrepo/features/s3_select/errors/csv_use_headers_with_wrong_col_names/sql/query01.sql
@@ -12,8 +12,8 @@
-- m/file:.*;/
-- s/file:.*; lineNumber: \d+; columnNumber: \d+;/SOME_ERROR_LOCATION/g
--
--- m/Some headers in the query are missing from the file.*/
--- s/Some headers in the query are missing from the file.*/Some headers in the
query are missing from the file. Please check the file and try again./
+-- m/PXF server error.*(A column name or a path provided does not exist in the
SQL expression|S3 returned an error: column .* not found).*/
+-- s/PXF server error.*/PXF server error : S3 Select column not found/
--
-- m/DETAIL/
-- s/DETAIL/CONTEXT/
diff --git
a/automation/src/main/java/org/apache/cloudberry/pxf/automation/applications/S3Application.java
b/automation/src/main/java/org/apache/cloudberry/pxf/automation/applications/S3Application.java
index a92f3f39..cc50a4aa 100644
---
a/automation/src/main/java/org/apache/cloudberry/pxf/automation/applications/S3Application.java
+++
b/automation/src/main/java/org/apache/cloudberry/pxf/automation/applications/S3Application.java
@@ -29,7 +29,7 @@ import com.amazonaws.services.s3.model.ListObjectsV2Request;
import com.amazonaws.services.s3.model.ListObjectsV2Result;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3ObjectSummary;
-import org.apache.cloudberry.pxf.automation.testcontainers.MinIOContainer;
+import org.apache.cloudberry.pxf.automation.testcontainers.RustFSContainer;
import java.io.IOException;
import java.nio.file.Path;
@@ -38,16 +38,16 @@ import java.util.List;
/**
* S3 API access wrapper used by automation tests to seed and clean fixtures
- * in a MinIO bucket. Owns the AmazonS3 client; callers should call
+ * in a RustFS bucket. Owns the AmazonS3 client; callers should call
* `shutdown()` when done (typically in afterClass before stopping the
- * MinIO container).
+ * RustFS container).
*/
public class S3Application implements AutoCloseable {
private final AmazonS3 s3Client;
- public S3Application(MinIOContainer minio) {
- this.s3Client = buildS3Client(minio.getHostEndpoint(),
minio.getAccessKey(), minio.getSecretKey());
+ public S3Application(RustFSContainer s3Server) {
+ this.s3Client = buildS3Client(s3Server.getHostEndpoint(),
s3Server.getAccessKey(), s3Server.getSecretKey());
}
public void createBucket(String bucket) {
diff --git
a/automation/src/main/java/org/apache/cloudberry/pxf/automation/testcontainers/MinIOContainer.java
b/automation/src/main/java/org/apache/cloudberry/pxf/automation/testcontainers/RustFSContainer.java
similarity index 69%
rename from
automation/src/main/java/org/apache/cloudberry/pxf/automation/testcontainers/MinIOContainer.java
rename to
automation/src/main/java/org/apache/cloudberry/pxf/automation/testcontainers/RustFSContainer.java
index 11f5c3f1..57ebb83a 100644
---
a/automation/src/main/java/org/apache/cloudberry/pxf/automation/testcontainers/MinIOContainer.java
+++
b/automation/src/main/java/org/apache/cloudberry/pxf/automation/testcontainers/RustFSContainer.java
@@ -25,37 +25,37 @@ import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.utility.DockerImageName;
/**
- * TestContainers wrapper around MinIO for S3 / S3 Select automation tests.
- * The container joins a shared Docker network with alias minio, so PXF inside
the
- * Cloudberry container can reach it at http://minio:9000.
+ * TestContainers wrapper around RustFS for S3 / S3 Select automation tests.
+ * The container joins a shared Docker network with alias rustfs, so PXF
inside the
+ * Cloudberry container can reach it at http://rustfs:9000.
*
* This class only manages the container lifecycle and exposes endpoint /
* credential accessors. S3 API access (buckets, objects) lives in
* {@link org.apache.cloudberry.pxf.automation.applications.S3Application}.
*/
-public class MinIOContainer extends GenericContainer<MinIOContainer> {
+public class RustFSContainer extends GenericContainer<RustFSContainer> {
- private static final String DEFAULT_IMAGE =
"quay.io/minio/minio:RELEASE.2024-11-07T00-52-20Z";
- private static final String NETWORK_ALIAS = "minio";
+ private static final String DEFAULT_IMAGE =
"rustfs/rustfs:1.0.0@sha256:8cc9801755448b71a786705ce76692c77e14936cccd87cf2fc31842e58f4d1ff";
+ private static final String NETWORK_ALIAS = "rustfs";
public static final int API_PORT = 9000;
- public static final int CONSOLE_PORT = 9001;
public static final String ACCESS_KEY = "admin";
public static final String SECRET_KEY = "password";
public static final String DEFAULT_BUCKET = "gpdb-ud-scratch";
- public MinIOContainer(Network network) {
+ public RustFSContainer(Network network) {
super(DockerImageName.parse(DEFAULT_IMAGE));
withNetwork(network)
.withNetworkAliases(NETWORK_ALIAS)
- .withExposedPorts(API_PORT, CONSOLE_PORT)
- .withEnv("MINIO_ROOT_USER", ACCESS_KEY)
- .withEnv("MINIO_ROOT_PASSWORD", SECRET_KEY)
- .withEnv("MINIO_API_SELECT_PARQUET", "on")
- .withCommand("server", "/data", "--console-address", ":" +
CONSOLE_PORT)
-
.waitingFor(Wait.forHttp("/minio/health/live").forPort(API_PORT));
+ .withExposedPorts(API_PORT)
+ .withEnv("RUSTFS_ACCESS_KEY", ACCESS_KEY)
+ .withEnv("RUSTFS_SECRET_KEY", SECRET_KEY)
+ .withEnv("RUSTFS_ADDRESS", ":" + API_PORT)
+ .withEnv("RUSTFS_CONSOLE_ENABLE", "false")
+ .withCommand("/data")
+ .waitingFor(Wait.forHttp("/health/ready").forPort(API_PORT));
}
/** S3 API endpoint reachable from the test JVM (mapped port). */
diff --git
a/automation/src/main/resources/testcontainers/pxf-cbdb/servers/s3-invalid/s3-site.xml
b/automation/src/main/resources/testcontainers/pxf-cbdb/servers/s3-invalid/s3-site.xml
index b7557be3..70ca1754 100644
---
a/automation/src/main/resources/testcontainers/pxf-cbdb/servers/s3-invalid/s3-site.xml
+++
b/automation/src/main/resources/testcontainers/pxf-cbdb/servers/s3-invalid/s3-site.xml
@@ -13,7 +13,7 @@
<!--
Pre-baked PXF server 's3-invalid' configuration for testcontainers
automation.
- Endpoint points at the MinIO sidecar, but the credentials are deliberately
+ Endpoint points at the RustFS sidecar, but the credentials are deliberately
invalid. Used by CloudAccessTest to verify PXF credential-resolution
precedence: a server config with bad keys should surface an authentication
failure unless the external table provides overriding credentials (in
@@ -25,7 +25,7 @@
<configuration>
<property>
<name>fs.s3a.endpoint</name>
- <value>http://minio:9000</value>
+ <value>http://rustfs:9000</value>
</property>
<property>
<name>fs.s3a.access.key</name>
diff --git
a/automation/src/main/resources/testcontainers/pxf-cbdb/servers/s3/s3-site.xml
b/automation/src/main/resources/testcontainers/pxf-cbdb/servers/s3/s3-site.xml
index b76f19bc..a84e8339 100644
---
a/automation/src/main/resources/testcontainers/pxf-cbdb/servers/s3/s3-site.xml
+++
b/automation/src/main/resources/testcontainers/pxf-cbdb/servers/s3/s3-site.xml
@@ -13,15 +13,15 @@
<!--
Pre-baked PXF server 's3' configuration for testcontainers automation.
- Pointed at the MinIO sidecar that joins the shared Docker network with
- alias 'minio' (see MinIOContainer).
+ Pointed at the RustFS sidecar that joins the shared Docker network with
+ alias 'rustfs' (see RustFSContainer).
Shipped as-is and copied into ${PXF_BASE}/servers/s3/ by entrypoint.sh.
-->
<configuration>
<property>
<name>fs.s3a.endpoint</name>
- <value>http://minio:9000</value>
+ <value>http://rustfs:9000</value>
</property>
<property>
<name>fs.s3a.access.key</name>
diff --git
a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/cloud/CloudAccessTest.java
b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/cloud/CloudAccessTest.java
index 6fcdb3e3..a0304589 100644
---
a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/cloud/CloudAccessTest.java
+++
b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/cloud/CloudAccessTest.java
@@ -26,7 +26,7 @@ import
org.apache.cloudberry.pxf.automation.applications.PXFApplication;
import org.apache.cloudberry.pxf.automation.applications.S3Application;
import
org.apache.cloudberry.pxf.automation.structures.tables.pxf.ExternalTable;
import
org.apache.cloudberry.pxf.automation.structures.tables.utils.TableFactory;
-import org.apache.cloudberry.pxf.automation.testcontainers.MinIOContainer;
+import org.apache.cloudberry.pxf.automation.testcontainers.RustFSContainer;
import org.testng.annotations.Test;
import java.io.IOException;
@@ -57,7 +57,7 @@ public class CloudAccessTest extends
AbstractTestcontainersTest {
private final SmallDataFactory dataFactory = new SmallDataFactory();
- private MinIOContainer s3Server;
+ private RustFSContainer s3Server;
private S3Application s3Application;
private String s3PathRead;
private String s3PathWrite;
@@ -66,26 +66,26 @@ public class CloudAccessTest extends
AbstractTestcontainersTest {
@Override
public void beforeClass() throws Exception {
- s3Server = new MinIOContainer(container.getSharedNetwork());
+ s3Server = new RustFSContainer(container.getSharedNetwork());
s3Server.start();
s3Application = new S3Application(s3Server);
- s3Application.createBucket(MinIOContainer.DEFAULT_BUCKET);
+ s3Application.createBucket(RustFSContainer.DEFAULT_BUCKET);
String random = UUID.randomUUID().toString();
readObjectKeyPrefix =
String.format("tmp/pxf_automation_data_read/%s/", random);
writeObjectKeyPrefix =
String.format("tmp/pxf_automation_data_write/%s/", random);
- s3PathRead = MinIOContainer.DEFAULT_BUCKET + "/" + readObjectKeyPrefix;
- s3PathWrite = MinIOContainer.DEFAULT_BUCKET + "/" +
writeObjectKeyPrefix;
+ s3PathRead = RustFSContainer.DEFAULT_BUCKET + "/" +
readObjectKeyPrefix;
+ s3PathWrite = RustFSContainer.DEFAULT_BUCKET + "/" +
writeObjectKeyPrefix;
}
@Override
public void afterClass() throws Exception {
if (s3Application != null) {
if (readObjectKeyPrefix != null) {
- s3Application.deletePrefix(MinIOContainer.DEFAULT_BUCKET,
readObjectKeyPrefix);
+ s3Application.deletePrefix(RustFSContainer.DEFAULT_BUCKET,
readObjectKeyPrefix);
}
if (writeObjectKeyPrefix != null) {
- s3Application.deletePrefix(MinIOContainer.DEFAULT_BUCKET,
writeObjectKeyPrefix);
+ s3Application.deletePrefix(RustFSContainer.DEFAULT_BUCKET,
writeObjectKeyPrefix);
}
s3Application.shutdown();
}
@@ -96,7 +96,7 @@ public class CloudAccessTest extends
AbstractTestcontainersTest {
@Override
protected void beforeMethod() throws Exception {
- uploadSmallCsvFixture(MinIOContainer.DEFAULT_BUCKET,
readObjectKeyPrefix + fileName);
+ uploadSmallCsvFixture(RustFSContainer.DEFAULT_BUCKET,
readObjectKeyPrefix + fileName);
}
// Uploads small CSV test data (see BaseTCFunctionality#getSmallData()) to
the given S3 object.
@@ -113,8 +113,8 @@ public class CloudAccessTest extends
AbstractTestcontainersTest {
@Override
protected void afterMethod() throws Exception {
if (s3Application != null) {
- s3Application.deletePrefix(MinIOContainer.DEFAULT_BUCKET,
readObjectKeyPrefix);
- s3Application.deletePrefix(MinIOContainer.DEFAULT_BUCKET,
writeObjectKeyPrefix);
+ s3Application.deletePrefix(RustFSContainer.DEFAULT_BUCKET,
readObjectKeyPrefix);
+ s3Application.deletePrefix(RustFSContainer.DEFAULT_BUCKET,
writeObjectKeyPrefix);
}
}
@@ -207,7 +207,7 @@ public class CloudAccessTest extends
AbstractTestcontainersTest {
String serverParam = (server == null) ? null : "server=" + server;
exTable.setServer(serverParam);
if (creds) {
- exTable.setUserParameters(new String[]{"accesskey=" +
MinIOContainer.ACCESS_KEY, "secretkey=" + MinIOContainer.SECRET_KEY});
+ exTable.setUserParameters(new String[]{"accesskey=" +
RustFSContainer.ACCESS_KEY, "secretkey=" + RustFSContainer.SECRET_KEY});
}
cloudberry.createTableAndVerify(exTable);
@@ -222,7 +222,7 @@ public class CloudAccessTest extends
AbstractTestcontainersTest {
String serverParam = (server == null) ? null : "server=" + server;
exTable.setServer(serverParam);
if (creds) {
- exTable.setUserParameters(new String[]{"accesskey=" +
MinIOContainer.ACCESS_KEY, "secretkey=" + MinIOContainer.SECRET_KEY});
+ exTable.setUserParameters(new String[]{"accesskey=" +
RustFSContainer.ACCESS_KEY, "secretkey=" + RustFSContainer.SECRET_KEY});
}
cloudberry.createTableAndVerify(exTable);
@@ -232,7 +232,7 @@ public class CloudAccessTest extends
AbstractTestcontainersTest {
exTable.setProfile("s3:text");
exTable.setServer(serverParam);
if (creds) {
- exTable.setUserParameters(new String[]{"accesskey=" +
MinIOContainer.ACCESS_KEY, "secretkey=" + MinIOContainer.SECRET_KEY});
+ exTable.setUserParameters(new String[]{"accesskey=" +
RustFSContainer.ACCESS_KEY, "secretkey=" + RustFSContainer.SECRET_KEY});
}
cloudberry.createTableAndVerify(exTable);
diff --git
a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/cloud/S3SelectTest.java
b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/cloud/S3SelectTest.java
index 9986635c..0155db06 100644
---
a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/cloud/S3SelectTest.java
+++
b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/cloud/S3SelectTest.java
@@ -19,10 +19,11 @@ package org.apache.cloudberry.pxf.automation.features.cloud;
* under the License.
*/
+import annotations.WorksWithFDW;
import org.apache.cloudberry.pxf.automation.AbstractTestcontainersTest;
import org.apache.cloudberry.pxf.automation.applications.S3Application;
import
org.apache.cloudberry.pxf.automation.structures.tables.pxf.ReadableExternalTable;
-import org.apache.cloudberry.pxf.automation.testcontainers.MinIOContainer;
+import org.apache.cloudberry.pxf.automation.testcontainers.RustFSContainer;
import org.apache.cloudberry.pxf.automation.utils.AutomationUtils;
import org.testng.annotations.Test;
@@ -35,6 +36,7 @@ import java.util.UUID;
import static
org.apache.cloudberry.pxf.automation.features.tpch.LineItem.LINEITEM_SCHEMA;
/** Functional S3 Select Test */
+@WorksWithFDW
public class S3SelectTest extends AbstractTestcontainersTest {
private static final String[] PXF_S3_SELECT_INVALID_COLS = {
@@ -56,7 +58,7 @@ public class S3SelectTest extends AbstractTestcontainersTest {
"invalid_comment VARCHAR(44)"
};
- private MinIOContainer s3Server;
+ private RustFSContainer s3Server;
private S3Application s3Application;
private String s3Path;
private String objectKeyPrefix;
@@ -86,23 +88,23 @@ public class S3SelectTest extends
AbstractTestcontainersTest {
*/
@Override
public void beforeClass() throws Exception {
- s3Server = new MinIOContainer(container.getSharedNetwork());
+ s3Server = new RustFSContainer(container.getSharedNetwork());
s3Server.start();
s3Application = new S3Application(s3Server);
- s3Application.createBucket(MinIOContainer.DEFAULT_BUCKET);
+ s3Application.createBucket(RustFSContainer.DEFAULT_BUCKET);
String uuid = UUID.randomUUID().toString();
objectKeyPrefix = "tmp/pxf_automation_data/" + uuid + "/s3select/";
- s3Path = MinIOContainer.DEFAULT_BUCKET + "/" + objectKeyPrefix;
+ s3Path = RustFSContainer.DEFAULT_BUCKET + "/" + objectKeyPrefix;
- uploadFixtures(MinIOContainer.DEFAULT_BUCKET, objectKeyPrefix);
+ uploadFixtures(RustFSContainer.DEFAULT_BUCKET, objectKeyPrefix);
// Server 's3' is pre-baked into the container image by entrypoint.sh.
}
@Override
public void afterClass() throws Exception {
if (s3Application != null && objectKeyPrefix != null) {
- s3Application.deletePrefix(MinIOContainer.DEFAULT_BUCKET,
objectKeyPrefix);
+ s3Application.deletePrefix(RustFSContainer.DEFAULT_BUCKET,
objectKeyPrefix);
s3Application.shutdown();
}
if (s3Server != null) {
@@ -110,7 +112,7 @@ public class S3SelectTest extends
AbstractTestcontainersTest {
}
}
- // Uploads committed S3 Select fixture files from
src/test/resources/data/s3select/ into MinIO.
+ // Uploads committed S3 Select fixture files from
src/test/resources/data/s3select/ into RustFS.
private void uploadFixtures(String bucket, String objectKeyPrefix) throws
IOException {
Path fixturesDir = resolveFixturesDirectory();
for (String filename : FIXTURE_FILES) {
diff --git a/ci/docker/pxf-cbdb-dev/common/script/run_tests.sh
b/ci/docker/pxf-cbdb-dev/common/script/run_tests.sh
index 226825fd..89690031 100755
--- a/ci/docker/pxf-cbdb-dev/common/script/run_tests.sh
+++ b/ci/docker/pxf-cbdb-dev/common/script/run_tests.sh
@@ -38,7 +38,7 @@ export PXF_SKIP_TINC=${PXF_SKIP_TINC:-false}
export EXCLUDED_GROUPS=${EXCLUDED_GROUPS:-}
# Keep test data on HDFS between classes to avoid missing inputs
export PXF_TEST_KEEP_DATA=${PXF_TEST_KEEP_DATA:-true}
-# Provide S3 credentials so MinIO seeding and user-parameter overrides succeed.
+# Provide S3 credentials so RustFS seeding and user-parameter overrides
succeed.
export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-admin}
export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-password}
diff --git
a/server/pxf-s3/src/main/java/org/apache/cloudberry/pxf/plugins/s3/S3SelectQueryBuilder.java
b/server/pxf-s3/src/main/java/org/apache/cloudberry/pxf/plugins/s3/S3SelectQueryBuilder.java
index b62a78d7..e5137c0b 100644
---
a/server/pxf-s3/src/main/java/org/apache/cloudberry/pxf/plugins/s3/S3SelectQueryBuilder.java
+++
b/server/pxf-s3/src/main/java/org/apache/cloudberry/pxf/plugins/s3/S3SelectQueryBuilder.java
@@ -61,8 +61,10 @@ public class S3SelectQueryBuilder extends SQLQueryBuilder {
@Override
protected String buildColumnsQuery() {
+ // Give placeholder values distinct names for S3 Select
implementations that reject
+ // duplicate expressions in the SELECT list (for example, RustFS
1.0.0).
return columns.stream()
- .map(c -> c.isProjected() ? getColumnName(c) : "null")
+ .map(c -> c.isProjected() ? getColumnName(c) :
String.format("null AS pxf_unused_%d", c.columnIndex() + 1))
.collect(Collectors.joining(", "));
}
diff --git
a/server/pxf-s3/src/test/java/org/apache/cloudberry/pxf/plugins/s3/S3SelectQueryBuilderTest.java
b/server/pxf-s3/src/test/java/org/apache/cloudberry/pxf/plugins/s3/S3SelectQueryBuilderTest.java
index 08e31dbc..dcf71bf7 100644
---
a/server/pxf-s3/src/test/java/org/apache/cloudberry/pxf/plugins/s3/S3SelectQueryBuilderTest.java
+++
b/server/pxf-s3/src/test/java/org/apache/cloudberry/pxf/plugins/s3/S3SelectQueryBuilderTest.java
@@ -47,6 +47,17 @@ public class S3SelectQueryBuilderTest {
assertEquals(SQL_NO_POSITION, builderNoPosition.buildSelectQuery());
}
+ @Test
+ public void testUnprojectedColumnsHaveDistinctAliases() {
+ context.getTupleDescription().get(1).setProjected(false);
+ context.getTupleDescription().get(3).setProjected(false);
+
+ assertEquals("SELECT s._1, null AS pxf_unused_2, s._3, null AS
pxf_unused_4, s._5, s._6, s._7, s._8, s._9 FROM S3Object s",
+ builderPosition.buildSelectQuery());
+ assertEquals("SELECT s.\"id\", null AS pxf_unused_2, s.\"amt\", null
AS pxf_unused_4, s.\"pass\", s.\"weight\", s.\"col_varchar\", s.\"col_char\",
s.\"col_numeric\" FROM S3Object s",
+ builderNoPosition.buildSelectQuery());
+ }
+
@Test
public void testIdFilter() {
context.setFilterString("a0c20s1d1o5"); // id = 1
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
