This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new c249758de0 [test] Fix URI-based temporary paths (#9191)
c249758de0 is described below
commit c249758de0e58f8de2220d30fc012da5b7e57ebb
Author: Jingsong Lee <[email protected]>
AuthorDate: Wed Aug 12 17:43:00 2026 +0800
[test] Fix URI-based temporary paths (#9191)
---
.../test/java/org/apache/paimon/JavaPyE2ETest.java | 175 +++++++++++----------
.../apache/paimon/rest/MockRESTCatalogTest.java | 49 +++---
.../paimon/table/PrimaryKeySimpleTableTest.java | 8 +-
3 files changed, 127 insertions(+), 105 deletions(-)
diff --git a/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
b/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
index b6801b4a7b..6f1baffd56 100644
--- a/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java
@@ -94,6 +94,7 @@ import org.apache.paimon.utils.TraceableFileIO;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfSystemProperty;
+import org.junit.jupiter.api.io.TempDir;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -112,6 +113,7 @@ import java.util.UUID;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import static org.apache.paimon.CoreOptions.BUCKET;
import static org.apache.paimon.CoreOptions.DATA_EVOLUTION_ENABLED;
@@ -134,6 +136,7 @@ public class JavaPyE2ETest {
private static final Logger LOG =
LoggerFactory.getLogger(JavaPyE2ETest.class);
java.nio.file.Path tempDir =
Paths.get("../paimon-python/pypaimon/tests/e2e").toAbsolutePath();
+ @TempDir java.nio.file.Path ioTempDir;
// Fields from TableTestBase that we need
protected final String commitUser = UUID.randomUUID().toString();
@@ -387,40 +390,42 @@ public class JavaPyE2ETest {
String tableName = "test_pk_dv";
Path tablePath = new Path(warehouse.toString() + "/default.db/" +
tableName);
FileStoreTable table = createFileStoreTable(optionsSetter, tablePath);
- StreamTableWrite write = table.newWrite(commitUser);
- IOManager ioManager = IOManager.create(tablePath.toString());
- write.withIOManager(ioManager);
- StreamTableCommit commit = table.newCommit(commitUser);
+ try (IOManager ioManager = IOManager.create(ioTempDir.toString());
+ StreamTableWrite write =
table.newWrite(commitUser).withIOManager(ioManager);
+ StreamTableCommit commit = table.newCommit(commitUser)) {
- write.write(createRow3Cols(1, 10, 100L));
- write.write(createRow3Cols(2, 20, 200L));
- write.write(createRow3Cols(1, 11, 101L));
- commit.commit(0, write.prepareCommit(true, 0));
-
- write.write(createRow3Cols(1, 10, 1000L));
- write.write(createRow3Cols(2, 21, 201L));
- write.write(createRow3Cols(2, 21, 2001L));
- commit.commit(1, write.prepareCommit(true, 1));
-
- write.write(createRow3Cols(1, 11, 1001L));
- write.write(createRow3Cols(2, 21, 20001L));
- write.write(createRow3Cols(2, 22, 202L));
- write.write(createRow3ColsWithKind(RowKind.DELETE, 1, 11, 1001L));
- commit.commit(2, write.prepareCommit(true, 2));
- write.write(createRow3ColsWithKind(RowKind.DELETE, 2, 20, 200L));
- commit.commit(2, write.prepareCommit(true, 2));
-
- // test result
- Function<InternalRow, String> rowDataToString =
- row ->
- internalRowToString(
- row,
- DataTypes.ROW(
- DataTypes.INT(), DataTypes.INT(),
DataTypes.BIGINT()));
- List<String> result =
- getResult(table.newRead(), table.newScan().plan().splits(),
rowDataToString);
- assertThat(result)
- .containsExactlyInAnyOrder("+I[1, 10, 1000]", "+I[2, 21,
20001]", "+I[2, 22, 202]");
+ write.write(createRow3Cols(1, 10, 100L));
+ write.write(createRow3Cols(2, 20, 200L));
+ write.write(createRow3Cols(1, 11, 101L));
+ commit.commit(0, write.prepareCommit(true, 0));
+
+ write.write(createRow3Cols(1, 10, 1000L));
+ write.write(createRow3Cols(2, 21, 201L));
+ write.write(createRow3Cols(2, 21, 2001L));
+ commit.commit(1, write.prepareCommit(true, 1));
+
+ write.write(createRow3Cols(1, 11, 1001L));
+ write.write(createRow3Cols(2, 21, 20001L));
+ write.write(createRow3Cols(2, 22, 202L));
+ write.write(createRow3ColsWithKind(RowKind.DELETE, 1, 11, 1001L));
+ commit.commit(2, write.prepareCommit(true, 2));
+ write.write(createRow3ColsWithKind(RowKind.DELETE, 2, 20, 200L));
+ commit.commit(2, write.prepareCommit(true, 2));
+
+ // test result
+ Function<InternalRow, String> rowDataToString =
+ row ->
+ internalRowToString(
+ row,
+ DataTypes.ROW(
+ DataTypes.INT(), DataTypes.INT(),
DataTypes.BIGINT()));
+ List<String> result =
+ getResult(table.newRead(),
table.newScan().plan().splits(), rowDataToString);
+ assertThat(result)
+ .containsExactlyInAnyOrder(
+ "+I[1, 10, 1000]", "+I[2, 21, 20001]", "+I[2, 22,
202]");
+ }
+ assertSpillDirectoryEmpty();
}
@Test
@@ -435,37 +440,38 @@ public class JavaPyE2ETest {
String tableName = "test_pk_dv_multi_batch";
Path tablePath = new Path(warehouse.toString() + "/default.db/" +
tableName);
FileStoreTable table = createFileStoreTable(optionsSetter, tablePath);
- StreamTableWrite write = table.newWrite(commitUser);
- IOManager ioManager = IOManager.create(tablePath.toString());
- write.withIOManager(ioManager);
- StreamTableCommit commit = table.newCommit(commitUser);
+ try (IOManager ioManager = IOManager.create(ioTempDir.toString());
+ StreamTableWrite write =
table.newWrite(commitUser).withIOManager(ioManager);
+ StreamTableCommit commit = table.newCommit(commitUser)) {
- // Write 10000 records
- for (int i = 1; i <= 10000; i++) {
- write.write(createRow3Cols(1, i * 10, (long) i * 100));
- }
- commit.commit(0, write.prepareCommit(false, 0));
+ // Write 10000 records
+ for (int i = 1; i <= 10000; i++) {
+ write.write(createRow3Cols(1, i * 10, (long) i * 100));
+ }
+ commit.commit(0, write.prepareCommit(false, 0));
- // Delete the 81930th record
- write.write(createRow3ColsWithKind(RowKind.DELETE, 1, 81930, 819300L));
- commit.commit(1, write.prepareCommit(true, 1));
+ // Delete the 81930th record
+ write.write(createRow3ColsWithKind(RowKind.DELETE, 1, 81930,
819300L));
+ commit.commit(1, write.prepareCommit(true, 1));
- Function<InternalRow, String> rowDataToString =
- row ->
- internalRowToString(
- row,
- DataTypes.ROW(
- DataTypes.INT(), DataTypes.INT(),
DataTypes.BIGINT()));
- List<String> result =
- getResult(table.newRead(), table.newScan().plan().splits(),
rowDataToString);
+ Function<InternalRow, String> rowDataToString =
+ row ->
+ internalRowToString(
+ row,
+ DataTypes.ROW(
+ DataTypes.INT(), DataTypes.INT(),
DataTypes.BIGINT()));
+ List<String> result =
+ getResult(table.newRead(),
table.newScan().plan().splits(), rowDataToString);
- // Verify the count is 9999
- assertThat(result).hasSize(9999);
+ // Verify the count is 9999
+ assertThat(result).hasSize(9999);
- assertThat(result).doesNotContain("+I[1, 81930, 819300]");
+ assertThat(result).doesNotContain("+I[1, 81930, 819300]");
- assertThat(result).contains("+I[1, 10, 100]");
- assertThat(result).contains("+I[1, 100000, 1000000]");
+ assertThat(result).contains("+I[1, 10, 100]");
+ assertThat(result).contains("+I[1, 100000, 1000000]");
+ }
+ assertSpillDirectoryEmpty();
}
@Test
@@ -478,35 +484,36 @@ public class JavaPyE2ETest {
String tableName = "test_pk_dv_raw_convertable";
Path tablePath = new Path(warehouse.toString() + "/default.db/" +
tableName);
FileStoreTable table = createFileStoreTable(optionsSetter, tablePath);
- StreamTableWrite write = table.newWrite(commitUser);
- IOManager ioManager = IOManager.create(tablePath.toString());
- write.withIOManager(ioManager);
- StreamTableCommit commit = table.newCommit(commitUser);
+ try (IOManager ioManager = IOManager.create(ioTempDir.toString());
+ StreamTableWrite write =
table.newWrite(commitUser).withIOManager(ioManager);
+ StreamTableCommit commit = table.newCommit(commitUser)) {
- for (int i = 1; i <= 10000; i++) {
- write.write(createRow3Cols(1, i * 10, (long) i * 100));
- }
- commit.commit(0, write.prepareCommit(false, 0));
+ for (int i = 1; i <= 10000; i++) {
+ write.write(createRow3Cols(1, i * 10, (long) i * 100));
+ }
+ commit.commit(0, write.prepareCommit(false, 0));
- write.write(createRow3ColsWithKind(RowKind.DELETE, 1, 81930, 819300L));
- commit.commit(1, write.prepareCommit(true, 1));
+ write.write(createRow3ColsWithKind(RowKind.DELETE, 1, 81930,
819300L));
+ commit.commit(1, write.prepareCommit(true, 1));
- Function<InternalRow, String> rowDataToString =
- row ->
- internalRowToString(
- row,
- DataTypes.ROW(
- DataTypes.INT(), DataTypes.INT(),
DataTypes.BIGINT()));
- List<String> result =
- getResult(table.newRead(), table.newScan().plan().splits(),
rowDataToString);
+ Function<InternalRow, String> rowDataToString =
+ row ->
+ internalRowToString(
+ row,
+ DataTypes.ROW(
+ DataTypes.INT(), DataTypes.INT(),
DataTypes.BIGINT()));
+ List<String> result =
+ getResult(table.newRead(),
table.newScan().plan().splits(), rowDataToString);
- assertThat(result).hasSize(9999);
+ assertThat(result).hasSize(9999);
- assertThat(result).doesNotContain("+I[1, 81930, 819300]");
+ assertThat(result).doesNotContain("+I[1, 81930, 819300]");
- // Verify some sample records exist
- assertThat(result).contains("+I[1, 10, 100]");
- assertThat(result).contains("+I[1, 100000, 1000000]");
+ // Verify some sample records exist
+ assertThat(result).contains("+I[1, 10, 100]");
+ assertThat(result).contains("+I[1, 100000, 1000000]");
+ }
+ assertSpillDirectoryEmpty();
}
@Test
@@ -610,6 +617,12 @@ public class JavaPyE2ETest {
return assigner.assign(BinaryRow.EMPTY_ROW, keyHash);
}
+ private void assertSpillDirectoryEmpty() throws Exception {
+ try (Stream<java.nio.file.Path> files = Files.list(ioTempDir)) {
+ assertThat(files).isEmpty();
+ }
+ }
+
@Test
@EnabledIfSystemProperty(named = "run.e2e.tests", matches = "true")
public void testBtreeIndexWrite() throws Exception {
diff --git
a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java
b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java
index 2b91f1311b..a68b7933e5 100644
--- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java
@@ -64,8 +64,10 @@ import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
-import java.io.File;
import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
@@ -153,25 +155,32 @@ class MockRESTCatalogTest extends RESTCatalogTest {
void testDlfStSTokenPathAuth() throws Exception {
String uri = "https://cn-hangzhou-vpc.dlf.aliyuncs.com";;
String region = "cn-hangzhou";
- String tokenPath = dataPath + UUID.randomUUID();
- generateTokenAndWriteToFile(tokenPath);
- DLFTokenLoader tokenLoader =
- DLFTokenLoaderFactory.createDLFTokenLoader(
- "local_file",
- new Options(
- ImmutableMap.of(
-
RESTCatalogOptions.DLF_TOKEN_PATH.key(), tokenPath)));
- DLFToken dlfToken = tokenLoader.loadToken();
- this.authProvider = new TestDLFAuthProvider(dlfToken, uri, region);
- this.authMap =
- ImmutableMap.of(
- RESTCatalogOptions.TOKEN_PROVIDER.key(),
AuthProviderEnum.DLF.identifier(),
- RESTCatalogOptions.DLF_REGION.key(), region,
- RESTCatalogOptions.DLF_TOKEN_PATH.key(), tokenPath);
- RESTCatalog restCatalog = initCatalog(false);
- testDlfAuth(restCatalog);
- File file = new File(tokenPath);
- file.delete();
+ java.nio.file.Path tokenFile =
+
Paths.get(URI.create(dataPath)).resolve(UUID.randomUUID().toString());
+ String tokenPath = tokenFile.toString();
+ try {
+ generateTokenAndWriteToFile(tokenPath);
+ DLFTokenLoader tokenLoader =
+ DLFTokenLoaderFactory.createDLFTokenLoader(
+ "local_file",
+ new Options(
+ ImmutableMap.of(
+
RESTCatalogOptions.DLF_TOKEN_PATH.key(), tokenPath)));
+ DLFToken dlfToken = tokenLoader.loadToken();
+ this.authProvider = new TestDLFAuthProvider(dlfToken, uri, region);
+ this.authMap =
+ ImmutableMap.of(
+ RESTCatalogOptions.TOKEN_PROVIDER.key(),
+ AuthProviderEnum.DLF.identifier(),
+ RESTCatalogOptions.DLF_REGION.key(),
+ region,
+ RESTCatalogOptions.DLF_TOKEN_PATH.key(),
+ tokenPath);
+ RESTCatalog restCatalog = initCatalog(false);
+ testDlfAuth(restCatalog);
+ } finally {
+ Files.deleteIfExists(tokenFile);
+ }
}
@Test
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeySimpleTableTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeySimpleTableTest.java
index 0fa383dd4d..aa8da137d7 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeySimpleTableTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeySimpleTableTest.java
@@ -2383,7 +2383,7 @@ public class PrimaryKeySimpleTableTest extends
SimpleTableTestBase {
FileStoreTable table = createFileStoreTable(optionsSetter);
StreamTableWrite write = table.newWrite(commitUser);
- IOManager ioManager = IOManager.create(tablePath.toString());
+ IOManager ioManager = IOManager.create(tempDir.toString());
write.withIOManager(ioManager);
StreamTableCommit commit = table.newCommit(commitUser);
@@ -2540,7 +2540,7 @@ public class PrimaryKeySimpleTableTest extends
SimpleTableTestBase {
options.set(CHANGELOG_PRODUCER, LOOKUP);
options.set(CoreOptions.LOOKUP_REMOTE_FILE_ENABLED, true);
});
- IOManager ioManager = IOManager.create(tablePath.toString());
+ IOManager ioManager = IOManager.create(tempDir.toString());
StreamTableWrite write =
table.newWrite(commitUser).withIOManager(ioManager);
StreamTableCommit commit = table.newCommit(commitUser);
@@ -2658,7 +2658,7 @@ public class PrimaryKeySimpleTableTest extends
SimpleTableTestBase {
conf.set(CHANGELOG_FILE_STATS_MODE, "none");
}
});
- IOManager ioManager = IOManager.create(tablePath.toString());
+ IOManager ioManager = IOManager.create(tempDir.toString());
StreamTableWrite write =
table.newWrite(commitUser).withIOManager(ioManager);
StreamTableCommit commit = table.newCommit(commitUser);
write.write(rowData(1, 1, 100L));
@@ -2844,7 +2844,7 @@ public class PrimaryKeySimpleTableTest extends
SimpleTableTestBase {
}
private void innerTestTableQuery(FileStoreTable table) throws Exception {
- IOManager ioManager = IOManager.create(tablePath.toString());
+ IOManager ioManager = IOManager.create(tempDir.toString());
StreamTableWrite write =
table.newWrite(commitUser).withIOManager(ioManager);
StreamTableCommit commit = table.newCommit(commitUser);
