This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new c5a8c4ae6f [python] Batch mixed Java write tests (#8405)
c5a8c4ae6f is described below
commit c5a8c4ae6fe48f49b7e00c1b2726342abeebdf12
Author: Jingsong Lee <[email protected]>
AuthorDate: Wed Jul 1 13:13:28 2026 +0800
[python] Batch mixed Java write tests (#8405)
Batch the Java write side of `run_mixed_tests.sh` by module before the
existing Python validation flow. This reduces repeated Maven invocations
while preserving an opt-out path for the original one-by-one execution.
---
paimon-python/dev/run_mixed_tests.sh | 467 +++++++++++++++++++++++------------
1 file changed, 315 insertions(+), 152 deletions(-)
diff --git a/paimon-python/dev/run_mixed_tests.sh
b/paimon-python/dev/run_mixed_tests.sh
index 8da1cb345b..efd066524c 100755
--- a/paimon-python/dev/run_mixed_tests.sh
+++ b/paimon-python/dev/run_mixed_tests.sh
@@ -58,10 +58,119 @@ cleanup_warehouse() {
echo ""
}
+JAVA_WRITE_BATCH_DONE=false
+
+java_write_batch_enabled() {
+ [[ "${MIXED_TESTS_BATCH_JAVA_WRITES:-true}" != "false" ]]
+}
+
+skip_batched_java_write() {
+ if [[ "${JAVA_WRITE_BATCH_DONE:-false}" == "true" ]]; then
+ echo "Java write already completed in batched phase; skipping Maven
call."
+ return 0
+ fi
+ return 1
+}
+
+run_maven_test_batch() {
+ local description="$1"
+ local module="$2"
+ local test_spec="$3"
+ shift 3
+
+ cd "$PROJECT_ROOT"
+ echo "Running Maven batch for $description..."
+ echo " Module: $module"
+ echo " Tests: $test_spec"
+ if mvn test -Dtest="$test_spec" -pl "$module" -Drun.e2e.tests=true "$@";
then
+ echo -e "${GREEN}✓ $description completed successfully${NC}"
+ return 0
+ else
+ echo -e "${RED}✗ $description failed${NC}"
+ return 1
+ fi
+}
+
+run_batched_java_write_tests() {
+ echo -e "${YELLOW}=== Pre-step: Running Batched Java Write Tests ===${NC}"
+ echo "Set MIXED_TESTS_BATCH_JAVA_WRITES=false to use the original
one-by-one Java write flow."
+
+ local result=0
+
+ local core_tests="org.apache.paimon.JavaPyE2ETest#testJavaWriteReadPkTable"
+ core_tests="${core_tests}+testPKDeletionVectorWrite"
+ core_tests="${core_tests}+testBtreeIndexWrite"
+ core_tests="${core_tests}+testBtreeRawFallbackWrite"
+ core_tests="${core_tests}+testBitmapIndexWrite"
+ core_tests="${core_tests}+testCompressedGlobalIndexWrite"
+ core_tests="${core_tests}+testJavaWriteCompressedTextAppendTable"
+ core_tests="${core_tests}+testJavaWriteVectorAppendTable"
+ core_tests="${core_tests}+testCompactConflictWriteBase"
+ core_tests="${core_tests}+testBlobCompactConflictWriteBase"
+ core_tests="${core_tests}+testBlobWriteAlterCompact"
+ core_tests="${core_tests}+testDataEvolutionWrite"
+ core_tests="${core_tests}+testJavaWriteRowAppendTable"
+ if [[ "$PYTHON_MINOR" -ge 7 ]]; then
+ core_tests="${core_tests}+testJavaWriteVariantTable"
+ fi
+ if ! run_maven_test_batch "paimon-core Java write tests" "paimon-core"
"$core_tests" -q; then
+ result=1
+ fi
+
+ local
lance_tests="org.apache.paimon.JavaPyLanceE2ETest#testJavaWriteReadPkTableLance"
+ lance_tests="${lance_tests}+testDataEvolutionWriteLance"
+ if ! run_maven_test_batch "paimon-lance Java write tests" "paimon-lance"
"$lance_tests" -q; then
+ result=1
+ fi
+
+ if [[ "$PYTHON_MINOR" -ge 11 ]]; then
+ local
vortex_tests="org.apache.paimon.JavaPyVortexE2ETest#testJavaWriteVectorDedicatedFile"
+ vortex_tests="${vortex_tests}+testJavaWriteMultiVectorDedicatedFile"
+ if ! run_maven_test_batch "paimon-vortex Java write tests"
"paimon-vortex/paimon-vortex-format" "$vortex_tests" -q; then
+ result=1
+ fi
+ fi
+
+ if [[ "$PYTHON_MINOR" -ge 10 ]]; then
+ if ! run_maven_test_batch \
+ "paimon-tantivy Java write tests" \
+ "paimon-tantivy/paimon-tantivy-index" \
+
"org.apache.paimon.tantivy.index.JavaPyTantivyE2ETest#testTantivyFullTextIndexWrite"
\
+ -q; then
+ result=1
+ fi
+ fi
+
+ local
lumina_tests="org.apache.paimon.lumina.index.JavaPyLuminaE2ETest#testLuminaVectorIndexWrite"
+ lumina_tests="${lumina_tests}+testLuminaVectorWithBTreeIndexWrite"
+ if ! run_maven_test_batch "paimon-lumina Java write tests" "paimon-lumina"
"$lumina_tests" -q; then
+ result=1
+ fi
+
+ if [[ "$PYTHON_MINOR" -ge 9 ]]; then
+ local
vindex_tests="org.apache.paimon.JavaPyE2ETest#testVindexVectorIndexWrite"
+ vindex_tests="${vindex_tests}+testVindexVectorRawFallbackWrite"
+ if ! run_maven_test_batch \
+ "paimon-vector Java write tests" \
+ "paimon-vector" \
+ "$vindex_tests" \
+ -am -q -DfailIfNoTests=false; then
+ result=1
+ fi
+ fi
+
+ echo ""
+ return $result
+}
+
# Function to run Java test
run_java_write_test() {
echo -e "${YELLOW}=== Step 1: Running Java Write Tests (Parquet/Orc/Avro +
Lance) ===${NC}"
+ if skip_batched_java_write; then
+ return 0
+ fi
+
cd "$PROJECT_ROOT"
# Run the Java test method for Parquet/Orc/Avro format
@@ -172,15 +281,17 @@ run_java_read_test() {
run_pk_dv_test() {
echo -e "${YELLOW}=== Step 5: Running Primary Key & Deletion Vector Test
(testPKDeletionVectorWriteRead) ===${NC}"
- cd "$PROJECT_ROOT"
+ if ! skip_batched_java_write; then
+ cd "$PROJECT_ROOT"
- # Run the specific Java test method
- echo "Running Maven test for JavaPyE2ETest.testPKDeletionVectorWrite..."
- if mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testPKDeletionVectorWrite -pl
paimon-core -q -Drun.e2e.tests=true; then
- echo -e "${GREEN}✓ Java test completed successfully${NC}"
- else
- echo -e "${RED}✗ Java test failed${NC}"
- return 1
+ # Run the specific Java test method
+ echo "Running Maven test for
JavaPyE2ETest.testPKDeletionVectorWrite..."
+ if mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testPKDeletionVectorWrite -pl
paimon-core -q -Drun.e2e.tests=true; then
+ echo -e "${GREEN}✓ Java test completed successfully${NC}"
+ else
+ echo -e "${RED}✗ Java test failed${NC}"
+ return 1
+ fi
fi
cd "$PAIMON_PYTHON_DIR"
# Run the specific Python test method
@@ -198,14 +309,16 @@ run_pk_dv_test() {
run_btree_index_test() {
echo -e "${YELLOW}=== Step 6: Running BTree Index Test (Java Write, Python
Read) ===${NC}"
- cd "$PROJECT_ROOT"
+ if ! skip_batched_java_write; then
+ cd "$PROJECT_ROOT"
- echo "Running Maven test for JavaPyE2ETest.testBtreeIndexWrite..."
- if mvn test -Dtest=org.apache.paimon.JavaPyE2ETest#testBtreeIndexWrite -pl
paimon-core -q -Drun.e2e.tests=true; then
- echo -e "${GREEN}✓ Java test completed successfully${NC}"
- else
- echo -e "${RED}✗ Java test failed${NC}"
- return 1
+ echo "Running Maven test for JavaPyE2ETest.testBtreeIndexWrite..."
+ if mvn test -Dtest=org.apache.paimon.JavaPyE2ETest#testBtreeIndexWrite
-pl paimon-core -q -Drun.e2e.tests=true; then
+ echo -e "${GREEN}✓ Java test completed successfully${NC}"
+ else
+ echo -e "${RED}✗ Java test failed${NC}"
+ return 1
+ fi
fi
cd "$PAIMON_PYTHON_DIR"
# Run the specific Python test method
@@ -222,14 +335,16 @@ run_btree_index_test() {
run_btree_raw_fallback_test() {
echo -e "${YELLOW}=== Running BTree Raw Fallback Test (Java Write, Python
Read) ===${NC}"
- cd "$PROJECT_ROOT"
+ if ! skip_batched_java_write; then
+ cd "$PROJECT_ROOT"
- echo "Running Maven test for JavaPyE2ETest.testBtreeRawFallbackWrite..."
- if mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testBtreeRawFallbackWrite -pl
paimon-core -am -q -DfailIfNoTests=false -Drun.e2e.tests=true; then
- echo -e "${GREEN}✓ Java test completed successfully${NC}"
- else
- echo -e "${RED}✗ Java test failed${NC}"
- return 1
+ echo "Running Maven test for
JavaPyE2ETest.testBtreeRawFallbackWrite..."
+ if mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testBtreeRawFallbackWrite -pl
paimon-core -am -q -DfailIfNoTests=false -Drun.e2e.tests=true; then
+ echo -e "${GREEN}✓ Java test completed successfully${NC}"
+ else
+ echo -e "${RED}✗ Java test failed${NC}"
+ return 1
+ fi
fi
cd "$PAIMON_PYTHON_DIR"
echo "Running Python test for
JavaPyReadWriteTest.test_read_btree_raw_fallback..."
@@ -245,14 +360,16 @@ run_btree_raw_fallback_test() {
run_bitmap_index_test() {
echo -e "${YELLOW}=== Step 6b: Running Bitmap Index Test (Java Write,
Python Read) ===${NC}"
- cd "$PROJECT_ROOT"
+ if ! skip_batched_java_write; then
+ cd "$PROJECT_ROOT"
- echo "Running Maven test for JavaPyE2ETest.testBitmapIndexWrite..."
- if mvn test -Dtest=org.apache.paimon.JavaPyE2ETest#testBitmapIndexWrite
-pl paimon-core -q -Drun.e2e.tests=true; then
- echo -e "${GREEN}✓ Java test completed successfully${NC}"
- else
- echo -e "${RED}✗ Java test failed${NC}"
- return 1
+ echo "Running Maven test for JavaPyE2ETest.testBitmapIndexWrite..."
+ if mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testBitmapIndexWrite -pl paimon-core -q
-Drun.e2e.tests=true; then
+ echo -e "${GREEN}✓ Java test completed successfully${NC}"
+ else
+ echo -e "${RED}✗ Java test failed${NC}"
+ return 1
+ fi
fi
cd "$PAIMON_PYTHON_DIR"
# Run the specific Python test method
@@ -269,14 +386,16 @@ run_bitmap_index_test() {
run_compressed_global_index_test() {
echo -e "${YELLOW}=== Step 6c: Running Compressed Global Index Test (Java
Write, Python Read) ===${NC}"
- cd "$PROJECT_ROOT"
+ if ! skip_batched_java_write; then
+ cd "$PROJECT_ROOT"
- echo "Running Maven test for
JavaPyE2ETest.testCompressedGlobalIndexWrite..."
- if mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testCompressedGlobalIndexWrite -pl
paimon-core -q -Drun.e2e.tests=true; then
- echo -e "${GREEN}✓ Java test completed successfully${NC}"
- else
- echo -e "${RED}✗ Java test failed${NC}"
- return 1
+ echo "Running Maven test for
JavaPyE2ETest.testCompressedGlobalIndexWrite..."
+ if mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testCompressedGlobalIndexWrite -pl
paimon-core -q -Drun.e2e.tests=true; then
+ echo -e "${GREEN}✓ Java test completed successfully${NC}"
+ else
+ echo -e "${RED}✗ Java test failed${NC}"
+ return 1
+ fi
fi
cd "$PAIMON_PYTHON_DIR"
echo "Running Python test for
JavaPyReadWriteTest.test_read_compressed_global_index_fallback_scan..."
@@ -292,14 +411,16 @@ run_compressed_global_index_test() {
run_compressed_text_test() {
echo -e "${YELLOW}=== Step 7: Running Compressed Text Test (Java Write,
Python Read) ===${NC}"
- cd "$PROJECT_ROOT"
+ if ! skip_batched_java_write; then
+ cd "$PROJECT_ROOT"
- echo "Running Maven test for
JavaPyE2ETest.testJavaWriteCompressedTextAppendTable..."
- if mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testJavaWriteCompressedTextAppendTable
-pl paimon-core -q -Drun.e2e.tests=true; then
- echo -e "${GREEN}✓ Java test completed successfully${NC}"
- else
- echo -e "${RED}✗ Java test failed${NC}"
- return 1
+ echo "Running Maven test for
JavaPyE2ETest.testJavaWriteCompressedTextAppendTable..."
+ if mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testJavaWriteCompressedTextAppendTable
-pl paimon-core -q -Drun.e2e.tests=true; then
+ echo -e "${GREEN}✓ Java test completed successfully${NC}"
+ else
+ echo -e "${RED}✗ Java test failed${NC}"
+ return 1
+ fi
fi
cd "$PAIMON_PYTHON_DIR"
echo "Running Python test for
JavaPyReadWriteTest.test_read_compressed_text_append_table..."
@@ -315,14 +436,16 @@ run_compressed_text_test() {
run_vector_append_table_test() {
echo -e "${YELLOW}=== Running Vector Append Table Test (Java Write, Python
Read) ===${NC}"
- cd "$PROJECT_ROOT"
+ if ! skip_batched_java_write; then
+ cd "$PROJECT_ROOT"
- echo "Running Maven test for
JavaPyE2ETest.testJavaWriteVectorAppendTable..."
- if mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testJavaWriteVectorAppendTable -pl
paimon-core -q -Drun.e2e.tests=true; then
- echo -e "${GREEN}✓ Java test completed successfully${NC}"
- else
- echo -e "${RED}✗ Java test failed${NC}"
- return 1
+ echo "Running Maven test for
JavaPyE2ETest.testJavaWriteVectorAppendTable..."
+ if mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testJavaWriteVectorAppendTable -pl
paimon-core -q -Drun.e2e.tests=true; then
+ echo -e "${GREEN}✓ Java test completed successfully${NC}"
+ else
+ echo -e "${RED}✗ Java test failed${NC}"
+ return 1
+ fi
fi
cd "$PAIMON_PYTHON_DIR"
echo "Running Python test for
JavaPyReadWriteTest.test_read_vector_append_table..."
@@ -338,14 +461,16 @@ run_vector_append_table_test() {
run_vector_dedicated_file_java_write_test() {
echo -e "${YELLOW}=== Running Vector Dedicated File Test (Java Write,
Python Read) ===${NC}"
- cd "$PROJECT_ROOT"
+ if ! skip_batched_java_write; then
+ cd "$PROJECT_ROOT"
- echo "Running Maven test for
JavaPyVortexE2ETest.testJavaWriteVectorDedicatedFile..."
- if mvn test
-Dtest=org.apache.paimon.JavaPyVortexE2ETest#testJavaWriteVectorDedicatedFile
-pl paimon-vortex/paimon-vortex-format -q -Drun.e2e.tests=true; then
- echo -e "${GREEN}✓ Java vector dedicated file write completed
successfully${NC}"
- else
- echo -e "${RED}✗ Java vector dedicated file write failed${NC}"
- return 1
+ echo "Running Maven test for
JavaPyVortexE2ETest.testJavaWriteVectorDedicatedFile..."
+ if mvn test
-Dtest=org.apache.paimon.JavaPyVortexE2ETest#testJavaWriteVectorDedicatedFile
-pl paimon-vortex/paimon-vortex-format -q -Drun.e2e.tests=true; then
+ echo -e "${GREEN}✓ Java vector dedicated file write completed
successfully${NC}"
+ else
+ echo -e "${RED}✗ Java vector dedicated file write failed${NC}"
+ return 1
+ fi
fi
cd "$PAIMON_PYTHON_DIR"
echo "Running Python test for
JavaPyReadWriteTest.test_read_vector_dedicated_file..."
@@ -385,14 +510,16 @@ run_vector_dedicated_file_py_write_test() {
run_multi_vector_dedicated_file_java_write_test() {
echo -e "${YELLOW}=== Running Multi-Vector Dedicated File Test (Java
Write, Python Read) ===${NC}"
- cd "$PROJECT_ROOT"
+ if ! skip_batched_java_write; then
+ cd "$PROJECT_ROOT"
- echo "Running Maven test for
JavaPyVortexE2ETest.testJavaWriteMultiVectorDedicatedFile..."
- if mvn test
-Dtest=org.apache.paimon.JavaPyVortexE2ETest#testJavaWriteMultiVectorDedicatedFile
-pl paimon-vortex/paimon-vortex-format -q -Drun.e2e.tests=true; then
- echo -e "${GREEN}✓ Java multi-vector dedicated file write completed
successfully${NC}"
- else
- echo -e "${RED}✗ Java multi-vector dedicated file write failed${NC}"
- return 1
+ echo "Running Maven test for
JavaPyVortexE2ETest.testJavaWriteMultiVectorDedicatedFile..."
+ if mvn test
-Dtest=org.apache.paimon.JavaPyVortexE2ETest#testJavaWriteMultiVectorDedicatedFile
-pl paimon-vortex/paimon-vortex-format -q -Drun.e2e.tests=true; then
+ echo -e "${GREEN}✓ Java multi-vector dedicated file write
completed successfully${NC}"
+ else
+ echo -e "${RED}✗ Java multi-vector dedicated file write
failed${NC}"
+ return 1
+ fi
fi
cd "$PAIMON_PYTHON_DIR"
echo "Running Python test for
JavaPyReadWriteTest.test_read_multi_vector_dedicated_file..."
@@ -433,14 +560,16 @@ run_multi_vector_dedicated_file_py_write_test() {
run_tantivy_fulltext_test() {
echo -e "${YELLOW}=== Step 8: Running Tantivy Full-Text Index Test (Java
Write, Python Read) ===${NC}"
- cd "$PROJECT_ROOT"
+ if ! skip_batched_java_write; then
+ cd "$PROJECT_ROOT"
- echo "Running Maven test for
JavaPyTantivyE2ETest.testTantivyFullTextIndexWrite..."
- if mvn test
-Dtest=org.apache.paimon.tantivy.index.JavaPyTantivyE2ETest#testTantivyFullTextIndexWrite
-pl paimon-tantivy/paimon-tantivy-index -q -Drun.e2e.tests=true; then
- echo -e "${GREEN}✓ Java test completed successfully${NC}"
- else
- echo -e "${RED}✗ Java test failed${NC}"
- return 1
+ echo "Running Maven test for
JavaPyTantivyE2ETest.testTantivyFullTextIndexWrite..."
+ if mvn test
-Dtest=org.apache.paimon.tantivy.index.JavaPyTantivyE2ETest#testTantivyFullTextIndexWrite
-pl paimon-tantivy/paimon-tantivy-index -q -Drun.e2e.tests=true; then
+ echo -e "${GREEN}✓ Java test completed successfully${NC}"
+ else
+ echo -e "${RED}✗ Java test failed${NC}"
+ return 1
+ fi
fi
cd "$PAIMON_PYTHON_DIR"
echo "Installing Python jieba tokenizer dependency for Tantivy jieba index
reads..."
@@ -462,14 +591,16 @@ run_tantivy_fulltext_test() {
run_lumina_vector_test() {
echo -e "${YELLOW}=== Step 9: Running Lumina Vector Index Test (Java
Write, Python Read) ===${NC}"
- cd "$PROJECT_ROOT"
+ if ! skip_batched_java_write; then
+ cd "$PROJECT_ROOT"
- echo "Running Maven test for
JavaPyLuminaE2ETest.testLuminaVectorIndexWrite..."
- if mvn test
-Dtest=org.apache.paimon.lumina.index.JavaPyLuminaE2ETest#testLuminaVectorIndexWrite
-pl paimon-lumina -q -Drun.e2e.tests=true; then
- echo -e "${GREEN}✓ Java test completed successfully${NC}"
- else
- echo -e "${RED}✗ Java test failed${NC}"
- return 1
+ echo "Running Maven test for
JavaPyLuminaE2ETest.testLuminaVectorIndexWrite..."
+ if mvn test
-Dtest=org.apache.paimon.lumina.index.JavaPyLuminaE2ETest#testLuminaVectorIndexWrite
-pl paimon-lumina -q -Drun.e2e.tests=true; then
+ echo -e "${GREEN}✓ Java test completed successfully${NC}"
+ else
+ echo -e "${RED}✗ Java test failed${NC}"
+ return 1
+ fi
fi
cd "$PAIMON_PYTHON_DIR"
echo "Running Python test for
JavaPyReadWriteTest.test_read_lumina_vector_index..."
@@ -486,14 +617,16 @@ run_lumina_vector_test() {
run_lumina_vector_btree_test() {
echo -e "${YELLOW}=== Running Lumina Vector + BTree Pre-Filter Test (Java
Write, Python Read) ===${NC}"
- cd "$PROJECT_ROOT"
+ if ! skip_batched_java_write; then
+ cd "$PROJECT_ROOT"
- echo "Running Maven test for
JavaPyLuminaE2ETest.testLuminaVectorWithBTreeIndexWrite..."
- if mvn test
-Dtest=org.apache.paimon.lumina.index.JavaPyLuminaE2ETest#testLuminaVectorWithBTreeIndexWrite
-pl paimon-lumina -q -Drun.e2e.tests=true; then
- echo -e "${GREEN}✓ Java test completed successfully${NC}"
- else
- echo -e "${RED}✗ Java test failed${NC}"
- return 1
+ echo "Running Maven test for
JavaPyLuminaE2ETest.testLuminaVectorWithBTreeIndexWrite..."
+ if mvn test
-Dtest=org.apache.paimon.lumina.index.JavaPyLuminaE2ETest#testLuminaVectorWithBTreeIndexWrite
-pl paimon-lumina -q -Drun.e2e.tests=true; then
+ echo -e "${GREEN}✓ Java test completed successfully${NC}"
+ else
+ echo -e "${RED}✗ Java test failed${NC}"
+ return 1
+ fi
fi
cd "$PAIMON_PYTHON_DIR"
echo "Running Python test for
JavaPyReadWriteTest.test_read_lumina_vector_with_btree_filter..."
@@ -541,14 +674,16 @@ ensure_paimon_vindex() {
run_vindex_vector_test() {
echo -e "${YELLOW}=== Running paimon-vindex Vector Index Test (Java Write,
Python Read) ===${NC}"
- cd "$PROJECT_ROOT"
+ if ! skip_batched_java_write; then
+ cd "$PROJECT_ROOT"
- echo "Running Maven test for JavaPyE2ETest.testVindexVectorIndexWrite..."
- if mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testVindexVectorIndexWrite -pl
paimon-vector -am -q -DfailIfNoTests=false -Drun.e2e.tests=true; then
- echo -e "${GREEN}✓ Java test completed successfully${NC}"
- else
- echo -e "${RED}✗ Java test failed${NC}"
- return 1
+ echo "Running Maven test for
JavaPyE2ETest.testVindexVectorIndexWrite..."
+ if mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testVindexVectorIndexWrite -pl
paimon-vector -am -q -DfailIfNoTests=false -Drun.e2e.tests=true; then
+ echo -e "${GREEN}✓ Java test completed successfully${NC}"
+ else
+ echo -e "${RED}✗ Java test failed${NC}"
+ return 1
+ fi
fi
cd "$PAIMON_PYTHON_DIR"
if ! ensure_paimon_vindex; then
@@ -567,14 +702,16 @@ run_vindex_vector_test() {
run_vindex_vector_raw_fallback_test() {
echo -e "${YELLOW}=== Running paimon-vindex Vector Raw Fallback Test (Java
Write, Python Read) ===${NC}"
- cd "$PROJECT_ROOT"
+ if ! skip_batched_java_write; then
+ cd "$PROJECT_ROOT"
- echo "Running Maven test for
JavaPyE2ETest.testVindexVectorRawFallbackWrite..."
- if mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testVindexVectorRawFallbackWrite -pl
paimon-vector -am -q -DfailIfNoTests=false -Drun.e2e.tests=true; then
- echo -e "${GREEN}✓ Java test completed successfully${NC}"
- else
- echo -e "${RED}✗ Java test failed${NC}"
- return 1
+ echo "Running Maven test for
JavaPyE2ETest.testVindexVectorRawFallbackWrite..."
+ if mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testVindexVectorRawFallbackWrite -pl
paimon-vector -am -q -DfailIfNoTests=false -Drun.e2e.tests=true; then
+ echo -e "${GREEN}✓ Java test completed successfully${NC}"
+ else
+ echo -e "${RED}✗ Java test failed${NC}"
+ return 1
+ fi
fi
cd "$PAIMON_PYTHON_DIR"
if ! ensure_paimon_vindex; then
@@ -593,15 +730,17 @@ run_vindex_vector_raw_fallback_test() {
run_compact_conflict_test() {
echo -e "${YELLOW}=== Running Compact Conflict Test (Java Write Base,
Python Shard Update + Java Compact) ===${NC}"
- cd "$PROJECT_ROOT"
+ if ! skip_batched_java_write; then
+ cd "$PROJECT_ROOT"
- # Step 1: Java writes 5 base files
- echo "Running Maven test for JavaPyE2ETest.testCompactConflictWriteBase..."
- if mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testCompactConflictWriteBase -pl
paimon-core -q -Drun.e2e.tests=true; then
- echo -e "${GREEN}✓ Java write base files completed successfully${NC}"
- else
- echo -e "${RED}✗ Java write base files failed${NC}"
- return 1
+ # Step 1: Java writes 5 base files
+ echo "Running Maven test for
JavaPyE2ETest.testCompactConflictWriteBase..."
+ if mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testCompactConflictWriteBase -pl
paimon-core -q -Drun.e2e.tests=true; then
+ echo -e "${GREEN}✓ Java write base files completed
successfully${NC}"
+ else
+ echo -e "${RED}✗ Java write base files failed${NC}"
+ return 1
+ fi
fi
# Step 2-4: Python shard update (scan -> Java compact -> commit conflict
detected)
@@ -619,14 +758,16 @@ run_compact_conflict_test() {
run_blob_compact_conflict_test() {
echo -e "${YELLOW}=== Running Blob Compact Conflict Test (Java Write Base,
Python Blob Update + Java Compact) ===${NC}"
- cd "$PROJECT_ROOT"
+ if ! skip_batched_java_write; then
+ cd "$PROJECT_ROOT"
- echo "Running Maven test for
JavaPyE2ETest.testBlobCompactConflictWriteBase..."
- if mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testBlobCompactConflictWriteBase -pl
paimon-core -q -Drun.e2e.tests=true; then
- echo -e "${GREEN}✓ Java write base blob files completed
successfully${NC}"
- else
- echo -e "${RED}✗ Java write base blob files failed${NC}"
- return 1
+ echo "Running Maven test for
JavaPyE2ETest.testBlobCompactConflictWriteBase..."
+ if mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testBlobCompactConflictWriteBase -pl
paimon-core -q -Drun.e2e.tests=true; then
+ echo -e "${GREEN}✓ Java write base blob files completed
successfully${NC}"
+ else
+ echo -e "${RED}✗ Java write base blob files failed${NC}"
+ return 1
+ fi
fi
cd "$PAIMON_PYTHON_DIR"
@@ -643,26 +784,29 @@ run_blob_compact_conflict_test() {
run_data_evolution_test() {
echo -e "${YELLOW}=== Running Data Evolution Test (Java Write, Python
Read) ===${NC}"
- cd "$PROJECT_ROOT"
-
- # Java write data evolution tables (parquet/orc/avro)
- echo "Running Maven test for JavaPyE2ETest.testDataEvolutionWrite..."
local core_result=0
- if mvn test -Dtest=org.apache.paimon.JavaPyE2ETest#testDataEvolutionWrite
-pl paimon-core -q -Drun.e2e.tests=true; then
- echo -e "${GREEN}✓ Java data evolution write (parquet/orc/avro)
completed successfully${NC}"
- else
- echo -e "${RED}✗ Java data evolution write (parquet/orc/avro)
failed${NC}"
- core_result=1
- fi
-
- # Java write data evolution table (lance)
- echo "Running Maven test for
JavaPyLanceE2ETest.testDataEvolutionWriteLance..."
local lance_result=0
- if mvn test
-Dtest=org.apache.paimon.JavaPyLanceE2ETest#testDataEvolutionWriteLance -pl
paimon-lance -q -Drun.e2e.tests=true; then
- echo -e "${GREEN}✓ Java data evolution write (lance) completed
successfully${NC}"
- else
- echo -e "${RED}✗ Java data evolution write (lance) failed${NC}"
- lance_result=1
+
+ if ! skip_batched_java_write; then
+ cd "$PROJECT_ROOT"
+
+ # Java write data evolution tables (parquet/orc/avro)
+ echo "Running Maven test for JavaPyE2ETest.testDataEvolutionWrite..."
+ if mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testDataEvolutionWrite -pl paimon-core
-q -Drun.e2e.tests=true; then
+ echo -e "${GREEN}✓ Java data evolution write (parquet/orc/avro)
completed successfully${NC}"
+ else
+ echo -e "${RED}✗ Java data evolution write (parquet/orc/avro)
failed${NC}"
+ core_result=1
+ fi
+
+ # Java write data evolution table (lance)
+ echo "Running Maven test for
JavaPyLanceE2ETest.testDataEvolutionWriteLance..."
+ if mvn test
-Dtest=org.apache.paimon.JavaPyLanceE2ETest#testDataEvolutionWriteLance -pl
paimon-lance -q -Drun.e2e.tests=true; then
+ echo -e "${GREEN}✓ Java data evolution write (lance) completed
successfully${NC}"
+ else
+ echo -e "${RED}✗ Java data evolution write (lance) failed${NC}"
+ lance_result=1
+ fi
fi
# Python read data evolution tables
@@ -726,14 +870,16 @@ run_data_evolution_py_write_test() {
run_blob_alter_compact_test() {
echo -e "${YELLOW}=== Running Blob Alter+Compact Test (Java
Write+Alter+Compact, Python Read) ===${NC}"
- cd "$PROJECT_ROOT"
+ if ! skip_batched_java_write; then
+ cd "$PROJECT_ROOT"
- echo "Running Maven test for JavaPyE2ETest.testBlobWriteAlterCompact..."
- if mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testBlobWriteAlterCompact -pl
paimon-core -q -Drun.e2e.tests=true; then
- echo -e "${GREEN}✓ Java blob write+alter+compact test completed
successfully${NC}"
- else
- echo -e "${RED}✗ Java blob write+alter+compact test failed${NC}"
- return 1
+ echo "Running Maven test for
JavaPyE2ETest.testBlobWriteAlterCompact..."
+ if mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testBlobWriteAlterCompact -pl
paimon-core -q -Drun.e2e.tests=true; then
+ echo -e "${GREEN}✓ Java blob write+alter+compact test completed
successfully${NC}"
+ else
+ echo -e "${RED}✗ Java blob write+alter+compact test failed${NC}"
+ return 1
+ fi
fi
cd "$PAIMON_PYTHON_DIR"
echo "Running Python test for
JavaPyReadWriteTest.test_read_blob_after_alter_and_compact..."
@@ -750,14 +896,16 @@ run_blob_alter_compact_test() {
run_java_variant_write_py_read_test() {
echo -e "${YELLOW}=== Running VARIANT Test (Java Write, Python Read)
===${NC}"
- cd "$PROJECT_ROOT"
+ if ! skip_batched_java_write; then
+ cd "$PROJECT_ROOT"
- echo "Running Maven test for JavaPyE2ETest.testJavaWriteVariantTable..."
- if mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testJavaWriteVariantTable -pl
paimon-core -q -Drun.e2e.tests=true; then
- echo -e "${GREEN}✓ Java VARIANT write test completed successfully${NC}"
- else
- echo -e "${RED}✗ Java VARIANT write test failed${NC}"
- return 1
+ echo "Running Maven test for
JavaPyE2ETest.testJavaWriteVariantTable..."
+ if mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testJavaWriteVariantTable -pl
paimon-core -q -Drun.e2e.tests=true; then
+ echo -e "${GREEN}✓ Java VARIANT write test completed
successfully${NC}"
+ else
+ echo -e "${RED}✗ Java VARIANT write test failed${NC}"
+ return 1
+ fi
fi
cd "$PAIMON_PYTHON_DIR"
echo "Running Python test for
JavaPyReadWriteTest.test_py_read_variant_table..."
@@ -799,14 +947,16 @@ run_py_variant_write_java_read_test() {
run_row_format_test() {
echo -e "${YELLOW}=== Running ROW Format Test (Java Write → Python Read,
Python Write → Java Read) ===${NC}"
- cd "$PROJECT_ROOT"
+ if ! skip_batched_java_write; then
+ cd "$PROJECT_ROOT"
- echo "Running Maven test for JavaPyE2ETest.testJavaWriteRowAppendTable..."
- if ! mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testJavaWriteRowAppendTable -pl
paimon-core -q -Drun.e2e.tests=true; then
- echo -e "${RED}✗ Java ROW write test failed${NC}"
- return 1
+ echo "Running Maven test for
JavaPyE2ETest.testJavaWriteRowAppendTable..."
+ if ! mvn test
-Dtest=org.apache.paimon.JavaPyE2ETest#testJavaWriteRowAppendTable -pl
paimon-core -q -Drun.e2e.tests=true; then
+ echo -e "${RED}✗ Java ROW write test failed${NC}"
+ return 1
+ fi
+ echo -e "${GREEN}✓ Java ROW write test completed successfully${NC}"
fi
- echo -e "${GREEN}✓ Java ROW write test completed successfully${NC}"
cd "$PAIMON_PYTHON_DIR"
echo "Running Python test for
JavaPyReadWriteTest.test_read_row_append_table..."
@@ -876,6 +1026,19 @@ main() {
echo -e "${YELLOW}Starting mixed language test execution...${NC}"
echo ""
+ if java_write_batch_enabled; then
+ if run_batched_java_write_tests; then
+ JAVA_WRITE_BATCH_DONE=true
+ echo -e "${GREEN}✓ Batched Java write phase completed; later Java
write steps will be skipped${NC}"
+ else
+ JAVA_WRITE_BATCH_DONE=false
+ echo -e "${RED}✗ Batched Java write phase failed${NC}"
+ echo -e "${YELLOW}Rerun with MIXED_TESTS_BATCH_JAVA_WRITES=false
for the original one-by-one Java write flow.${NC}"
+ return 1
+ fi
+ echo ""
+ fi
+
# Run Java write test
if ! run_java_write_test; then
java_write_result=1
