This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8417544e7e2 [test](hudi) add more hudi jni test cases (#47431)
8417544e7e2 is described below
commit 8417544e7e280b247b6e8517c5b333d26b408b82
Author: Socrates <suyit...@selectdb.com>
AuthorDate: Mon Feb 10 16:10:10 2025 +0800
[test](hudi) add more hudi jni test cases (#47431)
### What problem does this PR solve?
add more hudi jni test cases.
These tests depends on #47192 and part of #47299(the jni-util part)
---
.../hudi/test_hudi_incremental.out | Bin 5687 -> 7551 bytes
.../hudi/test_hudi_schema_evolution.out | Bin 7521 -> 11234 bytes
.../external_table_p2/hudi/test_hudi_snapshot.out | Bin 696105 -> 1044838
bytes
.../hudi/test_hudi_timetravel.out | Bin 2111 -> 3119 bytes
.../hudi/test_hudi_incremental.groovy | 18 +++++++++++++-----
.../hudi/test_hudi_schema_evolution.groovy | 16 ++++++++++++++--
.../hudi/test_hudi_snapshot.groovy | 10 +++++++++-
.../hudi/test_hudi_timetravel.groovy | 11 ++++++++++-
8 files changed, 46 insertions(+), 9 deletions(-)
diff --git
a/regression-test/data/external_table_p2/hudi/test_hudi_incremental.out
b/regression-test/data/external_table_p2/hudi/test_hudi_incremental.out
index 50644f34961..852aebe4896 100644
Binary files
a/regression-test/data/external_table_p2/hudi/test_hudi_incremental.out and
b/regression-test/data/external_table_p2/hudi/test_hudi_incremental.out differ
diff --git
a/regression-test/data/external_table_p2/hudi/test_hudi_schema_evolution.out
b/regression-test/data/external_table_p2/hudi/test_hudi_schema_evolution.out
index da7273d4c14..79f36e7124c 100644
Binary files
a/regression-test/data/external_table_p2/hudi/test_hudi_schema_evolution.out
and
b/regression-test/data/external_table_p2/hudi/test_hudi_schema_evolution.out
differ
diff --git a/regression-test/data/external_table_p2/hudi/test_hudi_snapshot.out
b/regression-test/data/external_table_p2/hudi/test_hudi_snapshot.out
index 114ca5b0a89..833ccab2ccd 100644
Binary files
a/regression-test/data/external_table_p2/hudi/test_hudi_snapshot.out and
b/regression-test/data/external_table_p2/hudi/test_hudi_snapshot.out differ
diff --git
a/regression-test/data/external_table_p2/hudi/test_hudi_timetravel.out
b/regression-test/data/external_table_p2/hudi/test_hudi_timetravel.out
index 00d15805baf..4e6d98a2aac 100644
Binary files
a/regression-test/data/external_table_p2/hudi/test_hudi_timetravel.out and
b/regression-test/data/external_table_p2/hudi/test_hudi_timetravel.out differ
diff --git
a/regression-test/suites/external_table_p2/hudi/test_hudi_incremental.groovy
b/regression-test/suites/external_table_p2/hudi/test_hudi_incremental.groovy
index 87e3e4a9c6a..6c73804a0cf 100644
--- a/regression-test/suites/external_table_p2/hudi/test_hudi_incremental.groovy
+++ b/regression-test/suites/external_table_p2/hudi/test_hudi_incremental.groovy
@@ -104,17 +104,25 @@ suite("test_hudi_incremental",
"p2,external,hudi,external_remote,external_remote
"20241114152334111",
]
- test_hudi_incremental_querys("user_activity_log_cow_non_partition",
timestamps_cow_non_partition)
- test_hudi_incremental_querys("user_activity_log_cow_partition",
timestamps_cow_partition)
+ sql """set force_jni_scanner=true;"""
+ sql """set hudi_jni_scanner='hadoop';"""
+ // TODO: @suxiaogang223 don't support incremental query for cow table by
jni reader
+ // test_hudi_incremental_querys("user_activity_log_cow_non_partition",
timestamps_cow_non_partition)
+ // test_hudi_incremental_querys("user_activity_log_cow_partition",
timestamps_cow_partition)
test_hudi_incremental_querys("user_activity_log_mor_non_partition",
timestamps_mor_non_partition)
test_hudi_incremental_querys("user_activity_log_mor_partition",
timestamps_mor_partition)
- sql """set force_jni_scanner=true;"""
- // don't support incremental query for cow table by jni reader
+
+ sql """set hudi_jni_scanner='spark';"""
// test_hudi_incremental_querys("user_activity_log_cow_non_partition",
timestamps_cow_non_partition)
// test_hudi_incremental_querys("user_activity_log_cow_partition",
timestamps_cow_partition)
test_hudi_incremental_querys("user_activity_log_mor_non_partition",
timestamps_mor_non_partition)
test_hudi_incremental_querys("user_activity_log_mor_partition",
timestamps_mor_partition)
- // sql """set force_jni_scanner=false;"""
+
+ sql """set force_jni_scanner=false;"""
+ test_hudi_incremental_querys("user_activity_log_cow_non_partition",
timestamps_cow_non_partition)
+ test_hudi_incremental_querys("user_activity_log_cow_partition",
timestamps_cow_partition)
+ test_hudi_incremental_querys("user_activity_log_mor_non_partition",
timestamps_mor_non_partition)
+ test_hudi_incremental_querys("user_activity_log_mor_partition",
timestamps_mor_partition)
sql """drop catalog if exists ${catalog_name};"""
}
diff --git
a/regression-test/suites/external_table_p2/hudi/test_hudi_schema_evolution.groovy
b/regression-test/suites/external_table_p2/hudi/test_hudi_schema_evolution.groovy
index 176484f25c6..bb99f3e7391 100644
---
a/regression-test/suites/external_table_p2/hudi/test_hudi_schema_evolution.groovy
+++
b/regression-test/suites/external_table_p2/hudi/test_hudi_schema_evolution.groovy
@@ -35,6 +35,19 @@ suite("test_hudi_schema_evolution",
"p2,external,hudi,external_remote,external_r
sql """ use regression_hudi;"""
sql """ set enable_fallback_to_original_planner=false """
+ sql """set force_jni_scanner = true;"""
+ sql """set hudi_jni_scanner='hadoop';"""
+ qt_adding_simple_columns_table """ select * from
adding_simple_columns_table order by id """
+ qt_altering_simple_columns_table """ select * from
altering_simple_columns_table order by id """
+ // qt_deleting_simple_columns_table """ select * from
deleting_simple_columns_table order by id """
+ // qt_renaming_simple_columns_table """ select * from
renaming_simple_columns_table order by id """
+
+ qt_adding_complex_columns_table """ select * from
adding_complex_columns_table order by id """
+ qt_altering_complex_columns_table """ select * from
altering_complex_columns_table order by id """
+ // qt_deleting_complex_columns_table """ select * from
deleting_complex_columns_table order by id """
+ // qt_renaming_complex_columns_table """ select * from
renaming_complex_columns_table order by id """
+
+ sql """set hudi_jni_scanner='spark';"""
qt_adding_simple_columns_table """ select * from
adding_simple_columns_table order by id """
qt_altering_simple_columns_table """ select * from
altering_simple_columns_table order by id """
// qt_deleting_simple_columns_table """ select * from
deleting_simple_columns_table order by id """
@@ -45,7 +58,7 @@ suite("test_hudi_schema_evolution",
"p2,external,hudi,external_remote,external_r
// qt_deleting_complex_columns_table """ select * from
deleting_complex_columns_table order by id """
// qt_renaming_complex_columns_table """ select * from
renaming_complex_columns_table order by id """
- sql """set force_jni_scanner = true;"""
+ sql """set force_jni_scanner = false;"""
qt_adding_simple_columns_table """ select * from
adding_simple_columns_table order by id """
qt_altering_simple_columns_table """ select * from
altering_simple_columns_table order by id """
// qt_deleting_simple_columns_table """ select * from
deleting_simple_columns_table order by id """
@@ -55,7 +68,6 @@ suite("test_hudi_schema_evolution",
"p2,external,hudi,external_remote,external_r
qt_altering_complex_columns_table """ select * from
altering_complex_columns_table order by id """
// qt_deleting_complex_columns_table """ select * from
deleting_complex_columns_table order by id """
// qt_renaming_complex_columns_table """ select * from
renaming_complex_columns_table order by id """
- sql """set force_jni_scanner = false;"""
sql """drop catalog if exists ${catalog_name};"""
}
diff --git
a/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy
b/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy
index d7f5488c055..b2fd574276d 100644
--- a/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy
+++ b/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy
@@ -82,17 +82,25 @@ suite("test_hudi_snapshot",
"p2,external,hudi,external_remote,external_remote_hu
qt_q15 """SELECT user_id, array_size(purchases) AS purchase_count FROM
${table_name} ORDER BY user_id LIMIT 5;"""
}
+ sql """set force_jni_scanner=true;"""
+ sql """set hudi_jni_scanner='hadoop';"""
test_hudi_snapshot_querys("user_activity_log_mor_non_partition")
test_hudi_snapshot_querys("user_activity_log_mor_partition")
test_hudi_snapshot_querys("user_activity_log_cow_non_partition")
test_hudi_snapshot_querys("user_activity_log_cow_partition")
- sql """set force_jni_scanner=true;"""
+ sql """set hudi_jni_scanner='spark';"""
test_hudi_snapshot_querys("user_activity_log_mor_non_partition")
test_hudi_snapshot_querys("user_activity_log_mor_partition")
test_hudi_snapshot_querys("user_activity_log_cow_non_partition")
test_hudi_snapshot_querys("user_activity_log_cow_partition")
+
sql """set force_jni_scanner=false;"""
+ test_hudi_snapshot_querys("user_activity_log_mor_non_partition")
+ test_hudi_snapshot_querys("user_activity_log_mor_partition")
+ test_hudi_snapshot_querys("user_activity_log_cow_non_partition")
+ test_hudi_snapshot_querys("user_activity_log_cow_partition")
+
sql """drop catalog if exists ${catalog_name};"""
}
diff --git
a/regression-test/suites/external_table_p2/hudi/test_hudi_timetravel.groovy
b/regression-test/suites/external_table_p2/hudi/test_hudi_timetravel.groovy
index f1e55a86b83..789db7f04e3 100644
--- a/regression-test/suites/external_table_p2/hudi/test_hudi_timetravel.groovy
+++ b/regression-test/suites/external_table_p2/hudi/test_hudi_timetravel.groovy
@@ -98,16 +98,25 @@ suite("test_hudi_timetravel",
"p2,external,hudi,external_remote,external_remote_
"20241114152334111",
]
+ sql """set force_jni_scanner=true;"""
+ sql """set hudi_jni_scanner='hadoop';"""
test_hudi_timetravel_querys("user_activity_log_cow_non_partition",
timestamps_cow_non_partition)
test_hudi_timetravel_querys("user_activity_log_cow_partition",
timestamps_cow_partition)
test_hudi_timetravel_querys("user_activity_log_mor_non_partition",
timestamps_mor_non_partition)
test_hudi_timetravel_querys("user_activity_log_mor_partition",
timestamps_mor_partition)
- sql """set force_jni_scanner=true;"""
+
+ sql """set hudi_jni_scanner='spark';"""
test_hudi_timetravel_querys("user_activity_log_cow_non_partition",
timestamps_cow_non_partition)
test_hudi_timetravel_querys("user_activity_log_cow_partition",
timestamps_cow_partition)
test_hudi_timetravel_querys("user_activity_log_mor_non_partition",
timestamps_mor_non_partition)
test_hudi_timetravel_querys("user_activity_log_mor_partition",
timestamps_mor_partition)
+
sql """set force_jni_scanner=false;"""
+ test_hudi_timetravel_querys("user_activity_log_cow_non_partition",
timestamps_cow_non_partition)
+ test_hudi_timetravel_querys("user_activity_log_cow_partition",
timestamps_cow_partition)
+ test_hudi_timetravel_querys("user_activity_log_mor_non_partition",
timestamps_mor_non_partition)
+ test_hudi_timetravel_querys("user_activity_log_mor_partition",
timestamps_mor_partition)
+
sql """drop catalog if exists ${catalog_name};"""
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
