This is an automated email from the ASF dual-hosted git repository.
kirs pushed a change to branch auto-pick-54555-branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
from c852138aa36 [case](new-catalog-property) add paimon cases (#54555)
add 8c1c25b3dc6 [bugfix](be) should set is inited = true only when init
successfully and not update counters if init == false (#54927)
add 71e1a18458a branch-3.1: [Fix](compaction) Remove incorrect DCHECK
#54650 (#54745)
add 2b84e052028 branch-3.1: [Fix](case) Fix case
`test_cloud_mow_new_tablet_compaction` #54736 (#54799)
add 0aecfef6309 branch-3.1: [chore](cloud) Remove deprecated
update_tablet_schema #54676 (#54838)
add 6827d9126e3 [fix](cases) fix global variables (#54866)
add 7fe4cf924de branch-3.1: [fix](prepare statement)Support FE execute
COM_STMT_EXECUTE show command. #54446 (#54876)
add 86ff07acbdc branch-3.1: [fix](nereids) fix regression case bug:
push_down_top_n_through_union #54480 (#54878)
add 394df088955 branch-3.1: [fix](variant)fix top-array data into variant
column #54396 (#54867)
add 0788fbc0e14 branch-3.1: [fix](regression)Fix test_analyze_mv #54727
(#54881)
add 4b87c30cb4b branch-3.1: [fix](mtmv) Fix aggregate function roll up
fail when function is always nullable #52960 (#54802)
add 97df48f40fa [fix](case) fix 230 message for case (#54895)
add 498199b22d5 branch-3.1: [fix](case) enable variant flattern in
check_before_quit #54893 (#54901)
add 11554bf868f branch-3.1: [fix](schema-change) Fix sync MV loss due to
index change #54285 (#54782)
add 4738eada239 branch-3.1: [fix](test) Disable alter resource test in
cloud mode #54823 (#54907)
add 96acf86b2b6 branch-3.1: [fix](prepare statement)Fix date_trunc using
prepare statement parameter type bug. #54847 (#54920)
add 589c0772e14 [fix](case) fix merge bug (#54921)
add c3779a08bc9 branch-3.1: [fix](variant) fix variant column reader
#54830 (#54922)
add eed65309c46 [fix](case) retry if stream load hangs (#54955)
add ddf98053387 branch-3.1: [feat](paimon) support paimon dlf rest catalog
(#54523) (#54887)
add 91503d80f40 branch-3.1: [chore](iceberg)Support Iceberg HMS Catalog
with FileIO integration #54951 (#54965)
add db56658c7c5 branch-3.1: [fix](test) increase timeout in
test_broker_load_func #54783 (#54811)
add 650e09dca1b [fix](case) fix some cases (#54897)
add 8150ae6958d branch-3.1: [fix](case) fix single replica load #54899
(#54915)
add 36e5ab41b00 branch-3.1: [fix](hudi) fix hudi p2 case #54948 (#54966)
add 70c2ebc68e7 branch-3.1: [Fix](case) Fix case
`test_cloud_mow_correctness_inject` due to config name change #54738 (#54800)
add b0f4caac7d3 branch-3.1: [chore](case) do not use global var in groovy
#54700 (#54806)
add 4c776839b18 branch-3.1: [fix](test) Fix hudi olap mv rewrite test
#54681 (#54879)
add 36b2dbe7ff6 branch-3.1: [fix](case) make some cases stable #54763
(#54982)
add 1647a198a71 branch-3.1: [Fix](docker case) Fix
`test_clean_tablet_when_drop_force_table` run … #54803 (#54892)
add 1cdab3de053 branch-3.1: [chore](paimon)Unify object storage access via
HDFS protocol #54776 (#54979)
add 2a7120ae45f branch-3.1: [chore](case) fix some cases #54885 (#54903)
add 0ef4918818c branch-3.1: [test](case) rename session var
disable_variant_flatten_nested to enable_variant_flatten_nested #54931 (#54934)
add f834f6d9f9a branch-3.1: [fix](deadlock) avoid deadlock on
tabletInvertedIndex #54197 (#54997)
add 30aaeeb1131 branch-3.1: [chore](param-refactor)Support independent AWS
connection timeout settings for each object storage #54882 (#54980)
add 5be3b92690c branch-3.1: [feat](paimon) Support paimon time travel and
branch/tag #53327 (#54975)
add 56e22695a84 branch-3.1: [fix](hdfs) remove cached file handle when
read fails #54926 (#54992)
add 6378509c3e0 [fix](cases) fix cases for variant (#55019)
add d28f7431500 Merge branch 'branch-3.1' into auto-pick-54555-branch-3.1
No new revisions were added by this update.
Summary of changes:
be/src/cloud/cloud_tablet.cpp | 7 -
be/src/io/fs/hdfs_file_reader.cpp | 17 +-
be/src/io/fs/hdfs_file_reader.h | 3 +
be/src/olap/compaction.cpp | 8 -
.../segment_v2/variant/variant_column_reader.cpp | 16 +-
be/src/olap/rowset_version_mgr.cpp | 3 +-
be/src/vec/columns/column_object.cpp | 11 +-
be/src/vec/exec/scan/new_olap_scanner.cpp | 2 +-
be/src/vec/exec/scan/scanner_scheduler.cpp | 6 +-
be/src/vec/json/json_parser.cpp | 176 ++++++---
be/src/vec/json/json_parser.h | 9 +
.../variant_column_writer_reader_test.cpp | 186 +++++++++-
be/test/vec/jsonb/json_parser_test.cpp | 175 +++++++++
cloud/src/common/bvars.cpp | 7 -
cloud/src/common/bvars.h | 5 -
cloud/src/meta-service/meta_service.cpp | 74 ----
cloud/src/meta-service/meta_service.h | 12 -
cloud/test/rpc_kv_bvar_test.cpp | 30 --
.../docker-compose/hive/hadoop-hive-3x.env.tpl | 2 +
.../create_preinstalled_scripts/paimon/run09.sql | 88 +++++
.../docker-compose/kerberos/hadoop-hive.env.tpl | 2 +
.../org/apache/doris/paimon/PaimonJniScanner.java | 44 +--
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 2 +-
.../org/apache/doris/alter/SchemaChangeJobV2.java | 8 +-
.../org/apache/doris/analysis/TableScanParams.java | 1 +
.../java/org/apache/doris/common/util/S3URI.java | 2 +-
.../apache/doris/datasource/CatalogProperty.java | 7 +-
.../credentials/CredentialExtractor.java} | 27 +-
.../credentials/CredentialUtils.java} | 21 +-
.../iceberg/IcebergS3CredentialExtractor.java | 53 +++
.../iceberg/IcebergVendedCredentialsProvider.java | 72 +---
.../paimon/PaimonDLFExternalCatalog.java | 1 +
.../datasource/paimon/PaimonExternalCatalog.java | 40 +-
.../paimon/PaimonExternalCatalogFactory.java | 1 +
.../datasource/paimon/PaimonExternalTable.java | 6 +-
.../paimon/PaimonFileExternalCatalog.java | 1 +
.../paimon/PaimonHMSExternalCatalog.java | 1 +
.../paimon/PaimonOssCredentialExtractor.java | 68 ++++
.../apache/doris/datasource/paimon/PaimonUtil.java | 291 +++++++++++++--
.../paimon/PaimonVendedCredentialsProvider.java | 100 +++++
.../datasource/paimon/source/PaimonScanNode.java | 56 ++-
.../metastore/AbstractIcebergProperties.java | 25 +-
.../metastore/AbstractPaimonProperties.java | 33 +-
.../metastore/HMSGlueMetaStoreProperties.java | 4 +
.../IcebergAliyunDLFMetaStoreProperties.java | 10 +-
.../IcebergFileSystemMetaStoreProperties.java | 5 +-
.../metastore/IcebergGlueMetaStoreProperties.java | 17 +-
.../metastore/IcebergHMSMetaStoreProperties.java | 15 +-
.../property/metastore/IcebergRestProperties.java | 3 +-
.../IcebergS3TablesMetaStoreProperties.java | 19 +-
.../PaimonAliyunDLFMetaStoreProperties.java | 1 -
.../metastore/PaimonPropertiesFactory.java | 1 +
.../metastore/PaimonRestMetaStoreProperties.java | 111 ++++++
.../storage/AbstractS3CompatibleProperties.java | 127 +++----
.../datasource/property/storage/COSProperties.java | 53 ++-
.../property/storage/MinioProperties.java | 57 +++
.../datasource/property/storage/OBSProperties.java | 56 +++
.../datasource/property/storage/OSSProperties.java | 106 +++++-
.../property/storage/ObjectStorageProperties.java | 11 +
.../datasource/property/storage/S3Properties.java | 29 +-
.../property/storage/S3PropertyUtils.java | 5 +-
.../java/org/apache/doris/fs/obj/S3ObjStorage.java | 38 +-
.../ContainDistinctFunctionRollupHandler.java | 9 +-
.../expressions/functions/scalar/DateTrunc.java | 13 +-
.../trees/plans/logical/LogicalHudiScan.java | 4 +-
.../org/apache/doris/qe/MysqlConnectProcessor.java | 2 +-
.../java/org/apache/doris/qe/StmtExecutor.java | 97 ++++-
.../tablefunction/PaimonTableValuedFunction.java | 4 +-
.../doris/transaction/DatabaseTransactionMgr.java | 16 +-
.../IcebergVendedCredentialsProviderTest.java | 25 +-
.../PaimonVendedCredentialsProviderTest.java | 407 +++++++++++++++++++++
...ogTest.java => IcebergGlueRestCatalogTest.java} | 78 ++--
.../property/metastore/PaimonCatalogTest.java | 95 +++++
.../metastore/PaimonDlfRestCatalogTest.java | 285 +++++++++++++++
.../PaimonRestMetaStorePropertiesTest.java | 400 ++++++++++++++++++++
.../property/storage/COSPropertiesTest.java | 16 +-
.../property/storage/MinioPropertiesTest.java | 21 +-
.../property/storage/OBSPropertyTest.java | 68 ++--
.../property/storage/OSSPropertiesTest.java | 58 +--
.../property/storage/S3PropertiesTest.java | 44 ++-
.../java/org/apache/doris/qe/StmtExecutorTest.java | 100 +++++
.../data/cloud_p0/tablets/all_types.csv.gz | Bin 0 -> 19554 bytes
.../paimon/paimon_time_travel.out | Bin 0 -> 40401 bytes
...atalog.out => test_paimon_dlf_rest_catalog.out} | Bin 133 -> 179 bytes
.../agg_use_key_direct/agg_use_key_direct.out | Bin 210 -> 212 bytes
.../push_down_topn_through_union.out | Bin 6411 -> 0 bytes
.../data/prepared_stmt_p0/prepared_show.out | Bin 0 -> 242 bytes
.../data/prepared_stmt_p0/prepared_stmt.out | Bin 5355 -> 5389 bytes
.../test_index_change_after_mv.out | Bin 0 -> 326 bytes
.../data/variant_p0/nested/nested_in_top_array.out | Bin 0 -> 5217 bytes
.../alter_p2/test_alter_colocate_group.groovy | 3 +
.../check_before_quit/check_before_quit.groovy | 1 +
.../test_schema_change_with_compaction11.groovy | 18 +-
.../test_clean_tablet_when_drop_force_table.groovy | 112 +++---
.../test_single_compaction_fault_injection.groovy | 4 +
.../csv_header_p0/test_csv_with_header.groovy | 2 +-
.../suites/export/test_array_export.groovy | 2 +-
.../broker_load/test_broker_load_func.groovy | 2 +-
.../jdbc/test_jdbc_query_mysql.groovy | 14 +-
.../paimon/paimon_incr_read.groovy | 4 -
.../paimon/paimon_time_travel.groovy | 278 ++++++++++++++
.../hudi_mtmv/test_hudi_olap_rewrite_mtmv.groovy | 2 +-
.../hudi/hudi_mtmv/test_hudi_rewrite_mtmv.groovy | 2 +-
.../iceberg/test_glue_rest_s3tables.groovy | 8 +-
....groovy => test_paimon_dlf_rest_catalog.groovy} | 34 +-
...action_get_delete_bitmap_from_cache_fail.groovy | 19 +-
.../cloud/test_cloud_mow_correctness_inject.groovy | 2 +-
.../test_cloud_mow_new_tablet_compaction.groovy | 8 +-
.../test_cloud_mow_partial_update_retry.groovy | 6 +-
..._cloud_mow_stale_resp_load_load_conflict.groovy | 6 +-
...test_cloud_sc_self_retry_with_stop_token.groovy | 2 +-
.../test_sc_fail_release_delete_bitmap_lock.groovy | 4 +-
.../test_fix_tablet_stat_fault_injection.groovy | 6 +-
.../test_load_stream_fault_injection.groovy | 6 +
.../test_segcompaction_fault_injection.groovy | 5 +
.../test_variant_bloom_filter.groovy | 7 +-
.../load_p0/broker_load/test_array_load.groovy | 2 +-
.../broker_load/test_broker_load_seq.groovy | 2 +-
.../test_broker_load_with_partition.groovy | 2 +-
.../test_broker_load_with_properties.groovy | 2 +-
.../broker_load/test_broker_load_with_where.groovy | 2 +-
...t_domain_connection_and_ak_sk_correction.groovy | 2 +-
.../test_ingestion_load_with_partition.groovy | 2 +-
.../test_insert_random_distribution_table.groovy | 6 +
.../load_p0/stream_load/test_json_load.groovy | 2 +-
.../suites/load_p2/test_single_replica_load.groovy | 9 +-
.../agg_use_key_direct/agg_use_key_direct.groovy | 2 +-
.../mv/agg_variety/agg_variety.groovy | 22 --
regression-test/suites/opensky_p2/load.groovy | 1 +
.../suites/prepared_stmt_p0/prepared_show.groovy | 54 +++
.../suites/prepared_stmt_p0/prepared_stmt.groovy | 6 +
.../test_dup_mv_schema_change.groovy | 1 +
.../test_index_change_after_mv.groovy | 73 ++++
.../suites/statistics/test_analyze_mv.groovy | 28 +-
.../suites/tpcds_sf1_unique_p1/sql/q24_2.sql | 4 -
.../test_partial_update_conflict_be_restart.groovy | 1 -
.../load.groovy | 2 +-
.../suites/variant_p0/nested/load.groovy | 4 +-
.../variant_p0/nested/nested_in_top_array.groovy | 131 +++++++
.../suites/variant_p0/with_index/var_index.groovy | 32 +-
.../suites/variant_p2/tpch_upgrade/load.groovy | 2 +-
141 files changed, 4084 insertions(+), 976 deletions(-)
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run09.sql
copy
fe/fe-core/src/main/java/org/apache/doris/{mtmv/MTMVRelatedPartitionDescGeneratorService.java
=> datasource/credentials/CredentialExtractor.java} (52%)
copy
fe/fe-core/src/main/java/org/apache/doris/{mysql/privilege/AccessControllerFactory.java
=> datasource/credentials/CredentialUtils.java} (55%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergS3CredentialExtractor.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonOssCredentialExtractor.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonVendedCredentialsProvider.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonRestMetaStoreProperties.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonVendedCredentialsProviderTest.java
rename
fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/{RestCatalogTest.java
=> IcebergGlueRestCatalogTest.java} (88%)
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonCatalogTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonDlfRestCatalogTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonRestMetaStorePropertiesTest.java
create mode 100644 regression-test/data/cloud_p0/tablets/all_types.csv.gz
create mode 100644
regression-test/data/external_table_p0/paimon/paimon_time_travel.out
copy
regression-test/data/external_table_p2/paimon/{test_paimon_dlf_catalog.out =>
test_paimon_dlf_rest_catalog.out} (59%)
delete mode 100644
regression-test/data/nereids_rules_p0/push_down_topn/push_down_topn_through_union.out
create mode 100644 regression-test/data/prepared_stmt_p0/prepared_show.out
create mode 100644
regression-test/data/schema_change_p0/test_index_change_after_mv.out
create mode 100644
regression-test/data/variant_p0/nested/nested_in_top_array.out
create mode 100644
regression-test/suites/external_table_p0/paimon/paimon_time_travel.groovy
copy
regression-test/suites/external_table_p2/paimon/{test_paimon_dlf_catalog.groovy
=> test_paimon_dlf_rest_catalog.groovy} (51%)
create mode 100644 regression-test/suites/prepared_stmt_p0/prepared_show.groovy
create mode 100644
regression-test/suites/schema_change_p0/test_index_change_after_mv.groovy
create mode 100644
regression-test/suites/variant_p0/nested/nested_in_top_array.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
