This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a change to branch auto-pick-66238-branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
discard d62eb6d2a40 [fix](streaming-job) Optimize snapshot offset persistence
(#66238)
add 06ffaef2520 branch-4.1: [feature](file-cache) Disable cache writes
after remote scan threshold (#66479)
add 3ba435c26e0 branch-4.1: [fix](be) Clean up spill directories during
query teardown (#66458)
add a15b79e4940 branch-4.1: [improvement](be) Add instance ID to hash join
profiles #66097 (#66453)
add e48fc7e97b9 [fix](test) Unset Paimon Variant shredding property
(#66536)
add f0e00809c81 branch-4.1:[chore](thirdparty) Upgrade Apache Arrow to
24.0.0(#66221) (#66480)
add ca02e7b3839 [fix](regression) Stabilize external table guardrail cases
(#66509)
add c3cfb3805d4 [opt](paimon) report error when mapping variant type
disable variant_v2 (#66466)
add d39e68c81ed [fix](be) Preserve floating-point equality in Parquet
pruning (#66470)
add 6c8ccecbd1e branch-4.1: [improvement](repeat) Optimize shuffle key
selection for Repeat decomposition #66532 (#66568)
add 762147325b7 branch-4.1: [fix](topn) Handle empty rowid fetch RPC
failures #66443 (#66522)
add 895c29374e0 branch-4.1: [Feature](multimodal_embed) Support
multimodal(file) embed (#66461)
add 6a4700877d6 [opt](paimon) support variant access path in paimon jni
reader (#66547)
add c76364c5aa3 branch-4.1: [fix](be) Stop spill GC threads in workload
group test teardown #66534 (#66544)
add 30d2ba72b84 [fix](be) Safely prune nested Parquet columns with Bloom
filters (#66471)
add fdeff73267d [fix](be) Backport nullable hash state exception safety to
branch-4.1 (#66517)
add 8375559ce88 [refactor](lance) improve lance catalog implementation in
FE (#66581)
add 688e3ce5e41 [opt](paimon) support prune complex type in paimon jni
reader (#66573)
add 43ed21eb7e4 branch-4.1: [fix](build) Exclude javax.resource:connector
from BDB JE dependency #66566 (#66569)
add 0012df65a1f branch-4.1: [fix](exec) preserve NULL in pushed-down CHAR
MIN/MAX #65952 (#66583)
add c6cde0c7148 branch-4.1: [fix](regression) Stabilize BE compaction
tasks test #65637 (#66543)
add 23772a38cbf branch-4.1: [Fix](build) Manage datasketches-cpp in the BE
CMake build tree (#66511) (#66577)
add 1d147d8ec65 branch-4.1: [fix](regression) Fetch streaming job metrics
from master FE #66460 (#66540)
add 4c2c191cebc branch-4.1: [feature](paimon) Support Variant V2 writes
(#66321)
add 847157cdf11 [enhancement](variant) Support deep Variant object leaf
projection (#66575)
add dc05b1e9477 branch-4.1: [fix](schema-change) Use actual schema change
alter version in cloud capture #62506 (#66606)
add 874089ab056 [opt](be) Backport nullable and projection optimizations
to branch-4.1 (#66586)
add e0c08a1e712 [fix](streaming-job) Optimize snapshot offset persistence
(#66238)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (d62eb6d2a40)
\
N -- N -- N refs/heads/auto-pick-66238-branch-4.1 (e0c08a1e712)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/workflows/be-ut-mac.yml | 50 +-
.github/workflows/build-thirdparty.yml | 38 +-
be/CMakeLists.txt | 12 +-
be/cmake/thirdparty.cmake | 1 +
be/src/cloud/cloud_schema_change_job.cpp | 6 +-
be/src/common/config.cpp | 2 +
be/src/common/config.h | 1 +
be/src/core/block/column_with_type_and_name.cpp | 88 +-
be/src/core/block/column_with_type_and_name.h | 19 +
be/src/core/column/column.h | 11 +
be/src/core/column/column_const.h | 9 +
be/src/core/column/column_nullable.cpp | 23 +
be/src/core/column/column_nullable.h | 8 +
be/src/core/column/column_variant.cpp | 14 +
be/src/core/column/column_variant.h | 2 +
.../data_type_serde/data_type_variant_v2_serde.cpp | 133 +++
be/src/core/field.h | 13 +
be/src/exec/common/agg_utils.h | 13 +
be/src/exec/common/columns_hashing.h | 9 +-
be/src/exec/common/hash_table/hash_map_context.h | 12 +-
be/src/exec/operator/aggregation_sink_operator.cpp | 20 +-
be/src/exec/operator/hashjoin_build_sink.cpp | 1 +
be/src/exec/operator/hashjoin_probe_operator.cpp | 2 +
be/src/exec/operator/materialization_opertor.cpp | 57 +-
be/src/exec/operator/materialization_opertor.h | 1 +
be/src/exec/operator/operator.cpp | 80 +-
be/src/exec/operator/operator.h | 4 -
.../operator/streaming_aggregation_operator.cpp | 9 +-
be/src/exec/scan/access_path_parser.cpp | 48 +-
be/src/exec/scan/file_scan_io_context.h | 44 +
be/src/exec/scan/file_scanner.h | 4 +-
be/src/exec/scan/file_scanner_v2.cpp | 4 +-
be/src/exec/scan/olap_scanner.cpp | 27 +-
be/src/exec/scan/olap_scanner.h | 9 +
be/src/exec/scan/parallel_scanner_builder.cpp | 21 +-
be/src/exec/scan/scanner.cpp | 62 +-
.../writer/paimon/jni_paimon_write_backend.cpp | 70 +-
be/src/exec/spill/spill_file.cpp | 4 +-
be/src/exec/spill/spill_file_manager.cpp | 85 +-
be/src/exec/spill/spill_file_manager.h | 25 +-
be/src/exec/spill/spill_file_writer.cpp | 3 +
be/src/exprs/CMakeLists.txt | 4 +-
...aggregate_function_datasketches_hll_union_agg.h | 2 +-
be/src/exprs/expr_zonemap_filter.cpp | 198 +++-
be/src/exprs/expr_zonemap_filter.h | 34 +-
be/src/exprs/function/ai/ai_adapter.h | 433 ++++++-
be/src/exprs/function/ai/ai_classify.h | 4 +-
be/src/exprs/function/ai/ai_extract.h | 4 +-
be/src/exprs/function/ai/ai_filter.h | 4 +-
be/src/exprs/function/ai/ai_fix_grammar.h | 2 +-
be/src/exprs/function/ai/ai_functions.cpp | 164 +--
be/src/exprs/function/ai/ai_functions.h | 215 ++--
be/src/exprs/function/ai/ai_generate.h | 4 +-
be/src/exprs/function/ai/ai_mask.h | 4 +-
be/src/exprs/function/ai/ai_sentiment.h | 2 +-
be/src/exprs/function/ai/ai_similarity.h | 4 +-
be/src/exprs/function/ai/ai_summarize.h | 2 +-
be/src/exprs/function/ai/ai_translate.h | 4 +-
be/src/exprs/function/ai/embed.h | 371 +++++-
be/src/exprs/function/cast/function_cast.cpp | 27 +-
.../cast/variant_v2/cast_array_to_variant.cpp | 6 +-
.../exprs/function/comparison_equal_for_null.cpp | 16 +-
be/src/exprs/function/function.cpp | 60 +-
be/src/exprs/function/function.h | 3 +
be/src/exprs/function/functions_comparison.h | 22 +-
be/src/exprs/hybrid_set.h | 18 +-
be/src/exprs/vdirect_in_predicate.h | 5 +-
be/src/exprs/vectorized_fn_call.cpp | 19 +-
be/src/exprs/vin_predicate.cpp | 20 +-
be/src/exprs/vin_predicate.h | 1 +
be/src/format/jni/jni_data_bridge.cpp | 2 +-
be/src/format/jni/jni_data_bridge.h | 4 +-
be/src/format/table/iceberg/arrow_schema_util.cpp | 2 +-
.../table/iceberg_delete_file_reader_helper.cpp | 8 +-
be/src/format/table/paimon_jni_reader.cpp | 4 +-
be/src/format_v2/column_mapper.cpp | 220 +++-
be/src/format_v2/column_mapper_nested.cpp | 76 +-
be/src/format_v2/column_mapper_nested.h | 4 +
be/src/format_v2/file_reader.cpp | 2 +
be/src/format_v2/file_reader.h | 7 +
be/src/format_v2/jni/jni_table_reader.cpp | 4 +-
be/src/format_v2/jni/paimon_jni_reader.cpp | 13 +
be/src/format_v2/orc/orc_reader.cpp | 9 +-
be/src/format_v2/parquet/native_schema_desc.cpp | 17 +-
be/src/format_v2/parquet/native_schema_desc.h | 2 +-
be/src/format_v2/parquet/parquet_profile.cpp | 22 +
be/src/format_v2/parquet/parquet_profile.h | 9 +-
be/src/format_v2/parquet/parquet_reader.cpp | 178 +--
be/src/format_v2/parquet/parquet_reader.h | 9 -
be/src/format_v2/parquet/parquet_scan.cpp | 409 ++++++-
be/src/format_v2/parquet/parquet_scan.h | 44 +-
be/src/format_v2/parquet/parquet_statistics.cpp | 236 +++-
be/src/format_v2/parquet/parquet_statistics.h | 8 +-
.../parquet/reader/variant_column_reader.cpp | 16 +-
be/src/format_v2/table/iceberg_reader.cpp | 92 --
be/src/format_v2/table/iceberg_reader.h | 4 -
be/src/format_v2/table_reader.cpp | 65 +
be/src/format_v2/table_reader.h | 8 +-
be/src/io/cache/block_file_cache.cpp | 23 +-
be/src/io/cache/block_file_cache_profile.cpp | 28 +
be/src/io/cache/block_file_cache_profile.h | 6 +
be/src/io/cache/cached_remote_file_reader.cpp | 33 +-
be/src/io/cache/file_cache_common.h | 10 +
.../io/cache/remote_scan_cache_write_limiter.cpp | 78 ++
be/src/io/cache/remote_scan_cache_write_limiter.h | 47 +
be/src/io/io_common.h | 20 +
be/src/runtime/query_context.cpp | 23 +
be/src/runtime/query_context.h | 18 +
be/src/service/point_query_executor.cpp | 38 +-
be/src/service/point_query_executor.h | 4 +
.../index/inverted/inverted_index_fs_directory.cpp | 4 +
.../index/inverted/util/term_position_iterator.h | 2 +-
be/src/storage/index/primary_key_index.cpp | 27 +-
be/src/storage/index/primary_key_index.h | 12 +-
.../index/zone_map/zonemap_eval_context.cpp | 5 +
.../storage/index/zone_map/zonemap_eval_context.h | 3 +
be/src/storage/iterator/vgeneric_iterators.cpp | 4 +-
be/src/storage/rowset/beta_rowset.cpp | 20 +-
be/src/storage/rowset/beta_rowset.h | 7 +-
be/src/storage/rowset/beta_rowset_reader.cpp | 6 +
be/src/storage/segment/column_meta_accessor.cpp | 65 +-
be/src/storage/segment/column_meta_accessor.h | 14 +-
be/src/storage/segment/column_reader_cache.cpp | 25 +-
be/src/storage/segment/column_reader_cache.h | 25 +-
be/src/storage/segment/external_col_meta_util.cpp | 25 +-
be/src/storage/segment/external_col_meta_util.h | 6 +-
.../storage/segment/lazy_init_segment_iterator.cpp | 3 +-
be/src/storage/segment/segment.cpp | 147 ++-
be/src/storage/segment/segment.h | 37 +-
be/src/storage/segment/segment_iterator.cpp | 5 +-
be/src/storage/segment/segment_loader.cpp | 12 +-
be/src/storage/segment/segment_loader.h | 6 +-
.../segment/variant/hierarchical_data_iterator.cpp | 12 +-
.../segment/variant/hierarchical_data_iterator.h | 5 +-
.../segment/variant/variant_column_reader.cpp | 107 +-
.../segment/variant/variant_column_reader.h | 31 +-
.../variant/variant_external_meta_reader.cpp | 41 +-
.../segment/variant/variant_external_meta_reader.h | 19 +-
be/src/storage/tablet/base_tablet.cpp | 10 +-
be/src/storage/tablet/base_tablet.h | 3 +-
be/test/ai/ai_adapter_test.cpp | 423 ++++++-
be/test/ai/ai_function_test.cpp | 1236 ++++++++++++++++++--
be/test/ai/embed_test.cpp | 836 ++++++++++++-
.../core/block/column_with_type_and_name_test.cpp | 142 ++-
be/test/core/column/column_const_test.cpp | 11 +-
be/test/core/column/column_nullable_test.cpp | 7 +
be/test/core/column/column_variant_test.cpp | 38 +-
.../data_type_serde/data_type_serde_arrow_test.cpp | 1 +
.../data_type_serde_decimal_test.cpp | 4 +-
.../data_type_variant_v2_serde_output_test.cpp | 76 ++
be/test/core/field_test.cpp | 11 +
be/test/exec/hash_map/hash_table_method_test.cpp | 149 ++-
.../operator/materialization_shared_state_test.cpp | 41 +-
be/test/exec/operator/operator_projection_test.cpp | 81 ++
be/test/exec/scan/access_path_parser_test.cpp | 44 +-
be/test/exec/scan/file_scanner_v2_test.cpp | 83 ++
be/test/exec/scan/olap_scanner_test.cpp | 107 ++
be/test/exec/scan/scanner_late_arrival_rf_test.cpp | 45 +
be/test/exec/scan/vgeneric_iterators_test.cpp | 79 ++
.../agg_datasketches_hll_union_agg_test.cpp | 2 +-
be/test/exprs/expr_zonemap_filter_test.cpp | 404 ++++++-
.../function/cast/cast_variant_v2_from_test.cpp | 24 +
.../exprs/function/function_arithmetic_test.cpp | 8 +
be/test/exprs/hybrid_set_test.cpp | 29 +
be/test/format/parquet/parquet_expr_test.cpp | 2 +-
.../iceberg_delete_file_reader_helper_test.cpp | 43 +-
be/test/format_v2/column_mapper_test.cpp | 222 +++-
be/test/format_v2/jni/jni_table_reader_test.cpp | 6 +-
be/test/format_v2/jni/paimon_jni_reader_test.cpp | 29 +-
be/test/format_v2/orc/orc_reader_test.cpp | 30 +
be/test/format_v2/parquet/parquet_reader_test.cpp | 495 +++++++-
be/test/format_v2/parquet/parquet_scan_test.cpp | 291 ++++-
be/test/format_v2/parquet/parquet_schema_test.cpp | 36 +-
.../format_v2/parquet/parquet_statistics_test.cpp | 834 ++++++++++++-
.../parquet/variant_column_reader_test.cpp | 117 ++
be/test/format_v2/table/iceberg_reader_test.cpp | 39 -
.../format_v2/table/remote_doris_reader_test.cpp | 2 +-
be/test/format_v2/table_reader_test.cpp | 383 +++++-
.../block_file_cache_profile_reporter_test.cpp | 5 +
be/test/io/cache/block_file_cache_test.cpp | 516 ++++++++
.../workload_group/workload_group_manager_test.cpp | 2 +
be/test/storage/index/primary_key_index_test.cpp | 111 ++
.../storage/segment/column_reader_cache_test.cpp | 9 +-
.../segment/external_col_meta_util_test.cpp | 163 +++
.../segment/inverted_index_fs_directory_test.cpp | 58 +-
be/test/storage/segment/mock/mock_segment.h | 3 +-
.../storage/segment/segment_footer_cache_test.cpp | 104 ++
.../index_storage_variant_io_context_test.cpp | 113 ++
.../variant/variant_column_writer_reader_test.cpp | 14 +-
be/test/vec/spill/spill_file_test.cpp | 406 ++++++-
build.sh | 82 +-
docker/compilation/Dockerfile.gcc10 | 3 +-
docker/compilation/Dockerfile.gcc7 | 119 --
docker/compilation/arm/Dockerfile | 12 +
.../create_preinstalled_scripts/paimon/run13.sql | 38 +-
.../create_preinstalled_scripts/paimon/run14.sql | 35 +
.../iceberg/scripts/lance_rest_server.py | 27 +
.../java-udf/src/main/resources/package.xml | 1 +
.../apache/doris/paimon/PaimonArrowConverter.java | 73 +-
.../org/apache/doris/paimon/PaimonColumnValue.java | 28 +-
.../org/apache/doris/paimon/PaimonJniScanner.java | 99 +-
.../doris/paimon/PaimonReadTypeProjection.java | 126 ++
.../doris/paimon/PaimonVariantProjection.java | 228 ++++
.../doris/paimon/PaimonArrowConverterTest.java | 147 ++-
.../apache/doris/paimon/PaimonJniScannerTest.java | 36 +
.../doris/paimon/PaimonReadTypeProjectionTest.java | 93 ++
.../doris/paimon/PaimonVariantProjectionTest.java | 147 +++
fe/fe-core/pom.xml | 3 -
.../java/org/apache/doris/catalog/Resource.java | 8 +
.../datasource/lance/LanceExternalCatalog.java | 340 ++----
.../datasource/lance/LanceMetadataLoader.java | 66 +-
.../doris/datasource/lance/LanceNamespaceName.java | 81 +-
.../doris/datasource/lance/LanceReadOptions.java | 43 +
.../datasource/lance/LanceSnapshotResolver.java | 20 +
.../doris/datasource/lance/LanceVectorQuery.java | 75 +-
.../datasource/paimon/PaimonReaderOptions.java | 25 +-
.../doris/datasource/paimon/PaimonScanParams.java | 2 +-
.../doris/datasource/paimon/PaimonTransaction.java | 13 +-
.../apache/doris/datasource/paimon/PaimonUtil.java | 18 +
.../paimon/PaimonVariantWriteAnalyzer.java | 155 +++
.../datasource/paimon/PaimonWriteBinding.java | 22 +
.../doris/datasource/paimon/PaimonWriteTarget.java | 6 +
.../datasource/paimon/source/PaimonScanNode.java | 14 +
.../property/constants/AIProperties.java | 2 +-
.../metastore/AbstractLanceProperties.java | 104 ++
.../LanceFileSystemMetastoreProperties.java | 104 ++
.../property/metastore/LancePropertiesFactory.java | 45 +
.../metastore/LanceRestMetastoreProperties.java | 229 ++++
.../property/metastore/MetastoreProperties.java | 2 +-
.../org/apache/doris/nereids/StatementContext.java | 17 +-
.../doris/nereids/rules/analysis/BindSink.java | 20 +-
.../nereids/rules/expression/check/CheckCast.java | 5 +-
.../rules/rewrite/AccessPathPlanCollector.java | 21 +-
.../rewrite/DecomposeRepeatWithPreAggregation.java | 13 +-
.../nereids/rules/rewrite/NestedColumnPruning.java | 22 +-
.../trees/expressions/functions/agg/AIAgg.java | 1 +
.../trees/expressions/functions/ai/AIFunction.java | 1 +
.../trees/expressions/functions/ai/Embed.java | 48 +-
.../trees/expressions/functions/scalar/Array.java | 11 +-
.../expressions/functions/scalar/CreateMap.java | 11 +-
.../functions/scalar/CreateNamedStruct.java | 10 +-
.../expressions/functions/scalar/CreateStruct.java | 7 +-
.../trees/plans/commands/ExecuteCommand.java | 9 +-
.../trees/plans/commands/insert/InsertUtils.java | 19 +-
.../trees/plans/logical/LogicalFileScan.java | 3 +-
.../apache/doris/nereids/types/VariantType.java | 49 +-
.../doris/nereids/util/TypeCoercionUtils.java | 43 +-
.../main/java/org/apache/doris/qe/Coordinator.java | 26 +-
.../java/org/apache/doris/qe/SessionVariable.java | 21 +
.../apache/doris/qe/ShortCircuitQueryContext.java | 24 +
.../doris/qe/runtime/ThriftPlansBuilder.java | 30 +-
.../doris/statistics/util/StatisticsUtil.java | 7 +-
.../doris/tablefunction/S3TableValuedFunction.java | 2 +-
.../VectorSearchTableValuedFunction.java | 14 +-
.../lance/LanceFilesystemCatalogTest.java | 52 +-
.../datasource/lance/LanceRestCatalogTest.java | 51 +
.../datasource/lance/LanceVectorQueryTest.java | 25 +-
.../datasource/paimon/PaimonReaderOptionsTest.java | 23 +
.../paimon/PaimonVariantWriteAnalyzerTest.java | 201 ++++
.../datasource/paimon/PaimonWriteBindingTest.java | 39 +
.../datasource/paimon/PaimonWriteTargetTest.java | 24 +-
.../paimon/source/PaimonScanNodeTest.java | 47 +
.../property/metastore/LancePropertiesTest.java | 63 +
.../rules/analysis/FunctionRegistryTest.java | 47 +
.../rules/expression/check/CheckCastTest.java | 30 +
.../DecomposeRepeatWithPreAggregationTest.java | 2 +-
.../rules/rewrite/PruneNestedColumnTest.java | 88 +-
.../rules/rewrite/VariantPruningLogicTest.java | 13 +-
.../trees/plans/logical/LogicalFileScanTest.java | 17 +
.../doris/nereids/util/TypeCoercionUtilsTest.java | 5 +
.../org/apache/doris/qe/SessionVariablesTest.java | 14 +
.../doris/qe/ShortCircuitQueryContextTest.java | 58 +
fe/pom.xml | 9 +
gensrc/thrift/PaloInternalService.thrift | 4 +
.../iceberg/test_iceberg_variant_read.out | 4 +
.../iceberg/write/test_iceberg_write_stats2.out | 2 +-
.../paimon/test_paimon_catalog_variant.out | 37 +-
.../test_paimon_jni_complex_column_pruning.out | 16 +
.../paimon/test_paimon_jni_reader_guardrails.out | 6 -
.../paimon_write/test_paimon_write_variant.out | 26 +
.../paimon_write/test_paimon_write_variant_dml.out | 28 +
.../test_paimon_write_variant_errors.out | 5 +
.../test_paimon_write_variant_nested.out | 16 +
.../test_paimon_write_variant_shredding.out | 22 +
.../test_paimon_write_variant_table_modes.out | 15 +
.../suites/ai_p0/test_ai_functions.groovy | 5 +-
...e_cache_query_limit_segment_meta_profile.groovy | 475 ++++++++
...emote_scan_no_write_file_cache_threshold.groovy | 648 ++++++++++
.../test_inverted_index_sc_version1_race.groovy | 93 ++
.../compaction/test_be_compaction_tasks.groovy | 93 +-
.../iceberg/iceberg_schema_change_ddl.groovy | 2 +-
.../iceberg/test_iceberg_variant_read.groovy | 33 +-
.../paimon/test_paimon_catalog_variant.groovy | 176 ++-
.../test_paimon_jni_complex_column_pruning.groovy | 106 ++
.../test_paimon_jni_reader_guardrails.groovy | 7 +-
.../test_remote_doris_variant_select.groovy | 3 +-
.../cdc/test_streaming_mysql_job_metrics.groovy | 4 +-
.../test_paimon_write_schema_change.groovy | 2 +-
.../test_paimon_write_transaction.groovy | 5 +-
.../paimon_write/test_paimon_write_variant.groovy | 156 +++
.../test_paimon_write_variant_dml.groovy | 185 +++
.../test_paimon_write_variant_errors.groovy | 133 +++
.../test_paimon_write_variant_nested.groovy | 195 +++
.../test_paimon_write_variant_shredding.groovy | 303 +++++
.../test_paimon_write_variant_table_modes.groovy | 165 +++
.../spill_p0/test_spill_directory_cleanup.groovy | 113 ++
.../test_variant_equality_contexts.groovy | 2 +-
run-be-ut.sh | 32 +-
thirdparty/arrow-paimon-vars.sh | 408 +++++++
thirdparty/build-thirdparty.sh | 16 +-
thirdparty/download-thirdparty.sh | 64 +-
thirdparty/paimon-cpp-cache.cmake | 3 +-
...rrow-24.0.0-force-write-int96-timestamps.patch} | 42 +-
...0.0-lzo.patch => apache-arrow-24.0.0-lzo.patch} | 72 +-
...imon.patch => apache-arrow-24.0.0-paimon.patch} | 116 +-
.../paimon-cpp-arrow-24-compatibility.patch | 64 +
.../patches/paimon-cpp-arrow-24-compute.patch | 44 +
thirdparty/test/arrow-paimon-lifecycle-test.sh | 492 ++++++++
thirdparty/vars.sh | 31 +-
319 files changed, 19149 insertions(+), 2333 deletions(-)
create mode 100644 be/src/exec/scan/file_scan_io_context.h
create mode 100644 be/src/io/cache/remote_scan_cache_write_limiter.cpp
create mode 100644 be/src/io/cache/remote_scan_cache_write_limiter.h
create mode 100644 be/test/exec/operator/operator_projection_test.cpp
create mode 100644 be/test/exec/scan/olap_scanner_test.cpp
create mode 100644
be/test/storage/variant/index_storage_variant_io_context_test.cpp
delete mode 100644 docker/compilation/Dockerfile.gcc7
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run14.sql
create mode 100644
fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonReadTypeProjection.java
create mode 100644
fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonVariantProjection.java
create mode 100644
fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonReadTypeProjectionTest.java
create mode 100644
fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonVariantProjectionTest.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceReadOptions.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonVariantWriteAnalyzer.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractLanceProperties.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/LanceFileSystemMetastoreProperties.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/LancePropertiesFactory.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/LanceRestMetastoreProperties.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonVariantWriteAnalyzerTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/LancePropertiesTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/qe/ShortCircuitQueryContextTest.java
create mode 100644
regression-test/data/external_table_p0/paimon/test_paimon_jni_complex_column_pruning.out
create mode 100644
regression-test/data/paimon_write/test_paimon_write_variant.out
create mode 100644
regression-test/data/paimon_write/test_paimon_write_variant_dml.out
create mode 100644
regression-test/data/paimon_write/test_paimon_write_variant_errors.out
create mode 100644
regression-test/data/paimon_write/test_paimon_write_variant_nested.out
create mode 100644
regression-test/data/paimon_write/test_paimon_write_variant_shredding.out
create mode 100644
regression-test/data/paimon_write/test_paimon_write_variant_table_modes.out
create mode 100644
regression-test/suites/cloud_p0/cache/remote_scan_no_write_file_cache/test_file_cache_query_limit_segment_meta_profile.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/remote_scan_no_write_file_cache/test_remote_scan_no_write_file_cache_threshold.groovy
create mode 100644
regression-test/suites/cloud_p1/schema_change/compaction_optimization/test_inverted_index_sc_version1_race.groovy
create mode 100644
regression-test/suites/external_table_p0/paimon/test_paimon_jni_complex_column_pruning.groovy
create mode 100644
regression-test/suites/paimon_write/test_paimon_write_variant.groovy
create mode 100644
regression-test/suites/paimon_write/test_paimon_write_variant_dml.groovy
create mode 100644
regression-test/suites/paimon_write/test_paimon_write_variant_errors.groovy
create mode 100644
regression-test/suites/paimon_write/test_paimon_write_variant_nested.groovy
create mode 100644
regression-test/suites/paimon_write/test_paimon_write_variant_shredding.groovy
create mode 100644
regression-test/suites/paimon_write/test_paimon_write_variant_table_modes.groovy
create mode 100644
regression-test/suites/spill_p0/test_spill_directory_cleanup.groovy
create mode 100644 thirdparty/arrow-paimon-vars.sh
rename
thirdparty/patches/{apache-arrow-17.0.0-force-write-int96-timestamps.patch =>
apache-arrow-24.0.0-force-write-int96-timestamps.patch} (74%)
rename thirdparty/patches/{apache-arrow-17.0.0-lzo.patch =>
apache-arrow-24.0.0-lzo.patch} (55%)
rename thirdparty/patches/{apache-arrow-17.0.0-paimon.patch =>
apache-arrow-24.0.0-paimon.patch} (68%)
create mode 100644 thirdparty/patches/paimon-cpp-arrow-24-compatibility.patch
create mode 100644 thirdparty/patches/paimon-cpp-arrow-24-compute.patch
create mode 100755 thirdparty/test/arrow-paimon-lifecycle-test.sh
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
