This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a change to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
from c35767f565c [improvement](be) Refactor thread-pool scan scheduling
(branch-4.2) (#68571)
new 8792a8bab7c branch-4.1: [fix](thirdparty) Upgrade libunwind to 1.8.3
to fix aarch64 BE startup crash on 64 KiB-page kernels #67877 (#68339)
new 98739b59da5 [opt](lance) skip full-snapshot row-id prefilters and add
profile counters (#68375)
new 750a9a027cf [fix](regression) Remove bucketed percentile merge case
from branch-4.1 (#68509)
new cb629f58bf3 branch-4.1: [fix](statistics) Fix NPE in SHOW ANALYZE TASK
STATUS for missing job id #68469 (#68493)
new eccb49267a5 branch-4.1: [fix](regression) Fix tablet state change
publish test #68434 (#68477)
new b0dcaaa5549 branch-4.1: [fix](be) Enforce the sort cursor block
invariant in release builds #68495 (#68519)
new 1b3e2a585db branch-4.1: [fix](be) Fix constant nullable JSON path
evaluation #68427 (#68525)
new 7f117e69f79 branch-4.1: [fix](cloud) Drain bvar timer callbacks before
destruction (#68522) (#68523)
new 09ede400ef2 [fix](file-format) Align Parquet and ORC timestamp
semantics (#68441)
new 90ec8b38294 branch-4.1: [fix](function) correct misleading aliases for
FunctionNextDay and FunctionPreviousDay #68528 (#68542)
new aeab32316ed branch-4.1: [improvement](build) Avoid unnecessary
submodule initialization (#68533) (#68534)
new b8d6c254b6a branch-4.1: [fix](s3) Keep the response stream usable when
an error body overflows the read buffer (#68544)
new 4965856ef84 [fix](external) Preserve external binary and timestamp
mappings (#68532)
new 0673e0daa16 [fix](be) Backport Boolean top-N support to branch-4.1
(#68409)
new 3e40be750bb [feat](inverted-index) Backport IK custom analyzers to
branch-4.1 (#68550)
new 6f2c64f5ae7 branch-4.1: [fix](fe) Bump vulnerable FE dependency
versions (#68557)
new 479fbc7823d branch-4.1: [test](cloud) Make SyncExecutor delay test
deterministic (#68145) (#68586)
new 3905e523da1 branch-4.1: [fix](cloud) Preserve clone source mappings
for deleted instances #68343 (#68570)
new 8b3cccfef42 [fix](iceberg) Bind historical scan specs to the selected
schema (#68568)
The 19 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/build-thirdparty.yml | 5 +
AGENTS.md | 5 +
.../data_type_serde/data_type_datetimev2_serde.cpp | 50 +-
.../data_type_serde/data_type_datetimev2_serde.h | 5 +
.../data_type_serde/data_type_string_serde.cpp | 7 +-
.../data_type_timestamptz_serde.cpp | 47 +-
.../data_type_serde/data_type_timestamptz_serde.h | 5 +
.../data_type_serde/data_type_varbinary_serde.cpp | 20 +
.../data_type_serde/data_type_varbinary_serde.h | 5 +
be/src/core/data_type_serde/orc_serde_utils.cpp | 47 +
be/src/core/data_type_serde/orc_serde_utils.h | 14 +
be/src/exec/operator/file_scan_operator.cpp | 24 +-
.../sink/writer/iceberg/partition_transformers.cpp | 75 +-
.../sink/writer/iceberg/partition_transformers.h | 219 +-
.../sink/writer/iceberg/viceberg_table_writer.cpp | 125 +-
.../sink/writer/iceberg/viceberg_table_writer.h | 2 +
be/src/exec/sink/writer/vhive_partition_writer.cpp | 12 +-
be/src/exec/sink/writer/vhive_partition_writer.h | 3 +
be/src/exec/sort/sort_cursor.h | 2 +-
.../aggregate/aggregate_function_topn_array.cpp | 9 +-
.../aggregate/aggregate_function_topn_weighted.cpp | 9 +-
.../function_date_or_datetime_computation.h | 4 +-
be/src/exprs/function/function_jsonb.cpp | 10 +-
be/src/exprs/function/function_search.cpp | 25 +
be/src/exprs/function/function_search.h | 10 +
be/src/exprs/function/match.cpp | 24 +-
.../function/variant_inverted_index_search.cpp | 37 +
.../exprs/function/variant_inverted_index_search.h | 5 +
be/src/exprs/vmatch_predicate.cpp | 53 +-
be/src/exprs/vmatch_predicate.h | 4 +
.../parquet/parquet_arrow_block_convertor.cpp | 5 +-
.../format/parquet/parquet_arrow_block_convertor.h | 7 +-
be/src/format/parquet/schema_desc.cpp | 13 +-
.../format/table/hive/hive_arrow_block_convertor.h | 2 +-
be/src/format/transformer/vhive_parquet_writer.h | 5 +-
.../format/transformer/viceberg_parquet_writer.cpp | 2 +-
.../format/transformer/viceberg_parquet_writer.h | 2 +-
be/src/format/transformer/vorc_transformer.cpp | 25 +-
be/src/format/transformer/vparquet_writer.cpp | 19 +-
be/src/format/transformer/vparquet_writer.h | 5 +-
be/src/format_v2/column_data.h | 26 +-
be/src/format_v2/column_mapper.cpp | 47 +-
be/src/format_v2/column_mapper.h | 1 +
be/src/format_v2/column_mapper_nested.cpp | 3 +
be/src/format_v2/orc/orc_reader.cpp | 75 +-
be/src/format_v2/orc/orc_search_argument.cpp | 196 +-
be/src/format_v2/parquet/native_schema_desc.cpp | 13 +-
be/src/format_v2/parquet/parquet_column_schema.h | 2 +
be/src/format_v2/parquet/parquet_reader.cpp | 113 +-
be/src/format_v2/parquet/parquet_reader.h | 6 +-
be/src/format_v2/parquet/parquet_scan.cpp | 18 +-
be/src/format_v2/parquet/parquet_scan.h | 4 +
be/src/format_v2/parquet/parquet_statistics.cpp | 13 +-
.../parquet/reader/native/column_reader.cpp | 30 +-
.../parquet/reader/native/column_reader.h | 21 +-
.../parquet/reader/native_column_reader.cpp | 76 +-
.../parquet/reader/native_column_reader.h | 17 +-
be/src/format_v2/table/lance_reader.cpp | 29 +-
be/src/format_v2/table/paimon_reader.cpp | 58 +
be/src/format_v2/table/schema_history_util.cpp | 4 +
be/src/format_v2/table_reader.cpp | 51 +-
be/src/format_v2/table_reader.h | 3 +
be/src/io/fs/err_utils.cpp | 9 +-
be/src/io/fs/s3_common.h | 145 +-
be/src/io/fs/s3_file_reader.cpp | 10 +-
be/src/io/fs/s3_obj_storage_client.cpp | 14 +-
be/src/runtime/index_policy/index_policy_mgr.cpp | 286 ++-
be/src/runtime/index_policy/index_policy_mgr.h | 29 +-
.../index/inverted/analysis_factory_mgr.cpp | 7 +-
.../storage/index/inverted/analyzer/analyzer.cpp | 64 +-
be/src/storage/index/inverted/analyzer/analyzer.h | 6 +-
.../analyzer_provider.h} | 24 +-
.../index/inverted/analyzer/custom_analyzer.cpp | 28 +-
.../index/inverted/analyzer/custom_analyzer.h | 19 +-
.../inverted/analyzer/custom_analyzer_config.cpp | 6 +-
.../inverted/analyzer/custom_analyzer_config.h | 7 +-
.../index/inverted/analyzer/ik/IKTokenizer.cpp | 120 +-
.../index/inverted/analyzer/ik/IKTokenizer.h | 23 +-
.../inverted/analyzer/ik/core/AnalyzeContext.cpp | 4 +-
.../index/inverted/char_filter/char_filter.h | 20 +-
.../char_filter/icu_normalizer_char_filter.cpp | 61 +-
.../char_filter/icu_normalizer_char_filter.h | 9 +
.../index/inverted/inverted_index_iterator.cpp | 244 +--
.../index/inverted/inverted_index_iterator.h | 30 +-
.../index/inverted/inverted_index_parser.cpp | 155 +-
.../storage/index/inverted/inverted_index_parser.h | 75 +-
.../index/inverted/inverted_index_query_type.h | 7 +
.../index/inverted/inverted_index_reader.cpp | 7 +-
.../index/inverted/inverted_index_selector.cpp | 149 ++
.../index/inverted/inverted_index_selector.h | 58 +
.../index/inverted/inverted_index_writer.cpp | 40 +-
.../storage/index/inverted/inverted_index_writer.h | 9 +
.../inverted/token_filter/ascii_folding_filter.cpp | 65 +-
.../inverted/token_filter/ascii_folding_filter.h | 16 +-
.../token_filter/icu_normalizer_filter.cpp | 40 +-
.../inverted/token_filter/icu_normalizer_filter.h | 11 +-
.../inverted/token_filter/lower_case_filter.h | 71 +-
.../index/inverted/token_filter/pinyin_filter.cpp | 239 ++-
.../index/inverted/token_filter/pinyin_filter.h | 46 +-
.../token_filter/pinyin_filter_factory.cpp | 5 +-
.../index/inverted/token_filter/token_filter.h | 41 +-
.../token_filter/word_delimiter_filter.cpp | 173 +-
.../inverted/token_filter/word_delimiter_filter.h | 60 +-
be/src/storage/index/inverted/token_stream.h | 49 +
.../inverted/tokenizer/basic/basic_tokenizer.cpp | 7 +-
.../inverted/tokenizer/char/char_tokenizer.cpp | 5 +-
.../tokenizer/empty/empty_tokenizer_factory.h | 5 +-
.../index/inverted/tokenizer/icu/icu_tokenizer.cpp | 62 +
.../index/inverted/tokenizer/icu/icu_tokenizer.h | 14 +-
.../ik/ik_tokenizer_factory.h} | 32 +-
.../inverted/tokenizer/keyword/keyword_tokenizer.h | 9 +-
.../inverted/tokenizer/ngram/ngram_tokenizer.cpp | 11 +
.../inverted/tokenizer/ngram/ngram_tokenizer.h | 2 +-
.../inverted/tokenizer/pinyin/pinyin_tokenizer.cpp | 115 +-
.../inverted/tokenizer/pinyin/pinyin_tokenizer.h | 16 +
.../inverted/tokenizer/pinyin/smart_get_word.cpp | 56 +-
.../tokenizer/standard/standard_tokenizer.h | 11 +-
.../tokenizer/standard/standard_tokenizer_impl.h | 9 +-
.../storage/index/inverted/tokenizer/tokenizer.h | 159 +-
be/src/storage/predicate_collector.cpp | 142 +-
be/src/storage/predicate_collector.h | 9 +
.../data_type_serde/data_type_serde_arrow_test.cpp | 55 +-
be/test/exec/rowid_fetcher_test.cpp | 16 +
be/test/exec/scan/file_scanner_v2_test.cpp | 45 +
.../writer/iceberg/partition_transformers_test.cpp | 185 +-
.../writer/iceberg/viceberg_table_writer_test.cpp | 243 +++
be/test/exprs/aggregate/agg_topn_boolean_test.cpp | 117 +
be/test/exprs/function/function_jsonb_test.cpp | 111 +-
.../exprs/function/function_search_nested_test.cpp | 34 +
be/test/exprs/function/function_search_test.cpp | 52 +-
.../format/arrow/arrow_block_convertor_test.cpp | 11 +-
...eberg_position_delete_sys_table_reader_test.cpp | 36 +
.../format/transformer/vorc_transformer_test.cpp | 76 +
.../format/transformer/vparquet_writer_test.cpp | 88 +-
be/test/format_v2/column_mapper_test.cpp | 117 +-
be/test/format_v2/orc/orc_reader_test.cpp | 616 +++++-
be/test/format_v2/parquet/parquet_reader_test.cpp | 189 +-
be/test/format_v2/parquet/parquet_schema_test.cpp | 44 +
be/test/format_v2/table/lance_reader_test.cpp | 16 +
be/test/format_v2/table/paimon_reader_test.cpp | 294 +++
be/test/format_v2/table_reader_test.cpp | 222 +-
be/test/io/fs/s3_response_stream_test.cpp | 189 ++
.../runtime/index_policy/index_policy_mgr_test.cpp | 333 ++-
.../compaction/collection_statistics_test.cpp | 136 ++
.../index/inverted/analysis_factory_mgr_test.cpp | 7 +-
.../index/inverted/analyzer/ik_anayzer_test.cpp | 119 +
.../icu_normalizer_char_filter_factory_test.cpp | 58 +-
.../icu_normalizer_filter_factory_test.cpp | 71 +-
.../inverted/token_filter/pinyin_filter_test.cpp | 1475 +++++++++++++
.../token_filter/word_delimiter_filter_test.cpp | 42 +
.../tokenizer/icu_tokenizer_factory_test.cpp | 105 +-
.../inverted/tokenizer/keyword_analyzer_test.cpp | 33 +-
.../inverted/tokenizer/ngram_tokenizer_test.cpp | 57 +-
.../storage/index/inverted_index_parser_test.cpp | 147 +-
.../segment/inverted_index_iterator_test.cpp | 573 ++++-
.../storage/segment/inverted_index_reader_test.cpp | 65 +
.../storage/segment/inverted_index_writer_test.cpp | 155 ++
cloud/src/common/bvars.h | 188 +-
cloud/src/resource-manager/resource_manager.cpp | 9 -
cloud/test/bvars_test.cpp | 50 +-
cloud/test/clone_chain_reader_test.cpp | 62 +
cloud/test/resource_test.cpp | 10 +-
cloud/test/util_test.cpp | 85 +-
docs/lance-prefilter-profile.md | 58 +
.../apache/doris/hudi/HadoopHudiColumnValue.java | 36 +-
.../doris/hudi/HadoopHudiColumnValueTest.java | 98 +
fe/be-java-extensions/jdbc-scanner/pom.xml | 5 +
.../org/apache/doris/jdbc/BaseJdbcExecutor.java | 14 +-
.../apache/doris/jdbc/ClickHouseJdbcExecutor.java | 39 +-
.../org/apache/doris/jdbc/MySQLJdbcExecutor.java | 33 +-
.../org/apache/doris/jdbc/OracleJdbcExecutor.java | 16 +-
.../apache/doris/jdbc/PostgreSQLJdbcExecutor.java | 34 +-
.../apache/doris/jdbc/SQLServerJdbcExecutor.java | 16 +
.../org/apache/doris/jdbc/TrinoJdbcExecutor.java | 35 +
.../jdbc-scanner/src/test/README.md | 69 +
.../apache/doris/jdbc/JdbcTimestampBindTest.java | 110 +
.../doris/jdbc/JdbcTimestampSemanticsTest.java | 320 +++
.../jdbc/JdbcZonedTimestampIntegrationTest.java | 374 ++++
.../doris/jdbc/MySqlTimestampIntegrationTest.java | 167 ++
.../max-compute-connector/pom.xml | 3 +-
.../doris/maxcompute/MaxComputeColumnValue.java | 4 +-
.../doris/maxcompute/MaxComputeJniWriter.java | 39 +-
.../doris/maxcompute/MaxComputeJniWriterTest.java | 80 +
.../trinoconnector/TrinoConnectorColumnValue.java | 4 +-
.../TrinoTimestampSemanticsTest.java | 61 +
fe/check/checkstyle/suppressions.xml | 2 +
.../apache/doris/alter/SchemaChangeHandler.java | 6 +
.../apache/doris/analysis/AnalyzerSelector.java | 13 +-
.../doris/analysis/InvertedIndexProperties.java | 198 ++
.../apache/doris/analysis/InvertedIndexUtil.java | 131 +-
.../org/apache/doris/analysis/MatchPredicate.java | 3 +-
.../invertedindex/AnalyzerIdentityBuilder.java | 1317 +++++++++++-
.../invertedindex/AnalyzerKeyNormalizer.java | 3 +-
.../main/java/org/apache/doris/catalog/Env.java | 5 +
.../org/apache/doris/catalog/JdbcResource.java | 4 +-
.../java/org/apache/doris/catalog/OlapTable.java | 10 +
.../doris/common/util/FileFormatConstants.java | 1 +
.../apache/doris/common/util/FileFormatUtils.java | 33 +
.../org/apache/doris/datasource/CatalogMgr.java | 50 +-
.../apache/doris/datasource/CatalogProperty.java | 32 +-
.../apache/doris/datasource/ExternalCatalog.java | 10 +-
.../org/apache/doris/datasource/ExternalTable.java | 19 +
.../apache/doris/datasource/FileQueryScanNode.java | 23 +-
.../doris/datasource/hive/HMSExternalCatalog.java | 8 +
.../doris/datasource/hive/HMSExternalTable.java | 6 +-
.../datasource/hive/HiveMetaStoreClientHelper.java | 7 +-
.../apache/doris/datasource/hudi/HudiUtils.java | 14 +-
.../doris/datasource/hudi/source/HudiScanNode.java | 18 +-
.../doris/datasource/iceberg/IcebergUtils.java | 108 +-
.../iceberg/IcebergWriteSchemaContext.java | 20 +-
.../datasource/iceberg/source/IcebergScanNode.java | 50 +-
.../doris/datasource/jdbc/JdbcExternalCatalog.java | 1 -
.../jdbc/client/JdbcClickHouseClient.java | 5 +-
.../doris/datasource/jdbc/client/JdbcClient.java | 2 -
.../datasource/jdbc/client/JdbcClientConfig.java | 21 +-
.../datasource/jdbc/client/JdbcDB2Client.java | 3 +-
.../datasource/jdbc/client/JdbcMySQLClient.java | 7 +-
.../datasource/jdbc/client/JdbcOracleClient.java | 13 +-
.../jdbc/client/JdbcPostgreSQLClient.java | 10 +-
.../jdbc/client/JdbcSQLServerClient.java | 7 +-
.../datasource/jdbc/client/JdbcTrinoClient.java | 4 +-
.../doris/datasource/jdbc/source/JdbcScanNode.java | 85 +-
.../doris/datasource/maxcompute/MCTransaction.java | 3 +-
.../maxcompute/MaxComputeExternalTable.java | 2 +
.../maxcompute/MaxComputeMetadataOps.java | 3 +
.../maxcompute/source/MaxComputeScanNode.java | 4 +
.../paimon/DorisToPaimonTypeVisitor.java | 9 +-
.../apache/doris/datasource/paimon/PaimonUtil.java | 45 +-
.../datasource/paimon/PaimonWriteBinding.java | 32 +-
.../doris/datasource/paimon/PaimonWriteTarget.java | 4 +-
.../datasource/paimon/source/PaimonScanNode.java | 90 +-
.../property/fileformat/FileFormatProperties.java | 6 +-
.../fileformat/ParquetFileFormatProperties.java | 16 +-
.../TrinoConnectorExternalTable.java | 3 +-
.../source/TrinoConnectorPredicateConverter.java | 31 +-
.../org/apache/doris/indexpolicy/IndexPolicy.java | 18 +-
.../apache/doris/indexpolicy/IndexPolicyMgr.java | 355 ++-
.../apache/doris/job/util/StreamingJobUtils.java | 19 +
.../main/java/org/apache/doris/load/ExportJob.java | 8 +
.../glue/translator/ExpressionTranslator.java | 4 +
.../doris/nereids/trees/expressions/Match.java | 6 +-
.../expressions/literal/VarBinaryLiteral.java | 6 +
.../trees/plans/commands/ExportCommand.java | 8 +
.../plans/commands/ShowAnalyzeTaskCommand.java | 3 +
.../trees/plans/commands/info/BaseViewInfo.java | 4 -
.../trees/plans/commands/info/CreateIndexOp.java | 6 +
.../insert/InsertOverwriteTableCommand.java | 8 +-
.../org/apache/doris/nereids/types/DataType.java | 10 +-
.../doris/nereids/util/TypeCoercionUtils.java | 80 +-
.../org/apache/doris/planner/HiveTableSink.java | 8 +
.../org/apache/doris/planner/IcebergTableSink.java | 10 +-
.../CdcStreamTableValuedFunction.java | 2 +
.../ExternalFileTableValuedFunction.java | 32 +-
.../tablefunction/FileTableValuedFunction.java | 5 +
.../doris/tablefunction/TableValuedFunctionIf.java | 4 +
.../org/apache/iceberg/DorisDataTableScan.java | 71 +
.../doris/alter/SchemaChangeHandlerTest.java | 545 ++++-
.../analysis/InvertedIndexPropertiesTest.java | 1010 +++++++++
.../invertedindex/AnalyzerIdentityBuilderTest.java | 2266 +++++++++++++++++++-
.../org/apache/doris/catalog/CreateViewTest.java | 9 +-
.../doris/datasource/CatalogPropertyTest.java | 28 +
.../datasource/CatalogVarbinaryMigrationTest.java | 232 ++
.../datasource/ExternalTimestampMappingTest.java | 117 +
.../ExternalTimestampWireContractTest.java | 83 +
.../doris/datasource/FileQueryScanNodeTest.java | 4 +
.../HMSExternalCatalogParquetTimeZoneTest.java | 108 +
.../datasource/hive/HMSExternalTableTest.java | 26 +
.../datasource/hudi/source/HudiScanNodeTest.java | 73 +
.../iceberg/IcebergExternalMetaCacheTest.java | 4 +-
.../iceberg/IcebergMetadataOpsValidationTest.java | 14 +-
.../iceberg/IcebergSchemaDisplayTest.java | 8 +-
.../iceberg/IcebergSysExternalTableTest.java | 7 +-
.../datasource/iceberg/IcebergTransactionTest.java | 19 +
.../doris/datasource/iceberg/IcebergUtilsTest.java | 235 +-
.../iceberg/IcebergWriteSchemaContextTest.java | 71 +-
.../iceberg/helper/IcebergWriterHelperTest.java | 25 +
.../iceberg/source/IcebergScanNodeTest.java | 197 +-
.../datasource/jdbc/client/JdbcClientTest.java | 91 +
.../datasource/jdbc/source/JdbcScanNodeTest.java | 91 +
.../jdbc/source/JdbcTimestampProjectionTest.java | 163 ++
.../maxcompute/MaxComputeExternalTableTest.java | 6 +
.../doris/datasource/paimon/PaimonUtilTest.java | 102 +-
.../datasource/paimon/PaimonWriteBindingTest.java | 62 +
.../datasource/paimon/PaimonWriteTargetTest.java | 20 +
.../paimon/source/PaimonScanNodeTest.java | 156 ++
.../ParquetFileFormatPropertiesTest.java | 16 +-
.../TrinoConnectorPredicateTest.java | 32 +
.../doris/indexpolicy/PolicyValidatorTests.java | 439 +++-
.../doris/job/util/StreamingJobUtilsTest.java | 53 +
.../java/org/apache/doris/load/ExportJobTest.java | 48 +
.../glue/translator/ExpressionTranslatorTest.java | 37 +
.../trees/plans/commands/ExportCommandTest.java | 44 +
.../insert/IcebergStaticPartitionLiteralTest.java | 62 +
.../nereids/types/VarBinarySqlSupportTest.java | 181 ++
.../apache/doris/planner/HiveTableSinkTest.java | 26 +-
.../apache/doris/planner/IcebergTableSinkTest.java | 47 +
.../CdcStreamTableValuedFunctionTest.java | 38 +-
.../ExternalFileTableValuedFunctionTest.java | 66 +
fe/pom.xml | 26 +-
gensrc/thrift/DataSinks.thrift | 5 +
gensrc/thrift/ExternalTableSchema.thrift | 5 +-
gensrc/thrift/PlanNodes.thrift | 10 +-
.../parquet/test_outfile_parquet_array_type.out | 8 +-
.../parquet/test_outfile_parquet_complex_type.out | 40 +-
.../parquet/test_outfile_parquet_map_type.out | 16 +-
.../parquet/test_doris_int96_round_trip.out | 7 +
.../hive/hive_json_basic_test.out | 12 +-
.../external_table_p0/hive/test_complex_types.out | 2 +-
.../hive/test_hive_basic_type.out | 72 +-
.../hive/test_hive_get_schema_from_table.out | 4 +-
.../data/external_table_p0/hive/test_hive_orc.out | 1240 +++++------
.../hive/test_hive_varbinary_type.out | 108 +-
.../hive/write/test_hive_text_write_insert.out | 820 +++----
.../hive/write/test_hive_write_insert.out | 224 +-
.../hive/write/test_hive_write_partitions.out | 96 +-
.../test_external_catalog_iceberg_partition.out | 82 +-
.../iceberg/test_gen_iceberg_by_api.out | 8 +-
.../iceberg/test_iceberg_export_timestamp_tz.out | 16 +-
.../iceberg/test_iceberg_initial_defaults.out | 32 +-
.../iceberg/test_iceberg_predicate_conversion.out | 8 +-
.../iceberg/test_iceberg_statistics.out | 6 +-
.../iceberg/test_iceberg_sys_table.out | 104 +-
.../iceberg/test_iceberg_transform_partitions.out | 9 +-
.../iceberg/test_iceberg_varbinary.out | 28 +-
.../test_iceberg_write_partition_types_null.out | 2 +-
.../write/test_iceberg_write_timestamp_ntz.out | 7 +-
.../jdbc/test_clickhouse_jdbc_catalog.out | 50 +-
.../external_table_p0/jdbc/test_jdbc_query_tvf.out | 8 +-
.../jdbc/test_mariadb_jdbc_catalog.out | 8 +-
.../jdbc/test_mysql_jdbc_catalog.out | 57 +-
.../jdbc/test_mysql_jdbc_catalog_nereids.out | 5 +-
.../jdbc/test_pg_jdbc_catalog.out | 28 +-
.../type_test/ctas/test_mysql_all_types_ctas.out | 12 +-
.../select/test_clickhouse_all_types_select.out | 13 +-
.../select/test_mysql_all_types_select.out | 24 +-
.../type_test/select/test_pg_all_types_select.out | 14 +-
.../type_test/tvf/test_mysql_all_types_tvf.out | 25 +-
.../lance/test_lance_vector_search_metrics.out | 2 +-
.../paimon/paimon_base_filesystem.out | 16 +-
.../paimon/paimon_timestamp_types.out | 57 +-
.../paimon/test_paimon_catalog.out | 569 +++--
.../paimon/test_paimon_catalog_varbinary.out | 5 +-
.../external_table_p0/paimon/test_paimon_minio.out | 17 +-
.../paimon/test_paimon_statistics.out | 1 -
.../paimon/test_paimon_table_properties.out | 3 +-
.../test_paimon_timestamp_with_time_zone.out | 19 +-
.../paimon/write/test_paimon_write_types.out | 11 +-
.../trino_connector/jdbc/test_trino_mysql.out | 25 +-
.../tvf/orc_tvf/test_hdfs_orc_group4_orc_files.out | 2 +-
.../tvf/orc_tvf/test_hdfs_orc_group6_orc_files.out | 2 +-
.../tvf/test_hdfs_parquet_group0.out | 2 +-
.../tvf/test_hdfs_parquet_group1.out | 10 +-
.../tvf/test_hdfs_parquet_group2.out | 118 +-
.../tvf/test_hdfs_parquet_group4.out | 164 +-
.../tvf/test_hdfs_parquet_group5.out | 144 +-
.../tvf/test_hdfs_parquet_group6.out | 246 +--
.../tvf/test_local_tvf_with_complex_type.out | 50 +-
.../data/external_table_p0/tvf/test_tvf_p0.out | 10 +-
.../data/external_table_p0/upgrade/test.out | 52 +-
.../test_tablet_state_change_in_publish_phase.out | 4 +-
.../analyzer/test_analyzer_identity_semantics.out | 26 +
.../analyzer/test_ik_custom_analyzer.out | 13 +
.../agg_function/topn/topn_boolean.out | 27 +
.../aggregate/percentile_bucketed_agg_merge.out | 23 -
.../test_json_const_nullable_path.out | 6 +
.../test_nested_types_insert_into_with_s3.groovy | 4 +
.../datatype_p0/test_varbinary_sql_support.groovy | 87 +
.../parquet/test_doris_int96_round_trip.groovy | 103 +
.../parquet/test_hive_read_parquet.groovy | 19 +-
.../test_hive_read_parquet_complex_type.groovy | 35 +-
.../hive/ddl/test_hive_ctas.groovy | 3 +-
.../hive/test_complex_types.groovy | 3 +-
.../hive/test_external_catalog_hive.groovy | 3 +-
.../test_external_catalog_hive_partition.groovy | 3 +-
.../hive/test_hive_basic_type.groovy | 6 +-
.../hive/test_hive_compress_type.groovy | 3 +-
.../hive/test_hive_get_schema_from_table.groovy | 3 +-
.../external_table_p0/hive/test_hive_orc.groovy | 32 +-
.../hive/test_hive_page_index.groovy | 3 +-
.../hive/test_hive_schema_evolution.groovy | 3 +-
.../hive/test_hive_statistics_all_type_p0.groovy | 13 +-
.../hive/test_hive_varbinary_type.groovy | 7 +-
.../hive/test_parquet_lazy_mat_profile.groovy | 294 +--
.../write/test_hive_int96_write_timezone.groovy | 79 +
.../hive/write/test_hive_text_write_insert.groovy | 3 +-
.../hive/write/test_hive_write_insert.groovy | 3 +-
.../hive/write/test_hive_write_partitions.groovy | 3 +-
.../action/test_iceberg_execute_actions.groovy | 29 +-
.../test_external_catalog_iceberg_partition.groovy | 4 +-
.../iceberg/test_gen_iceberg_by_api.groovy | 3 +
.../test_iceberg_export_timestamp_tz.groovy | 13 +-
.../test_iceberg_historical_filter_planning.groovy | 116 +
.../iceberg/test_iceberg_initial_defaults.groovy | 7 +-
.../test_iceberg_predicate_conversion.groovy | 3 +
...iceberg_runtime_filter_partition_pruning.groovy | 21 +-
...ntime_filter_partition_pruning_transform.groovy | 33 +-
.../iceberg/test_iceberg_statistics.groovy | 2 +
.../iceberg/test_iceberg_sys_table.groovy | 6 +-
.../iceberg/test_iceberg_timestamp_tz.groovy | 3 +-
.../test_iceberg_transform_partitions.groovy | 16 +-
.../iceberg/test_iceberg_varbinary.groovy | 4 +-
.../iceberg/test_iceberg_variant_read.groovy | 8 +-
.../write/test_iceberg_insert_overwrite.groovy | 1 +
.../test_iceberg_static_partition_overwrite.groovy | 39 +
.../test_iceberg_write_binary_partitions.groovy | 81 +
.../iceberg/write/test_iceberg_write_insert.groovy | 1 +
.../test_iceberg_write_partition_types_null.groovy | 1 +
.../write/test_iceberg_write_timestamp_ntz.groovy | 9 +-
.../test_iceberg_write_timestamp_partitions.groovy | 98 +
.../jdbc/test_clickhouse_jdbc_catalog.groovy | 46 +-
.../jdbc/test_jdbc_query_tvf.groovy | 3 +
.../jdbc/test_mariadb_jdbc_catalog.groovy | 6 +-
.../jdbc/test_mysql_jdbc_catalog.groovy | 20 +-
.../jdbc/test_mysql_jdbc_catalog_nereids.groovy | 5 +
.../jdbc/test_pg_jdbc_catalog.groovy | 3 +
.../ctas/test_mysql_all_types_ctas.groovy | 3 +
.../select/test_clickhouse_all_types_select.groovy | 3 +
.../select/test_mysql_all_types_select.groovy | 9 +
.../select/test_pg_all_types_select.groovy | 48 +
.../type_test/tvf/test_mysql_all_types_tvf.groovy | 3 +
.../lance/test_lance_vector_search_metrics.groovy | 5 +-
.../paimon/paimon_timestamp_types.groovy | 59 +-
.../paimon/test_paimon_catalog.groovy | 4 +-
.../paimon/test_paimon_catalog_varbinary.groovy | 9 +-
.../paimon/test_paimon_catalog_variant.groovy | 11 +-
.../paimon/test_paimon_minio.groovy | 6 +-
.../paimon/test_paimon_table_properties.groovy | 2 +-
.../paimon/write/test_paimon_write_types.groovy | 49 +-
.../trino_connector/jdbc/test_trino_mysql.groovy | 9 +-
.../orc_tvf/test_hdfs_orc_group4_orc_files.groovy | 5 +
.../tvf/test_hdfs_parquet_group0.groovy | 35 +-
.../tvf/test_hdfs_parquet_group1.groovy | 5 +
.../tvf/test_hdfs_parquet_group2.groovy | 9 +-
.../tvf/test_hdfs_parquet_group6.groovy | 5 +
...t_local_tvf_with_complex_type_element_at.groovy | 24 +-
.../suites/external_table_p0/upgrade/test.groovy | 3 +-
...est_tablet_state_change_in_publish_phase.groovy | 3 +
.../test_analyzer_identity_semantics.groovy | 421 ++++
.../test_analyzer_malformed_utf8_write.groovy | 110 +
.../analyzer/test_ik_custom_analyzer.groovy | 381 ++++
.../suites/mtmv_p0/test_iceberg_mtmv.groovy | 5 +-
.../agg_function/topn/topn_boolean.groovy | 78 +
.../show/test_show_analyze_task_status_cmd.groovy | 6 +
.../aggregate/percentile_bucketed_agg_merge.groovy | 93 -
.../test_json_const_nullable_path.groovy | 43 +-
thirdparty/CHANGELOG.md | 11 +
thirdparty/build-thirdparty.sh | 4 +-
thirdparty/download-thirdparty.sh | 15 +-
.../patches/lance-c-0.1.9-prefilter-fts.patch | 148 ++
thirdparty/patches/lance-c-0.1.9-prefilter.patch | 147 ++
...atch => libunwind-1.8.3-doris-phdr-cache.patch} | 8 +-
thirdparty/test/libunwind-page-size-test.sh | 172 ++
thirdparty/vars.sh | 8 +-
453 files changed, 26494 insertions(+), 4697 deletions(-)
create mode 100644 AGENTS.md
copy be/src/storage/index/inverted/{token_filter/token_filter.h =>
analyzer/analyzer_provider.h} (66%)
create mode 100644 be/src/storage/index/inverted/inverted_index_selector.cpp
create mode 100644 be/src/storage/index/inverted/inverted_index_selector.h
copy be/src/storage/index/inverted/{token_filter/icu_normalizer_filter.h =>
tokenizer/ik/ik_tokenizer_factory.h} (51%)
create mode 100644 be/test/exprs/aggregate/agg_topn_boolean_test.cpp
create mode 100644 be/test/io/fs/s3_response_stream_test.cpp
create mode 100644 docs/lance-prefilter-profile.md
create mode 100644
fe/be-java-extensions/hadoop-hudi-scanner/src/test/java/org/apache/doris/hudi/HadoopHudiColumnValueTest.java
create mode 100644 fe/be-java-extensions/jdbc-scanner/src/test/README.md
create mode 100644
fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/JdbcTimestampBindTest.java
create mode 100644
fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/JdbcTimestampSemanticsTest.java
create mode 100644
fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/JdbcZonedTimestampIntegrationTest.java
create mode 100644
fe/be-java-extensions/jdbc-scanner/src/test/java/org/apache/doris/jdbc/MySqlTimestampIntegrationTest.java
create mode 100644
fe/be-java-extensions/trino-connector-scanner/src/test/java/org/apache/doris/trinoconnector/TrinoTimestampSemanticsTest.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexProperties.java
create mode 100644
fe/fe-core/src/main/java/org/apache/iceberg/DorisDataTableScan.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/analysis/InvertedIndexPropertiesTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogVarbinaryMigrationTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalTimestampMappingTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalTimestampWireContractTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HMSExternalCatalogParquetTimeZoneTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/source/JdbcTimestampProjectionTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/load/ExportJobTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExportCommandTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergStaticPartitionLiteralTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/nereids/types/VarBinarySqlSupportTest.java
create mode 100644
regression-test/data/external_table_p0/export/hive_read/parquet/test_doris_int96_round_trip.out
create mode 100644
regression-test/data/inverted_index_p0/analyzer/test_analyzer_identity_semantics.out
create mode 100644
regression-test/data/inverted_index_p0/analyzer/test_ik_custom_analyzer.out
create mode 100644
regression-test/data/nereids_function_p0/agg_function/topn/topn_boolean.out
delete mode 100644
regression-test/data/query_p0/aggregate/percentile_bucketed_agg_merge.out
create mode 100644
regression-test/data/query_p0/sql_functions/json_functions/test_json_const_nullable_path.out
create mode 100644
regression-test/suites/datatype_p0/test_varbinary_sql_support.groovy
create mode 100644
regression-test/suites/external_table_p0/export/hive_read/parquet/test_doris_int96_round_trip.groovy
create mode 100644
regression-test/suites/external_table_p0/hive/write/test_hive_int96_write_timezone.groovy
create mode 100644
regression-test/suites/external_table_p0/iceberg/test_iceberg_historical_filter_planning.groovy
create mode 100644
regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_binary_partitions.groovy
create mode 100644
regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_timestamp_partitions.groovy
create mode 100644
regression-test/suites/inverted_index_p0/analyzer/test_analyzer_identity_semantics.groovy
create mode 100644
regression-test/suites/inverted_index_p0/analyzer/test_analyzer_malformed_utf8_write.groovy
create mode 100644
regression-test/suites/inverted_index_p0/analyzer/test_ik_custom_analyzer.groovy
create mode 100644
regression-test/suites/nereids_function_p0/agg_function/topn/topn_boolean.groovy
delete mode 100644
regression-test/suites/query_p0/aggregate/percentile_bucketed_agg_merge.groovy
copy be/src/storage/index/inverted/token_filter/token_filter.h =>
regression-test/suites/query_p0/sql_functions/json_functions/test_json_const_nullable_path.groovy
(51%)
create mode 100644 thirdparty/patches/lance-c-0.1.9-prefilter-fts.patch
create mode 100644 thirdparty/patches/lance-c-0.1.9-prefilter.patch
rename thirdparty/patches/{libunwind-1.6.2-doris-phdr-cache.patch =>
libunwind-1.8.3-doris-phdr-cache.patch} (83%)
create mode 100755 thirdparty/test/libunwind-page-size-test.sh
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
