This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a change to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
from 5f4cc0badd7 branch:4.2: [fix](point query) Keep point-query scan when
partition pruning is empty #67161 (#68293)
new cb3815984b5 branch-4.1: [feature](hive) Support static partition
overwrite for hive tables #65991 (#68181)
new 8c7c9b9bf15 branch-4.1: [enhance](auth) introduction of configuration
property to prohibit login with empty LDAP password #61440 (#68098)
new b16d1151505 [feature](lance) Support prepared vector search on
branch-4.1 (#68162)
new 5d31e86dc56 branch-4.1: [fix](regression) Enable expression pushdown
for variant norms test (#68230)
new d2ffdeb73b0 branch-4.1: [fix](be) Avoid ASAN double-free during PHDR
unwinding #68217 (#68229)
new 79ee1b3c2b4 branch-4.1: [fix](regression) Align remote Doris VARIANT
checks with current semantics #68165 (#68228)
new e5ad03dc7dc [fix](be) exclude wal from variant format validation
(#68233) (#68242)
new 8c97ea386f5 branch-4.1: [fix](rbo) Guard partial TopN pushdown through
joins #67796 (#68235)
new 7bded76141a branch-4.1: [test](regression) Stabilize search score
cache test #65894 (#68252)
new 5cef6f05dea branch-4.1: [fix](varaint) Preserve variant sub-path order
#67803 (#68284)
new 80309dd1319 branch-4.1: [fix](fe) Prevent stale external name
publication after refresh #65126 (#67417)
new 60951d93e6c branch-4.1: [improvement](hive) Batch Hive metastore
partition access (#68197)
new fdf8b59464d branch-4.1: [Enhance](resource) Restrict AI resource
creation by user identity (#68147) (#68265)
new f7cfe170aa8 [opt](lance) Cache table access resolution in FE (#68305)
new 420a9c4be46 branch-4.1: [improvement](fe) Add connector property
validators (#68163)
new 17899d6080b [feat](iceberg) HDFS lazy open + iceberg delete file
file_size propagation (#66773) (#67842)
The 16 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/common/phdr_cache.cpp | 5 +
be/src/common/phdr_cache.h | 2 +
be/src/exec/scan/file_scanner.cpp | 10 +-
.../table/iceberg_delete_file_reader_helper.cpp | 10 +-
.../table/iceberg_delete_file_reader_helper.h | 2 +-
be/src/format/table/iceberg_reader.cpp | 24 +-
be/src/format/table/iceberg_reader.h | 4 +
be/src/format_v2/orc/orc_reader.cpp | 9 +-
be/src/format_v2/table/iceberg_reader.cpp | 5 +-
be/src/format_v2/table_reader.cpp | 9 +-
be/src/io/fs/file_handle_cache.cpp | 68 +-
be/src/io/fs/file_handle_cache.h | 10 +-
be/src/io/fs/hdfs_file_reader.cpp | 47 +-
be/src/io/fs/hdfs_file_system.cpp | 9 +-
be/src/io/fs/local_file_reader.cpp | 2 +-
be/src/io/hdfs_util.cpp | 1 +
be/src/io/hdfs_util.h | 1 +
be/src/storage/index/index_file_reader.cpp | 10 +
.../index/inverted/inverted_index_fs_directory.cpp | 6 +-
be/test/common/phdr_cache_test.cpp | 31 +
be/test/exec/scan/vfile_scanner_exception_test.cpp | 131 +-
.../iceberg_delete_file_reader_helper_test.cpp | 16 +-
.../format/table/iceberg/iceberg_reader_test.cpp | 65 +
.../format_v2/orc/orc_file_input_stream_test.cpp | 33 +
be/test/format_v2/orc/orc_reader_test.cpp | 34 +
be/test/format_v2/table/iceberg_reader_test.cpp | 120 +-
be/test/format_v2/table_reader_test.cpp | 11 +-
be/test/io/fs/file_handle_cache_test.cpp | 391 +++++
be/test/io/fs/hdfs_file_system_test.cpp | 70 +
.../segment/inverted_index_file_reader_test.cpp | 27 +
conf/ldap.conf | 13 +
.../main/java/org/apache/doris/common/Config.java | 4 +
.../java/org/apache/doris/common/LdapConfig.java | 10 +
.../org/apache/doris/catalog/RefreshManager.java | 14 +
.../java/org/apache/doris/common/CacheFactory.java | 6 +
.../org/apache/doris/datasource/CatalogMgr.java | 31 +-
.../apache/doris/datasource/ExternalCatalog.java | 125 +-
.../apache/doris/datasource/ExternalDatabase.java | 117 +-
.../doris/datasource/ExternalMetaCacheMgr.java | 12 +
.../doris/datasource/hive/HMSCachedClient.java | 9 +
.../doris/datasource/hive/HMSClientException.java | 11 +
.../doris/datasource/hive/HMSExternalCatalog.java | 14 +-
.../doris/datasource/hive/HMSExternalDatabase.java | 12 +-
.../doris/datasource/hive/HMSExternalTable.java | 10 +
.../apache/doris/datasource/hive/HiveDlaTable.java | 53 +
.../datasource/hive/HiveExternalMetaCache.java | 30 +-
.../datasource/hive/HmsPartitionBatchExecutor.java | 252 +++
...acheValue.java => HmsPartitionBatchResult.java} | 27 +-
.../datasource/hive/HmsPartitionBatchStats.java | 151 ++
.../datasource/hive/HmsPartitionIdentity.java | 98 ++
.../doris/datasource/hive/HmsPartitionRequest.java | 74 +
.../hive/HmsPartitionResultException.java | 98 ++
...nStatistics.java => HmsPartitionTransport.java} | 15 +-
.../datasource/hive/ThriftHMSCachedClient.java | 74 +-
.../datasource/iceberg/source/IcebergScanNode.java | 4 +
.../doris/datasource/lance/LanceCatalogClient.java | 27 +-
.../datasource/lance/LanceExternalCatalog.java | 16 +-
.../datasource/lance/LanceNamespaceClient.java | 120 +-
.../metacache/LegacyMetaCacheFactory.java | 25 +-
.../doris/datasource/metacache/MetaCache.java | 778 ++++++++-
.../metastore/AbstractLanceProperties.java | 18 +
.../property/storage/HdfsProperties.java | 2 +
.../property/storage/OSSHdfsProperties.java | 2 +
.../apache/doris/job/extensions/mtmv/MTMVTask.java | 11 +
.../org/apache/doris/mtmv/MTMVPartitionUtil.java | 53 +-
.../org/apache/doris/mtmv/MTMVRefreshContext.java | 113 ++
.../org/apache/doris/mtmv/MTMVRelatedTableIf.java | 19 +
.../org/apache/doris/mtmv/MTMVRewriteUtil.java | 21 +-
.../doris/mysql/authenticate/ldap/LdapManager.java | 9 +
.../nereids/analyzer/UnboundHiveTableSink.java | 49 +-
.../doris/nereids/analyzer/UnboundTVFRelation.java | 33 +-
.../nereids/analyzer/UnboundTableSinkCreator.java | 9 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 29 +-
.../apache/doris/nereids/properties/DataTrait.java | 3 +-
.../doris/nereids/properties/FuncDepsDG.java | 53 +
.../nereids/rules/analysis/BindExpression.java | 40 +
.../doris/nereids/rules/analysis/BindSink.java | 92 +-
.../rewrite/PushDownTopNDistinctThroughJoin.java | 42 +-
.../rules/rewrite/VariantSubPathPruning.java | 20 +-
.../expressions/functions/table/VectorSearch.java | 9 +-
.../trees/plans/commands/PrepareCommand.java | 3 +
.../plans/commands/info/CreateResourceInfo.java | 10 +
.../insert/InsertOverwriteTableCommand.java | 3 +-
.../trees/plans/commands/insert/InsertUtils.java | 18 +-
.../java/org/apache/doris/qe/ConnectContext.java | 5 +
.../java/org/apache/doris/qe/FEOpExecutor.java | 25 +-
.../org/apache/doris/qe/MysqlConnectProcessor.java | 10 +
.../VectorSearchTableValuedFunction.java | 71 +-
.../datasource/ExternalCatalogDeadlockTest.java | 689 +++++++-
.../org/apache/doris/datasource/MetaCacheTest.java | 1723 ++++++++++++++++++++
.../datasource/hive/HiveDDLAndDMLPlanTest.java | 104 ++
.../hive/HmsPartitionBatchExecutorTest.java | 282 ++++
.../iceberg/source/IcebergScanNodeTest.java | 77 +
.../lance/LanceCatalogLifecycleTest.java | 169 ++
.../lance/LanceTableAccessCacheTest.java | 279 ++++
...baseNameComparedLowercaseMetaCacheTrueTest.java | 16 +
...ableNameComparedLowercaseMetaCacheTrueTest.java | 26 +
.../property/ConnectorPropertiesUtilsTest.java | 32 +
.../property/metastore/LancePropertiesTest.java | 16 +
.../property/storage/HdfsPropertiesTest.java | 12 +
.../apache/doris/mtmv/MTMVPartitionUtilTest.java | 27 +-
.../doris/mtmv/MTMVRefreshContextBatchTest.java | 239 +++
.../org/apache/doris/mtmv/MTMVRewriteUtilTest.java | 24 +-
.../mysql/authenticate/ldap/LdapManagerTest.java | 77 +
.../nereids/parser/LancePreparedStatementTest.java | 87 +
.../doris/nereids/properties/FuncDepsDGTest.java | 15 +
.../rules/rewrite/PruneNestedColumnTest.java | 14 +
.../PushDownTopNDistinctThroughJoinTest.java | 141 ++
.../plans/commands/CreateResourceCommandTest.java | 75 +
.../qe/MysqlPreparedStatementForwardingTest.java | 148 ++
.../tablefunction/LancePreparedSearchTest.java | 258 +++
.../property/ConnectorPropertiesUtils.java | 23 +
.../foundation/property/ConnectorProperty.java | 2 +
.../property/ConnectorPropertyValidator.java | 50 +
.../property/NoPathTraversalValidator.java | 45 +
gensrc/thrift/PlanNodes.thrift | 1 +
.../test_remote_doris_variant_select.out | 81 -
.../push_down_top_n_distinct_through_join.out | 7 +
.../variant_p0/test_variant_sub_path_order.out | 11 +
.../write/test_hive_write_static_partition.groovy | 383 +++++
.../lance/test_lance_prepared_vector_search.groovy | 112 ++
.../test_remote_doris_variant_select.groovy | 70 +-
.../test_variant_subcolumn_index_norms.groovy | 1 +
.../push_down_top_n_distinct_through_join.groovy | 41 +
.../suites/search/test_search_score_cache.groovy | 5 +
.../variant_p0/test_variant_sub_path_order.groovy | 72 +
126 files changed, 9129 insertions(+), 470 deletions(-)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HmsPartitionBatchExecutor.java
copy
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/{HMSSchemaCacheValue.java
=> HmsPartitionBatchResult.java} (56%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HmsPartitionBatchStats.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HmsPartitionIdentity.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HmsPartitionRequest.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HmsPartitionResultException.java
copy
fe/fe-core/src/main/java/org/apache/doris/datasource/hive/{HiveColumnStatistics.java
=> HmsPartitionTransport.java} (69%)
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HmsPartitionBatchExecutorTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceTableAccessCacheTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRefreshContextBatchTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/nereids/parser/LancePreparedStatementTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownTopNDistinctThroughJoinTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/qe/MysqlPreparedStatementForwardingTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/tablefunction/LancePreparedSearchTest.java
create mode 100644
fe/fe-foundation/src/main/java/org/apache/doris/foundation/property/ConnectorPropertyValidator.java
create mode 100644
fe/fe-foundation/src/main/java/org/apache/doris/foundation/property/NoPathTraversalValidator.java
delete mode 100644
regression-test/data/external_table_p0/remote_doris/test_remote_doris_variant_select.out
create mode 100644
regression-test/data/variant_p0/test_variant_sub_path_order.out
create mode 100644
regression-test/suites/external_table_p0/hive/write/test_hive_write_static_partition.groovy
create mode 100644
regression-test/suites/external_table_p0/lance/test_lance_prepared_vector_search.groovy
create mode 100644
regression-test/suites/variant_p0/test_variant_sub_path_order.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
