This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch tpc_preview4
in repository https://gitbox.apache.org/repos/asf/doris.git
from 20dcce00b03 scanner merge after projection
add 58c267e4d32 Manifest cache for tpch1000 (#59178)
add ccaef260512 [opt](multi-catalog) Optimize file split size. (#59175)
add 9b897fc2a62 [Enhancement](parquet)update runtime filter when read next
parquet row group.(#59053) (#59181)
add e3aa5501fec Optimize location for tpch1000 (#59218)
No new revisions were added by this update.
Summary of changes:
be/src/clucene | 1 +
.../runtime_filter_consumer_helper.h | 2 +
.../exec/format/parquet/vparquet_group_reader.h | 7 +
be/src/vec/exec/format/parquet/vparquet_reader.cpp | 38 +-
be/src/vec/exec/format/parquet/vparquet_reader.h | 16 +
be/src/vec/exec/scan/file_scanner.cpp | 24 +-
be/src/vec/exec/scan/file_scanner.h | 3 +-
be/src/vec/exec/scan/scanner.cpp | 1 +
.../scripts/create_preinstalled_scripts/run84.hql | 20 +
.../runtime_filter_dim_small/dim_small.parquet | Bin 0 -> 4230 bytes
.../runtime_filter_fact_big/fact_big.parquet | Bin 0 -> 129338 bytes
fe/check/checkstyle/suppressions.xml | 3 +
.../main/java/org/apache/doris/common/Config.java | 12 +
.../org/apache/doris/common/util/LocationPath.java | 89 +-
.../doris/datasource/ExternalMetaCacheMgr.java | 7 +
.../doris/datasource/FederationBackendPolicy.java | 12 +-
.../apache/doris/datasource/FileQueryScanNode.java | 19 +-
.../org/apache/doris/datasource/FileScanNode.java | 10 +-
.../org/apache/doris/datasource/FileSplitter.java | 230 +++++-
.../apache/doris/datasource/SplitGenerator.java | 2 +-
.../doris/datasource/hive/source/HiveScanNode.java | 78 +-
...gSnapshot.java => IcebergManifestCacheMgr.java} | 21 +-
.../doris/datasource/iceberg/IcebergUtils.java | 8 +
.../iceberg/cache/ContentFileEstimater.java | 194 +++++
.../iceberg/cache/IcebergManifestCache.java | 96 +++
.../iceberg/cache/IcebergManifestCacheLoader.java | 89 ++
.../iceberg/cache/ManifestCacheKey.java} | 32 +-
.../iceberg/cache/ManifestCacheValue.java | 65 ++
.../datasource/iceberg/source/IcebergScanNode.java | 305 ++++++-
.../datasource/paimon/source/PaimonScanNode.java | 47 +-
.../metastore/AbstractIcebergProperties.java | 62 ++
.../property/storage/S3PropertyUtils.java | 48 +-
.../doris/datasource/tvf/source/TVFScanNode.java | 31 +-
.../java/org/apache/doris/qe/SessionVariable.java | 39 +
.../java/org/apache/iceberg/DeleteFileIndex.java | 906 +++++++++++++++++++++
.../apache/doris/datasource/FileSplitterTest.java | 216 +++++
.../paimon/source/PaimonScanNodeTest.java | 17 +
.../doris/planner/FederationBackendPolicyTest.java | 4 +-
.../hive/test_hive_compress_type.groovy | 2 +-
.../hive/test_parquet_join_runtime_filter.groovy | 174 ++++
40 files changed, 2780 insertions(+), 150 deletions(-)
create mode 160000 be/src/clucene
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run84.hql
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/runtime_filter_dim_small/dim_small.parquet
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/runtime_filter_fact_big/fact_big.parquet
copy
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/{IcebergSnapshot.java
=> IcebergManifestCacheMgr.java} (68%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ContentFileEstimater.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/IcebergManifestCache.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/IcebergManifestCacheLoader.java
copy
fe/fe-core/src/main/java/org/apache/doris/{nereids/metrics/EventEnhancer.java
=> datasource/iceberg/cache/ManifestCacheKey.java} (61%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java
create mode 100644
fe/fe-core/src/main/java/org/apache/iceberg/DeleteFileIndex.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/datasource/FileSplitterTest.java
create mode 100644
regression-test/suites/external_table_p0/hive/test_parquet_join_runtime_filter.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
