This is an automated email from the ASF dual-hosted git repository. panxiaolei pushed a change to branch refactor_rf in repository https://gitbox.apache.org/repos/asf/doris.git
from 0078e1f02cb [fix](func) Fix be core dump caused by mem out of bound (#48693) add 22cfc09ed9e [opt](vault) Check hdfs connectivity when creating hdfs storage vault (#48369) add 5ec7d348bf9 [opt](nereids) speedup huge InPredicate for partition pruning (#47608) add 2c18cf2d2d4 [test](cold hot separation) add a cold hot separation regression test for hdfs (#48452) add 5e64736aa96 [fix](Nereids) string constant folding process regex delim by mistake (#48783) add 6521cb6e182 [bugfix](hive) fix_hive_metastore_client_socket_timeout (#47378) add d1ad3e6b673 [Fix](merge-on-write) should re-calculate delete bitmaps between segments if BE restart before publish (#48775) add 4d9faeddeac [chore](TableName)remove useless method (#48790) add 63a601fea42 [refactor](minor) Rename scanners (#48810) add 00ec5875d21 [fix](load) add lock for runtime_state->tablet_commit_infos (#48709) add 2988dc3c274 [enhancement](maxcompute)support maxcompute timestamp column type. (#48768) add a9fb3d21695 [Chore](check) modify sanity_check's rows type from int32 to int64 (#48821) add 58fe4cb29df [refactor](pipeline) Refine pipeline logics (#48795) add 752fc5e7e06 [Feat](Nereids) refactor show table command (#48833) add 865b45c35be [improvement](statistics)Support convert ip type to double for column stats. (#48823) add 5a15b5dc5fe [Enhancement] (nereids) support show table status command (#48715) add 315e6a95942 [Fix](cloud) Should calculate delete bitmaps between segments when re-calculate for all historical data if cache missed in publish phase (#48748) add 2000a5c4762 [fix](binlog) Replaced partition should be record to Dropped Resources (#48686) new 3e19f581b32 refactor of RuntimeFilterSlots new d5817710883 some fix and add ut new d9a559dc2f1 [refactor](runtime filter) Refine null property in runtime filter (#48534) new 9db838121b6 [runtime filter](UT) test in filter (#48562) new dbc4eacaa20 add more test and some fix new 5640bdcf6d2 [runtime filter](UT) test bloom filter (#48575) new 874718c29da fix coredump new b804ee00000 [rf](ut) test minmax/bitmap filter (#48579) new 5eb01aa8779 fix compile fail and format new a1be0a66487 fix UT (#48591) new aacd25e62dd test in or bloom (#48596) new 930566af3e0 add more ut of roles new 74bcdbc464c test rf utils (#48615) new 6e4173933d9 test (#48630) new db3ebf6a587 format (#48658) new 5ee537f3007 test RuntimeFilterMgr (#48672) new 5e5a483be0f test RuntimeFilterMergeControllerEntity (#48678) new c51219ca78c add rpc fail test and eof test new 9d710a686f7 [test](ut) add ut for bloom filter and hybridset (#48698) new 4b6720a4033 [test](bloom_filter_func) add case about algorithm new d1953871c2e resolve conflict new 36be8a583b0 [test](rf) Improve test about bloom filter func new 6c6033c0fc9 add some metrics The 23 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: be/CMakeLists.txt | 2 + .../cloud/cloud_engine_calc_delete_bitmap_task.cpp | 10 + be/src/exec/olap_common.h | 26 +- be/src/exprs/bitmapfilter_predicate.h | 6 +- be/src/exprs/bloom_filter_func.h | 464 +----- be/src/exprs/bloom_filter_func_adaptor.h | 266 +++ be/src/exprs/bloom_filter_func_impl.h | 127 ++ be/src/exprs/create_predicate_function.h | 132 +- .../exec/scan/vscan_node.h => exprs/filter_base.h} | 34 +- be/src/exprs/hybrid_map.h | 57 - be/src/exprs/hybrid_set.h | 59 +- be/src/exprs/minmax_predicate.h | 125 +- be/src/exprs/runtime_filter.cpp | 1737 -------------------- be/src/exprs/runtime_filter.h | 557 ------- be/src/exprs/runtime_filter_slots.h | 186 --- be/src/exprs/runtime_filter_slots_cross.h | 89 - be/src/io/fs/multi_table_pipe.cpp | 4 +- be/src/olap/base_tablet.cpp | 3 + be/src/olap/bitmap_filter_predicate.h | 5 - be/src/olap/bloom_filter_predicate.h | 17 +- be/src/olap/column_predicate.h | 35 +- .../scan/vscan_node.h => olap/filter_olap_param.h} | 34 +- be/src/olap/in_list_predicate.h | 13 +- be/src/olap/parallel_scanner_builder.cpp | 14 +- be/src/olap/parallel_scanner_builder.h | 12 +- be/src/olap/rowset/segment_v2/segment_iterator.cpp | 9 +- be/src/olap/tablet.cpp | 1 - be/src/olap/tablet_meta.cpp | 4 + be/src/olap/tablet_meta.h | 2 + be/src/olap/tablet_reader.cpp | 45 +- be/src/olap/tablet_reader.h | 9 +- be/src/olap/txn_manager.cpp | 11 + be/src/pipeline/common/runtime_filter_consumer.cpp | 182 -- be/src/pipeline/dependency.cpp | 4 +- be/src/pipeline/dependency.h | 5 +- be/src/pipeline/exec/datagen_operator.cpp | 9 +- be/src/pipeline/exec/es_scan_operator.cpp | 19 +- be/src/pipeline/exec/es_scan_operator.h | 6 +- be/src/pipeline/exec/file_scan_operator.cpp | 6 +- be/src/pipeline/exec/file_scan_operator.h | 4 +- .../exec/group_commit_block_sink_operator.h | 1 + be/src/pipeline/exec/hashjoin_build_sink.cpp | 93 +- be/src/pipeline/exec/hashjoin_build_sink.h | 8 +- be/src/pipeline/exec/jdbc_scan_operator.cpp | 6 +- be/src/pipeline/exec/jdbc_scan_operator.h | 6 +- be/src/pipeline/exec/join_build_sink_operator.cpp | 5 - be/src/pipeline/exec/join_build_sink_operator.h | 8 - be/src/pipeline/exec/meta_scan_operator.cpp | 6 +- be/src/pipeline/exec/meta_scan_operator.h | 6 +- .../exec/multi_cast_data_stream_source.cpp | 14 +- .../pipeline/exec/multi_cast_data_stream_source.h | 8 +- .../exec/nested_loop_join_build_operator.cpp | 57 +- .../exec/nested_loop_join_build_operator.h | 12 +- be/src/pipeline/exec/olap_scan_operator.cpp | 28 +- be/src/pipeline/exec/olap_scan_operator.h | 8 +- be/src/pipeline/exec/operator.h | 2 +- .../exec/partitioned_hash_join_probe_operator.cpp | 14 +- .../exec/partitioned_hash_join_sink_operator.cpp | 5 +- be/src/pipeline/exec/scan_operator.cpp | 65 +- be/src/pipeline/exec/scan_operator.h | 32 +- be/src/pipeline/pipeline_fragment_context.cpp | 5 +- be/src/pipeline/pipeline_fragment_context.h | 17 - be/src/pipeline/task_scheduler.cpp | 3 - be/src/runtime/descriptor_helper.h | 55 + be/src/runtime/fragment_mgr.cpp | 190 +-- be/src/runtime/fragment_mgr.h | 8 +- be/src/runtime/query_context.cpp | 1 + be/src/runtime/query_context.h | 6 +- be/src/runtime/runtime_filter_mgr.h | 296 ---- be/src/runtime/runtime_state.cpp | 25 +- be/src/runtime/runtime_state.h | 48 +- .../runtime/stream_load/stream_load_executor.cpp | 2 +- be/src/runtime_filter/CMakeLists.txt | 27 + be/src/runtime_filter/runtime_filter.cpp | 132 ++ be/src/runtime_filter/runtime_filter.h | 132 ++ be/src/runtime_filter/runtime_filter_consumer.cpp | 229 +++ be/src/runtime_filter/runtime_filter_consumer.h | 140 ++ .../runtime_filter_consumer_helper.cpp | 151 ++ .../runtime_filter_consumer_helper.h} | 82 +- be/src/runtime_filter/runtime_filter_definitions.h | 100 ++ be/src/runtime_filter/runtime_filter_merger.h | 126 ++ .../runtime_filter_mgr.cpp | 295 ++-- be/src/runtime_filter/runtime_filter_mgr.h | 176 ++ be/src/runtime_filter/runtime_filter_producer.cpp | 244 +++ be/src/runtime_filter/runtime_filter_producer.h | 169 ++ .../runtime_filter_producer_helper.cpp | 138 ++ .../runtime_filter_producer_helper.h | 87 + .../runtime_filter_producer_helper_cross.h | 69 + be/src/runtime_filter/runtime_filter_wrapper.cpp | 688 ++++++++ be/src/runtime_filter/runtime_filter_wrapper.h | 142 ++ be/src/runtime_filter/utils.cpp | 196 +++ .../utils.h} | 41 +- be/src/service/internal_service.cpp | 1 - be/src/util/doris_metrics.cpp | 11 + be/src/util/doris_metrics.h | 5 + .../aggregate_function_group_array_intersect.h | 43 +- be/src/vec/columns/column_string.cpp | 16 +- be/src/vec/exec/format/csv/csv_reader.cpp | 2 +- be/src/vec/exec/format/json/new_json_reader.cpp | 2 +- be/src/vec/exec/format/orc/vorc_reader.cpp | 3 +- .../exec/format/parquet/vparquet_group_reader.cpp | 3 +- be/src/vec/exec/format/table/equality_delete.cpp | 2 +- .../scan/{new_es_scanner.cpp => es_scanner.cpp} | 33 +- .../exec/scan/{new_es_scanner.h => es_scanner.h} | 14 +- .../scan/{vfile_scanner.cpp => file_scanner.cpp} | 76 +- .../exec/scan/{vfile_scanner.h => file_scanner.h} | 20 +- .../{new_jdbc_scanner.cpp => jdbc_scanner.cpp} | 35 +- .../scan/{new_jdbc_scanner.h => jdbc_scanner.h} | 12 +- .../scan/{vmeta_scanner.cpp => meta_scanner.cpp} | 106 +- .../exec/scan/{vmeta_scanner.h => meta_scanner.h} | 12 +- .../{new_olap_scanner.cpp => olap_scanner.cpp} | 40 +- .../scan/{new_olap_scanner.h => olap_scanner.h} | 10 +- be/src/vec/exec/scan/{vscan_node.h => scan_node.h} | 8 +- be/src/vec/exec/scan/{vscanner.cpp => scanner.cpp} | 31 +- be/src/vec/exec/scan/{vscanner.h => scanner.h} | 10 +- be/src/vec/exec/scan/scanner_context.cpp | 2 +- be/src/vec/exec/scan/scanner_context.h | 4 +- be/src/vec/exec/scan/scanner_scheduler.cpp | 14 +- be/src/vec/exec/scan/scanner_scheduler.h | 2 +- be/src/vec/exprs/vbitmap_predicate.cpp | 2 +- be/src/vec/exprs/vbitmap_predicate.h | 2 +- be/src/vec/exprs/vbloom_predicate.cpp | 4 +- be/src/vec/exprs/vbloom_predicate.h | 3 +- be/src/vec/exprs/vruntimefilter_wrapper.cpp | 10 +- be/src/vec/exprs/vruntimefilter_wrapper.h | 5 +- be/src/vec/functions/in.h | 9 +- be/src/vec/runtime/shared_hash_table_controller.h | 17 +- be/src/vec/sink/writer/vtablet_writer.cpp | 21 +- be/src/vec/sink/writer/vtablet_writer_v2.cpp | 5 +- be/test/CMakeLists.txt | 1 - be/test/exprs/bloom_filter_func_test.cpp | 626 +++++++ be/test/exprs/hybrid_set_test.cpp | 301 +++- be/test/exprs/minmax_predicate_test.cpp | 107 ++ be/test/exprs/runtime_filter_test.cpp | 111 -- be/test/olap/date_bloom_filter_test.cpp | 8 +- be/test/pipeline/local_exchanger_test.cpp | 2 +- .../operator/partitioned_hash_join_test_helper.h | 15 + be/test/pipeline/pipeline_test.cpp | 76 +- be/test/pipeline/thrift_builder.h | 262 ++- .../runtime_filter_consumer_helper_test.cpp | 108 ++ .../runtime_filter_consumer_test.cpp | 188 +++ .../runtime_filter_definitions_test.cpp | 70 + .../runtime_filter/runtime_filter_merger_test.cpp | 212 +++ be/test/runtime_filter/runtime_filter_mgr_test.cpp | 185 +++ .../runtime_filter_producer_helper_cross_test.cpp | 84 + .../runtime_filter_producer_helper_test.cpp | 171 ++ .../runtime_filter_producer_test.cpp | 207 +++ be/test/runtime_filter/runtime_filter_test_utils.h | 74 + .../runtime_filter/runtime_filter_wrapper_test.cpp | 1400 ++++++++++++++++ be/test/runtime_filter/utils_test.cpp | 119 ++ be/test/scan/scanner_context_test.cpp | 70 +- be/test/vec/exec/vfile_scanner_exception_test.cpp | 10 +- be/test/vec/exec/vwal_scanner_test.cpp | 12 +- .../doris/maxcompute/MaxComputeColumnValue.java | 12 +- .../antlr4/org/apache/doris/nereids/DorisParser.g4 | 2 +- .../org/apache/doris/analysis/LargeIntLiteral.java | 2 +- .../java/org/apache/doris/analysis/TableName.java | 20 - .../java/org/apache/doris/binlog/DBBinlog.java | 9 + .../main/java/org/apache/doris/catalog/Env.java | 12 +- .../org/apache/doris/catalog/HdfsStorageVault.java | 66 +- .../java/org/apache/doris/catalog/OlapTable.java | 5 +- .../org/apache/doris/catalog/StorageVault.java | 1 - .../cache/NereidsSortedPartitionsCacheManager.java | 6 + .../doris/datasource/hive/HMSExternalCatalog.java | 2 +- .../datasource/hive/ThriftHMSCachedClient.java | 2 +- .../maxcompute/MaxComputeExternalCatalog.java | 10 +- .../maxcompute/MaxComputeExternalTable.java | 1 + .../maxcompute/source/MaxComputeScanNode.java | 43 +- .../property/constants/MCProperties.java | 4 + .../apache/doris/fs/remote/dfs/DFSFileSystem.java | 2 +- .../doris/nereids/parser/LogicalPlanBuilder.java | 29 + .../rules/exploration/mv/PredicatesSplitter.java | 2 +- .../rules/expression/rules/ColumnRange.java | 24 +- .../rules/OneListPartitionEvaluator.java | 89 +- .../rules/OneRangePartitionEvaluator.java | 52 +- .../nereids/rules/expression/rules/OrToIn.java | 1 - .../rules/PartitionPredicateToRange.java | 15 +- .../nereids/trees/expressions/Expression.java | 26 + .../nereids/trees/expressions/InPredicate.java | 58 +- .../functions/executable/StringArithmetic.java | 57 +- .../trees/expressions/literal/IPv4Literal.java | 5 + .../trees/expressions/literal/IPv6Literal.java | 5 + .../nereids/trees/expressions/literal/Literal.java | 98 +- .../visitor/ExpressionColumnFilterConverter.java | 5 + .../apache/doris/nereids/trees/plans/PlanType.java | 1 + .../trees/plans/commands/ShowTableCommand.java | 62 +- ...bleCommand.java => ShowTableStatusCommand.java} | 184 +-- .../trees/plans/commands/info/AliasInfo.java | 55 + .../trees/plans/commands/info/TableNameInfo.java | 15 + .../trees/plans/logical/LogicalOlapScan.java | 6 +- .../trees/plans/visitor/CommandVisitor.java | 5 + .../apache/doris/nereids/util/ExpressionUtils.java | 3 +- .../java/org/apache/doris/nereids/util/Utils.java | 42 + .../persist/ReplacePartitionOperationLog.java | 18 +- .../org/apache/doris/planner/RuntimeFilter.java | 4 +- .../java/org/apache/doris/qe/SessionVariable.java | 22 +- .../doris/statistics/util/StatisticsUtil.java | 12 + .../doris/cloud/catalog/HdfsStorageVaultTest.java | 76 +- gensrc/proto/internal_service.proto | 24 +- gensrc/thrift/PaloInternalService.thrift | 4 +- gensrc/thrift/PlanNodes.thrift | 4 +- .../maxcompute/test_max_compute_timestamp.out | Bin 0 -> 3944 bytes ...est_cloud_multi_segments_re_calc_in_publish.out | Bin 0 -> 202 bytes ...est_local_multi_segments_re_calc_in_publish.out | Bin 0 -> 202 bytes .../test_inject_send_filter_size_fail.out | Bin 0 -> 197 bytes .../org/apache/doris/regression/suite/Suite.groovy | 12 + .../suites/auth_call/test_dml_analyze_auth.groovy | 9 +- .../cold_data_compaction_by_hdfs.groovy | 129 ++ .../maxcompute/test_max_compute_timestamp.groovy | 216 +++ ..._cloud_multi_segments_re_calc_in_publish.groovy | 123 ++ ..._local_multi_segments_re_calc_in_publish.groovy | 174 ++ .../fold_constant_string_arithmatic.groovy | 123 +- .../test_inject_send_filter_size_fail.groovy | 120 ++ .../join/test_low_bucket/test_low_bucket.groovy | 1 + .../join/test_slow_close/test_slow_close.groovy | 2 + .../suites/statistics/test_analyze_ip_type.groovy | 97 ++ .../vault_p0/create/test_create_vault.groovy | 4 +- .../test_create_vault_with_case_sensitive.groovy | 7 +- .../create/test_create_vault_with_kerberos.groovy | 18 +- .../privilege/test_vault_privilege_restart.groovy | 3 +- 220 files changed, 10434 insertions(+), 5554 deletions(-) create mode 100644 be/src/exprs/bloom_filter_func_adaptor.h create mode 100644 be/src/exprs/bloom_filter_func_impl.h copy be/src/{vec/exec/scan/vscan_node.h => exprs/filter_base.h} (52%) delete mode 100644 be/src/exprs/hybrid_map.h delete mode 100644 be/src/exprs/runtime_filter.cpp delete mode 100644 be/src/exprs/runtime_filter.h delete mode 100644 be/src/exprs/runtime_filter_slots.h delete mode 100644 be/src/exprs/runtime_filter_slots_cross.h copy be/src/{vec/exec/scan/vscan_node.h => olap/filter_olap_param.h} (51%) delete mode 100644 be/src/pipeline/common/runtime_filter_consumer.cpp delete mode 100644 be/src/runtime/runtime_filter_mgr.h create mode 100644 be/src/runtime_filter/CMakeLists.txt create mode 100644 be/src/runtime_filter/runtime_filter.cpp create mode 100644 be/src/runtime_filter/runtime_filter.h create mode 100644 be/src/runtime_filter/runtime_filter_consumer.cpp create mode 100644 be/src/runtime_filter/runtime_filter_consumer.h create mode 100644 be/src/runtime_filter/runtime_filter_consumer_helper.cpp rename be/src/{pipeline/common/runtime_filter_consumer.h => runtime_filter/runtime_filter_consumer_helper.h} (51%) create mode 100644 be/src/runtime_filter/runtime_filter_definitions.h create mode 100644 be/src/runtime_filter/runtime_filter_merger.h rename be/src/{runtime => runtime_filter}/runtime_filter_mgr.cpp (60%) create mode 100644 be/src/runtime_filter/runtime_filter_mgr.h create mode 100644 be/src/runtime_filter/runtime_filter_producer.cpp create mode 100644 be/src/runtime_filter/runtime_filter_producer.h create mode 100644 be/src/runtime_filter/runtime_filter_producer_helper.cpp create mode 100644 be/src/runtime_filter/runtime_filter_producer_helper.h create mode 100644 be/src/runtime_filter/runtime_filter_producer_helper_cross.h create mode 100644 be/src/runtime_filter/runtime_filter_wrapper.cpp create mode 100644 be/src/runtime_filter/runtime_filter_wrapper.h create mode 100644 be/src/runtime_filter/utils.cpp rename be/src/{exprs/runtime_filter_convertor.h => runtime_filter/utils.h} (72%) rename be/src/vec/exec/scan/{new_es_scanner.cpp => es_scanner.cpp} (83%) rename be/src/vec/exec/scan/{new_es_scanner.h => es_scanner.h} (82%) rename be/src/vec/exec/scan/{vfile_scanner.cpp => file_scanner.cpp} (96%) rename be/src/vec/exec/scan/{vfile_scanner.h => file_scanner.h} (93%) rename be/src/vec/exec/scan/{new_jdbc_scanner.cpp => jdbc_scanner.cpp} (86%) rename be/src/vec/exec/scan/{new_jdbc_scanner.h => jdbc_scanner.h} (88%) rename be/src/vec/exec/scan/{vmeta_scanner.cpp => meta_scanner.cpp} (84%) rename be/src/vec/exec/scan/{vmeta_scanner.h => meta_scanner.h} (92%) rename be/src/vec/exec/scan/{new_olap_scanner.cpp => olap_scanner.cpp} (96%) rename be/src/vec/exec/scan/{new_olap_scanner.h => olap_scanner.h} (91%) rename be/src/vec/exec/scan/{vscan_node.h => scan_node.h} (90%) rename be/src/vec/exec/scan/{vscanner.cpp => scanner.cpp} (90%) rename be/src/vec/exec/scan/{vscanner.h => scanner.h} (97%) create mode 100644 be/test/exprs/bloom_filter_func_test.cpp create mode 100644 be/test/exprs/minmax_predicate_test.cpp delete mode 100644 be/test/exprs/runtime_filter_test.cpp create mode 100644 be/test/runtime_filter/runtime_filter_consumer_helper_test.cpp create mode 100644 be/test/runtime_filter/runtime_filter_consumer_test.cpp create mode 100644 be/test/runtime_filter/runtime_filter_definitions_test.cpp create mode 100644 be/test/runtime_filter/runtime_filter_merger_test.cpp create mode 100644 be/test/runtime_filter/runtime_filter_mgr_test.cpp create mode 100644 be/test/runtime_filter/runtime_filter_producer_helper_cross_test.cpp create mode 100644 be/test/runtime_filter/runtime_filter_producer_helper_test.cpp create mode 100644 be/test/runtime_filter/runtime_filter_producer_test.cpp create mode 100644 be/test/runtime_filter/runtime_filter_test_utils.h create mode 100644 be/test/runtime_filter/runtime_filter_wrapper_test.cpp create mode 100644 be/test/runtime_filter/utils_test.cpp copy fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/{ShowTableCommand.java => ShowTableStatusCommand.java} (50%) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AliasInfo.java create mode 100644 regression-test/data/external_table_p2/maxcompute/test_max_compute_timestamp.out create mode 100644 regression-test/data/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.out create mode 100644 regression-test/data/fault_injection_p0/test_local_multi_segments_re_calc_in_publish.out create mode 100644 regression-test/data/query_p0/join/test_inject_send_filter_size_fail/test_inject_send_filter_size_fail.out create mode 100644 regression-test/suites/cold_heat_separation/cold_data_compaction_by_hdfs.groovy create mode 100644 regression-test/suites/external_table_p2/maxcompute/test_max_compute_timestamp.groovy create mode 100644 regression-test/suites/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.groovy create mode 100644 regression-test/suites/fault_injection_p0/test_local_multi_segments_re_calc_in_publish.groovy create mode 100644 regression-test/suites/query_p0/join/test_inject_send_filter_size_fail/test_inject_send_filter_size_fail.groovy create mode 100644 regression-test/suites/statistics/test_analyze_ip_type.groovy --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org