This is an automated email from the ASF dual-hosted git repository. mrhhsg pushed a change to branch refactor_rf in repository https://gitbox.apache.org/repos/asf/doris.git
discard 36be8a583b0 [test](rf) Improve test about bloom filter func discard d1953871c2e resolve conflict discard 4b6720a4033 [test](bloom_filter_func) add case about algorithm discard 9d710a686f7 [test](ut) add ut for bloom filter and hybridset (#48698) discard c51219ca78c add rpc fail test and eof test discard 5e5a483be0f test RuntimeFilterMergeControllerEntity (#48678) discard 5ee537f3007 test RuntimeFilterMgr (#48672) discard db3ebf6a587 format (#48658) discard 6e4173933d9 test (#48630) discard 74bcdbc464c test rf utils (#48615) discard 930566af3e0 add more ut of roles discard aacd25e62dd test in or bloom (#48596) discard a1be0a66487 fix UT (#48591) discard 5eb01aa8779 fix compile fail and format discard b804ee00000 [rf](ut) test minmax/bitmap filter (#48579) discard 874718c29da fix coredump discard 5640bdcf6d2 [runtime filter](UT) test bloom filter (#48575) discard dbc4eacaa20 add more test and some fix discard 9db838121b6 [runtime filter](UT) test in filter (#48562) discard d9a559dc2f1 [refactor](runtime filter) Refine null property in runtime filter (#48534) discard d5817710883 some fix and add ut discard 3e19f581b32 refactor of RuntimeFilterSlots omit 2000a5c4762 [fix](binlog) Replaced partition should be record to Dropped Resources (#48686) omit 315e6a95942 [Fix](cloud) Should calculate delete bitmaps between segments when re-calculate for all historical data if cache missed in publish phase (#48748) omit 5a15b5dc5fe [Enhancement] (nereids) support show table status command (#48715) omit 865b45c35be [improvement](statistics)Support convert ip type to double for column stats. (#48823) omit 752fc5e7e06 [Feat](Nereids) refactor show table command (#48833) omit 58fe4cb29df [refactor](pipeline) Refine pipeline logics (#48795) omit a9fb3d21695 [Chore](check) modify sanity_check's rows type from int32 to int64 (#48821) omit 2988dc3c274 [enhancement](maxcompute)support maxcompute timestamp column type. (#48768) omit 00ec5875d21 [fix](load) add lock for runtime_state->tablet_commit_infos (#48709) omit 63a601fea42 [refactor](minor) Rename scanners (#48810) omit 4d9faeddeac [chore](TableName)remove useless method (#48790) omit d1ad3e6b673 [Fix](merge-on-write) should re-calculate delete bitmaps between segments if BE restart before publish (#48775) omit 6521cb6e182 [bugfix](hive) fix_hive_metastore_client_socket_timeout (#47378) omit 5e64736aa96 [fix](Nereids) string constant folding process regex delim by mistake (#48783) omit 2c18cf2d2d4 [test](cold hot separation) add a cold hot separation regression test for hdfs (#48452) omit 5ec7d348bf9 [opt](nereids) speedup huge InPredicate for partition pruning (#47608) omit 22cfc09ed9e [opt](vault) Check hdfs connectivity when creating hdfs storage vault (#48369) This update removed existing revisions from the reference, leaving the reference pointing at a previous point in the repository history. * -- * -- N refs/heads/refactor_rf (0078e1f02cb) \ O -- O -- O (36be8a583b0) Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: be/CMakeLists.txt | 2 - .../cloud/cloud_engine_calc_delete_bitmap_task.cpp | 10 - be/src/exec/olap_common.h | 26 +- be/src/exprs/bitmapfilter_predicate.h | 6 +- be/src/exprs/bloom_filter_func.h | 464 +++++- be/src/exprs/bloom_filter_func_adaptor.h | 266 --- be/src/exprs/bloom_filter_func_impl.h | 127 -- be/src/exprs/create_predicate_function.h | 132 +- be/src/exprs/filter_base.h | 38 - .../{service/http_service.h => exprs/hybrid_map.h} | 56 +- be/src/exprs/hybrid_set.h | 59 +- be/src/exprs/minmax_predicate.h | 125 +- be/src/exprs/runtime_filter.cpp | 1737 ++++++++++++++++++++ be/src/exprs/runtime_filter.h | 557 +++++++ .../utils.h => exprs/runtime_filter_convertor.h} | 41 +- be/src/exprs/runtime_filter_slots.h | 186 +++ be/src/exprs/runtime_filter_slots_cross.h | 89 + be/src/io/fs/multi_table_pipe.cpp | 4 +- be/src/olap/base_tablet.cpp | 3 - be/src/olap/bitmap_filter_predicate.h | 5 + be/src/olap/bloom_filter_predicate.h | 17 +- be/src/olap/column_predicate.h | 35 +- be/src/olap/filter_olap_param.h | 34 - be/src/olap/in_list_predicate.h | 13 +- be/src/olap/parallel_scanner_builder.cpp | 14 +- be/src/olap/parallel_scanner_builder.h | 12 +- be/src/olap/rowset/segment_v2/segment_iterator.cpp | 9 +- be/src/olap/tablet.cpp | 1 + be/src/olap/tablet_meta.cpp | 4 - be/src/olap/tablet_meta.h | 2 - be/src/olap/tablet_reader.cpp | 45 +- be/src/olap/tablet_reader.h | 9 +- be/src/olap/txn_manager.cpp | 11 - be/src/pipeline/common/runtime_filter_consumer.cpp | 182 ++ .../common/runtime_filter_consumer.h} | 82 +- be/src/pipeline/dependency.cpp | 4 +- be/src/pipeline/dependency.h | 5 +- be/src/pipeline/exec/datagen_operator.cpp | 9 +- be/src/pipeline/exec/es_scan_operator.cpp | 19 +- be/src/pipeline/exec/es_scan_operator.h | 6 +- be/src/pipeline/exec/file_scan_operator.cpp | 6 +- be/src/pipeline/exec/file_scan_operator.h | 4 +- .../exec/group_commit_block_sink_operator.h | 1 - be/src/pipeline/exec/hashjoin_build_sink.cpp | 93 +- be/src/pipeline/exec/hashjoin_build_sink.h | 8 +- be/src/pipeline/exec/jdbc_scan_operator.cpp | 6 +- be/src/pipeline/exec/jdbc_scan_operator.h | 6 +- be/src/pipeline/exec/join_build_sink_operator.cpp | 5 + be/src/pipeline/exec/join_build_sink_operator.h | 8 + be/src/pipeline/exec/meta_scan_operator.cpp | 6 +- be/src/pipeline/exec/meta_scan_operator.h | 6 +- .../exec/multi_cast_data_stream_source.cpp | 14 +- .../pipeline/exec/multi_cast_data_stream_source.h | 8 +- .../exec/nested_loop_join_build_operator.cpp | 57 +- .../exec/nested_loop_join_build_operator.h | 12 +- be/src/pipeline/exec/olap_scan_operator.cpp | 28 +- be/src/pipeline/exec/olap_scan_operator.h | 8 +- be/src/pipeline/exec/operator.h | 2 +- .../exec/partitioned_hash_join_probe_operator.cpp | 14 +- .../exec/partitioned_hash_join_sink_operator.cpp | 5 +- be/src/pipeline/exec/scan_operator.cpp | 65 +- be/src/pipeline/exec/scan_operator.h | 32 +- be/src/pipeline/pipeline_fragment_context.cpp | 5 +- be/src/pipeline/pipeline_fragment_context.h | 17 + be/src/pipeline/task_scheduler.cpp | 3 + be/src/runtime/descriptor_helper.h | 55 - be/src/runtime/fragment_mgr.cpp | 190 ++- be/src/runtime/fragment_mgr.h | 8 +- be/src/runtime/query_context.cpp | 1 - be/src/runtime/query_context.h | 6 +- .../runtime_filter_mgr.cpp | 295 ++-- be/src/runtime/runtime_filter_mgr.h | 296 ++++ be/src/runtime/runtime_state.cpp | 25 +- be/src/runtime/runtime_state.h | 48 +- .../runtime/stream_load/stream_load_executor.cpp | 2 +- be/src/runtime_filter/CMakeLists.txt | 27 - be/src/runtime_filter/runtime_filter.cpp | 132 -- be/src/runtime_filter/runtime_filter.h | 132 -- be/src/runtime_filter/runtime_filter_consumer.cpp | 225 --- be/src/runtime_filter/runtime_filter_consumer.h | 138 -- .../runtime_filter_consumer_helper.cpp | 151 -- be/src/runtime_filter/runtime_filter_definitions.h | 100 -- be/src/runtime_filter/runtime_filter_merger.h | 126 -- be/src/runtime_filter/runtime_filter_mgr.h | 176 -- be/src/runtime_filter/runtime_filter_producer.cpp | 244 --- be/src/runtime_filter/runtime_filter_producer.h | 169 -- .../runtime_filter_producer_helper.cpp | 138 -- .../runtime_filter_producer_helper.h | 87 - .../runtime_filter_producer_helper_cross.h | 69 - be/src/runtime_filter/runtime_filter_wrapper.cpp | 688 -------- be/src/runtime_filter/runtime_filter_wrapper.h | 142 -- be/src/runtime_filter/utils.cpp | 196 --- be/src/service/internal_service.cpp | 1 + .../aggregate_function_group_array_intersect.h | 43 +- be/src/vec/columns/column_string.cpp | 16 +- be/src/vec/exec/format/csv/csv_reader.cpp | 2 +- be/src/vec/exec/format/json/new_json_reader.cpp | 2 +- be/src/vec/exec/format/orc/vorc_reader.cpp | 3 +- .../exec/format/parquet/vparquet_group_reader.cpp | 3 +- be/src/vec/exec/format/table/equality_delete.cpp | 2 +- .../scan/{es_scanner.cpp => new_es_scanner.cpp} | 33 +- .../exec/scan/{es_scanner.h => new_es_scanner.h} | 14 +- .../{jdbc_scanner.cpp => new_jdbc_scanner.cpp} | 35 +- .../scan/{jdbc_scanner.h => new_jdbc_scanner.h} | 12 +- .../{olap_scanner.cpp => new_olap_scanner.cpp} | 40 +- .../scan/{olap_scanner.h => new_olap_scanner.h} | 10 +- be/src/vec/exec/scan/scanner_context.cpp | 2 +- be/src/vec/exec/scan/scanner_context.h | 4 +- be/src/vec/exec/scan/scanner_scheduler.cpp | 14 +- be/src/vec/exec/scan/scanner_scheduler.h | 2 +- .../scan/{file_scanner.cpp => vfile_scanner.cpp} | 76 +- .../exec/scan/{file_scanner.h => vfile_scanner.h} | 20 +- .../scan/{meta_scanner.cpp => vmeta_scanner.cpp} | 106 +- .../exec/scan/{meta_scanner.h => vmeta_scanner.h} | 12 +- be/src/vec/exec/scan/{scan_node.h => vscan_node.h} | 8 +- be/src/vec/exec/scan/{scanner.cpp => vscanner.cpp} | 31 +- be/src/vec/exec/scan/{scanner.h => vscanner.h} | 10 +- be/src/vec/exprs/vbitmap_predicate.cpp | 2 +- be/src/vec/exprs/vbitmap_predicate.h | 2 +- be/src/vec/exprs/vbloom_predicate.cpp | 4 +- be/src/vec/exprs/vbloom_predicate.h | 3 +- be/src/vec/exprs/vruntimefilter_wrapper.cpp | 10 +- be/src/vec/exprs/vruntimefilter_wrapper.h | 5 +- be/src/vec/functions/in.h | 9 +- be/src/vec/runtime/shared_hash_table_controller.h | 17 +- be/src/vec/sink/writer/vtablet_writer.cpp | 21 +- be/src/vec/sink/writer/vtablet_writer_v2.cpp | 5 +- be/test/CMakeLists.txt | 1 + be/test/exprs/bloom_filter_func_test.cpp | 626 ------- be/test/exprs/hybrid_set_test.cpp | 301 +--- be/test/exprs/minmax_predicate_test.cpp | 107 -- be/test/exprs/runtime_filter_test.cpp | 111 ++ be/test/olap/date_bloom_filter_test.cpp | 8 +- be/test/pipeline/local_exchanger_test.cpp | 2 +- .../operator/partitioned_hash_join_test_helper.h | 15 - be/test/pipeline/pipeline_test.cpp | 76 +- be/test/pipeline/thrift_builder.h | 262 +-- .../runtime_filter_consumer_helper_test.cpp | 108 -- .../runtime_filter_consumer_test.cpp | 188 --- .../runtime_filter_definitions_test.cpp | 70 - .../runtime_filter/runtime_filter_merger_test.cpp | 212 --- be/test/runtime_filter/runtime_filter_mgr_test.cpp | 185 --- .../runtime_filter_producer_helper_cross_test.cpp | 84 - .../runtime_filter_producer_helper_test.cpp | 171 -- .../runtime_filter_producer_test.cpp | 207 --- be/test/runtime_filter/runtime_filter_test_utils.h | 74 - .../runtime_filter/runtime_filter_wrapper_test.cpp | 1400 ---------------- be/test/runtime_filter/utils_test.cpp | 119 -- be/test/scan/scanner_context_test.cpp | 70 +- be/test/vec/exec/vfile_scanner_exception_test.cpp | 10 +- be/test/vec/exec/vwal_scanner_test.cpp | 12 +- .../doris/maxcompute/MaxComputeColumnValue.java | 12 +- .../antlr4/org/apache/doris/nereids/DorisParser.g4 | 2 +- .../org/apache/doris/analysis/LargeIntLiteral.java | 2 +- .../java/org/apache/doris/analysis/TableName.java | 20 + .../java/org/apache/doris/binlog/DBBinlog.java | 9 - .../main/java/org/apache/doris/catalog/Env.java | 12 +- .../org/apache/doris/catalog/HdfsStorageVault.java | 66 +- .../java/org/apache/doris/catalog/OlapTable.java | 5 +- .../org/apache/doris/catalog/StorageVault.java | 1 + .../cache/NereidsSortedPartitionsCacheManager.java | 6 - .../doris/datasource/hive/HMSExternalCatalog.java | 2 +- .../datasource/hive/ThriftHMSCachedClient.java | 2 +- .../maxcompute/MaxComputeExternalCatalog.java | 10 +- .../maxcompute/MaxComputeExternalTable.java | 1 - .../maxcompute/source/MaxComputeScanNode.java | 43 +- .../property/constants/MCProperties.java | 4 - .../apache/doris/fs/remote/dfs/DFSFileSystem.java | 2 +- .../doris/nereids/parser/LogicalPlanBuilder.java | 29 - .../rules/exploration/mv/PredicatesSplitter.java | 2 +- .../rules/expression/rules/ColumnRange.java | 24 +- .../rules/OneListPartitionEvaluator.java | 89 +- .../rules/OneRangePartitionEvaluator.java | 52 +- .../nereids/rules/expression/rules/OrToIn.java | 1 + .../rules/PartitionPredicateToRange.java | 15 +- .../nereids/trees/expressions/Expression.java | 26 - .../nereids/trees/expressions/InPredicate.java | 58 +- .../functions/executable/StringArithmetic.java | 57 +- .../trees/expressions/literal/IPv4Literal.java | 5 - .../trees/expressions/literal/IPv6Literal.java | 5 - .../nereids/trees/expressions/literal/Literal.java | 98 +- .../visitor/ExpressionColumnFilterConverter.java | 5 - .../apache/doris/nereids/trees/plans/PlanType.java | 1 - .../trees/plans/commands/ShowTableCommand.java | 62 +- .../plans/commands/ShowTableStatusCommand.java | 243 --- .../trees/plans/commands/info/AliasInfo.java | 55 - .../trees/plans/commands/info/TableNameInfo.java | 15 - .../trees/plans/logical/LogicalOlapScan.java | 6 +- .../trees/plans/visitor/CommandVisitor.java | 5 - .../apache/doris/nereids/util/ExpressionUtils.java | 3 +- .../java/org/apache/doris/nereids/util/Utils.java | 42 - .../persist/ReplacePartitionOperationLog.java | 18 +- .../org/apache/doris/planner/RuntimeFilter.java | 4 +- .../java/org/apache/doris/qe/SessionVariable.java | 22 +- .../doris/statistics/util/StatisticsUtil.java | 12 - .../doris/cloud/catalog/HdfsStorageVaultTest.java | 76 +- gensrc/proto/internal_service.proto | 24 +- gensrc/thrift/PaloInternalService.thrift | 4 +- gensrc/thrift/PlanNodes.thrift | 4 +- .../maxcompute/test_max_compute_timestamp.out | Bin 3944 -> 0 bytes ...est_cloud_multi_segments_re_calc_in_publish.out | Bin 202 -> 0 bytes ...est_local_multi_segments_re_calc_in_publish.out | Bin 202 -> 0 bytes .../test_inject_send_filter_size_fail.out | Bin 197 -> 0 bytes .../org/apache/doris/regression/suite/Suite.groovy | 12 - .../suites/auth_call/test_dml_analyze_auth.groovy | 9 +- .../cold_data_compaction_by_hdfs.groovy | 129 -- .../maxcompute/test_max_compute_timestamp.groovy | 216 --- ..._cloud_multi_segments_re_calc_in_publish.groovy | 123 -- ..._local_multi_segments_re_calc_in_publish.groovy | 174 -- .../fold_constant_string_arithmatic.groovy | 123 +- .../test_inject_send_filter_size_fail.groovy | 120 -- .../join/test_low_bucket/test_low_bucket.groovy | 1 - .../join/test_slow_close/test_slow_close.groovy | 2 - .../suites/statistics/test_analyze_ip_type.groovy | 97 -- .../vault_p0/create/test_create_vault.groovy | 4 +- .../test_create_vault_with_case_sensitive.groovy | 7 +- .../create/test_create_vault_with_kerberos.groovy | 18 +- .../privilege/test_vault_privilege_restart.groovy | 3 +- 218 files changed, 5388 insertions(+), 10640 deletions(-) delete mode 100644 be/src/exprs/bloom_filter_func_adaptor.h delete mode 100644 be/src/exprs/bloom_filter_func_impl.h delete mode 100644 be/src/exprs/filter_base.h copy be/src/{service/http_service.h => exprs/hybrid_map.h} (50%) create mode 100644 be/src/exprs/runtime_filter.cpp create mode 100644 be/src/exprs/runtime_filter.h rename be/src/{runtime_filter/utils.h => exprs/runtime_filter_convertor.h} (72%) create mode 100644 be/src/exprs/runtime_filter_slots.h create mode 100644 be/src/exprs/runtime_filter_slots_cross.h delete mode 100644 be/src/olap/filter_olap_param.h create mode 100644 be/src/pipeline/common/runtime_filter_consumer.cpp rename be/src/{runtime_filter/runtime_filter_consumer_helper.h => pipeline/common/runtime_filter_consumer.h} (51%) rename be/src/{runtime_filter => runtime}/runtime_filter_mgr.cpp (60%) create mode 100644 be/src/runtime/runtime_filter_mgr.h delete mode 100644 be/src/runtime_filter/CMakeLists.txt delete mode 100644 be/src/runtime_filter/runtime_filter.cpp delete mode 100644 be/src/runtime_filter/runtime_filter.h delete mode 100644 be/src/runtime_filter/runtime_filter_consumer.cpp delete mode 100644 be/src/runtime_filter/runtime_filter_consumer.h delete mode 100644 be/src/runtime_filter/runtime_filter_consumer_helper.cpp delete mode 100644 be/src/runtime_filter/runtime_filter_definitions.h delete mode 100644 be/src/runtime_filter/runtime_filter_merger.h delete mode 100644 be/src/runtime_filter/runtime_filter_mgr.h delete mode 100644 be/src/runtime_filter/runtime_filter_producer.cpp delete mode 100644 be/src/runtime_filter/runtime_filter_producer.h delete mode 100644 be/src/runtime_filter/runtime_filter_producer_helper.cpp delete mode 100644 be/src/runtime_filter/runtime_filter_producer_helper.h delete mode 100644 be/src/runtime_filter/runtime_filter_producer_helper_cross.h delete mode 100644 be/src/runtime_filter/runtime_filter_wrapper.cpp delete mode 100644 be/src/runtime_filter/runtime_filter_wrapper.h delete mode 100644 be/src/runtime_filter/utils.cpp rename be/src/vec/exec/scan/{es_scanner.cpp => new_es_scanner.cpp} (83%) rename be/src/vec/exec/scan/{es_scanner.h => new_es_scanner.h} (82%) rename be/src/vec/exec/scan/{jdbc_scanner.cpp => new_jdbc_scanner.cpp} (86%) rename be/src/vec/exec/scan/{jdbc_scanner.h => new_jdbc_scanner.h} (88%) rename be/src/vec/exec/scan/{olap_scanner.cpp => new_olap_scanner.cpp} (96%) rename be/src/vec/exec/scan/{olap_scanner.h => new_olap_scanner.h} (91%) rename be/src/vec/exec/scan/{file_scanner.cpp => vfile_scanner.cpp} (96%) rename be/src/vec/exec/scan/{file_scanner.h => vfile_scanner.h} (93%) rename be/src/vec/exec/scan/{meta_scanner.cpp => vmeta_scanner.cpp} (84%) rename be/src/vec/exec/scan/{meta_scanner.h => vmeta_scanner.h} (92%) rename be/src/vec/exec/scan/{scan_node.h => vscan_node.h} (90%) rename be/src/vec/exec/scan/{scanner.cpp => vscanner.cpp} (90%) rename be/src/vec/exec/scan/{scanner.h => vscanner.h} (97%) delete mode 100644 be/test/exprs/bloom_filter_func_test.cpp delete mode 100644 be/test/exprs/minmax_predicate_test.cpp create mode 100644 be/test/exprs/runtime_filter_test.cpp delete mode 100644 be/test/runtime_filter/runtime_filter_consumer_helper_test.cpp delete mode 100644 be/test/runtime_filter/runtime_filter_consumer_test.cpp delete mode 100644 be/test/runtime_filter/runtime_filter_definitions_test.cpp delete mode 100644 be/test/runtime_filter/runtime_filter_merger_test.cpp delete mode 100644 be/test/runtime_filter/runtime_filter_mgr_test.cpp delete mode 100644 be/test/runtime_filter/runtime_filter_producer_helper_cross_test.cpp delete mode 100644 be/test/runtime_filter/runtime_filter_producer_helper_test.cpp delete mode 100644 be/test/runtime_filter/runtime_filter_producer_test.cpp delete mode 100644 be/test/runtime_filter/runtime_filter_test_utils.h delete mode 100644 be/test/runtime_filter/runtime_filter_wrapper_test.cpp delete mode 100644 be/test/runtime_filter/utils_test.cpp delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowTableStatusCommand.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/AliasInfo.java delete mode 100644 regression-test/data/external_table_p2/maxcompute/test_max_compute_timestamp.out delete mode 100644 regression-test/data/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.out delete mode 100644 regression-test/data/fault_injection_p0/test_local_multi_segments_re_calc_in_publish.out delete mode 100644 regression-test/data/query_p0/join/test_inject_send_filter_size_fail/test_inject_send_filter_size_fail.out delete mode 100644 regression-test/suites/cold_heat_separation/cold_data_compaction_by_hdfs.groovy delete mode 100644 regression-test/suites/external_table_p2/maxcompute/test_max_compute_timestamp.groovy delete mode 100644 regression-test/suites/fault_injection_p0/cloud/test_cloud_multi_segments_re_calc_in_publish.groovy delete mode 100644 regression-test/suites/fault_injection_p0/test_local_multi_segments_re_calc_in_publish.groovy delete mode 100644 regression-test/suites/query_p0/join/test_inject_send_filter_size_fail/test_inject_send_filter_size_fail.groovy delete mode 100644 regression-test/suites/statistics/test_analyze_ip_type.groovy --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org