This is an automated email from the ASF dual-hosted git repository.
freemandealer pushed a change to branch auto-pick-62437-branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
from a73313005e3 [fix](filecache) avoid crash when late holder cleanup sees
removed cache cell (#62437)
add b8f09203bd5 branch-4.0: [fix](cloud-compaction) prevent
EMPTY_CUMULATIVE / BASE-CUMU races on the same tablet #64619 (#64701)
add bd1b866c169 branch-4.0: [fix](subquery) eliminate null aware by
mistake #64639 (#64715)
add b9b770af315 branch-4.0: [fix](fe) Fix broken pipe risk on stream load
redirect with unconsumed request body (#64303)
add 6e4fd993652 branch-4.0: [improvement](rowset) Aggregate non-MOW
segment key bounds (#64305)
add 02a319ebb16 branch-4.0: [test](docker) Verify DML and 3-replica create
table when one BE is down #63401 (#63698)
add 7c70c7f32b7 branch-4.0: [opt](cloud) cache cluster id per query and
drop redundant locks on getBackendId hot path #63636 (#64275)
add 540f1399e3e branch-4.0: [feature](cloud) Add table-level event-driven
warm up (#64544)
add e45bb3893d0 branch-4.0: [fix](filecache) exclude warmup reads from
file cache hit ratio metrics(pick#63394) (#64232)
add 6b900bf686b branch-4.0: [chore](cloud) Support dynamic recycler
instance filter config #63822 (#63940)
add ec0e782e4e7 branch-4.0: [improvement](cloud) Enable packed file and
empty rowset optimization by default #63475 (#63774)
add 5dc358a942b branch-4.0: [fix](s3) Avoid retrying object storage
SlowDown errors #63776 (#64326)
add cde9d5338ae branch-4.0: [fix](filecache) Disable sync file cache clear
in http api(pick#64321) (#64460)
add d71b0cd1af4 branch-4.0: [improvement](filecache) limit file cache LRU
replay queues(pick#64381) (#64462)
add 502d28a9fa9 branch-4.0: [test](regression) Remove no simdjson flexible
update case #64513 (#64540)
add d000f924d36 branch-4.0: [fix](cloud) Fix some docker case due to open
skip_writing_empty_rowset_metadata #64565 (#64594)
add 8b77d4f8628 branch-4.0: [fix](cloud) Recycle empty rowsets without
resource id #64630 (#64642)
add 0a516071b07 branch-4.0: [fix](cloud) Validate recycle rowset key state
during commit rowset #63985 (#64286)
add 41871624047 Merge branch 'branch-4.0' into auto-pick-62437-branch-4.0
No new revisions were added by this update.
Summary of changes:
be/src/cloud/cloud_backend_service.cpp | 6 +-
be/src/cloud/cloud_delete_task.cpp | 2 +-
be/src/cloud/cloud_delta_writer.cpp | 6 +-
be/src/cloud/cloud_internal_service.cpp | 367 ++++---
be/src/cloud/cloud_meta_mgr.cpp | 4 +-
be/src/cloud/cloud_meta_mgr.h | 2 +-
be/src/cloud/cloud_schema_change_job.cpp | 4 +-
be/src/cloud/cloud_snapshot_mgr.cpp | 8 +
be/src/cloud/cloud_warm_up_manager.cpp | 223 ++++-
be/src/cloud/cloud_warm_up_manager.h | 39 +-
be/src/cloud/cloud_warmup_metrics.cpp | 82 ++
be/src/cloud/cloud_warmup_metrics.h | 76 ++
be/src/cloud/config.cpp | 4 +-
be/src/cloud/pb_convert.cpp | 28 +-
be/src/common/config.cpp | 8 +-
be/src/common/config.h | 6 +
be/src/http/action/file_cache_action.cpp | 12 +-
be/src/http/action/warmup_stats_action.cpp | 121 +++
be/src/http/action/warmup_stats_action.h | 37 +
be/src/io/cache/block_file_cache.cpp | 116 ++-
be/src/io/cache/block_file_cache.h | 11 +-
be/src/io/cache/block_file_cache_profile.cpp | 37 +
be/src/io/cache/block_file_cache_profile.h | 7 +-
be/src/io/cache/cached_remote_file_reader.cpp | 82 +-
be/src/io/cache/file_cache_common.h | 3 +
be/src/io/cache/lru_queue_recorder.cpp | 68 +-
be/src/io/cache/lru_queue_recorder.h | 11 +-
be/src/olap/base_tablet.cpp | 13 +-
be/src/olap/compaction.cpp | 13 +-
be/src/olap/rowset/beta_rowset_writer.cpp | 9 +-
be/src/olap/rowset/rowset.h | 4 +
be/src/olap/rowset/rowset_meta.cpp | 33 +-
be/src/olap/rowset/rowset_meta.h | 16 +-
be/src/olap/snapshot_manager.cpp | 16 +-
be/src/olap/snapshot_manager.h | 2 +-
be/src/olap/storage_engine.cpp | 1 +
be/src/olap/task/index_builder.cpp | 5 +-
be/src/service/http_service.cpp | 5 +
be/src/util/bvar_windowed_adder.h | 162 ++++
be/src/util/s3_util.cpp | 3 +-
.../cloud/cloud_warm_up_manager_filter_test.cpp | 278 ++++++
be/test/cloud/cloud_warm_up_manager_test.cpp | 6 +-
.../block_file_cache_profile_reporter_test.cpp | 139 +++
be/test/io/cache/block_file_cache_test.cpp | 364 ++++++-
.../io/cache/block_file_cache_test_lru_dump.cpp | 79 +-
be/test/io/cache/cache_lru_dumper_test.cpp | 51 +-
be/test/olap/rowset/rowset_meta_test.cpp | 152 +++
.../olap/segments_key_bounds_truncation_test.cpp | 7 +
be/test/service/http/file_cache_action_test.cpp | 192 ++++
be/test/util/bvar_windowed_adder_test.cpp | 140 +++
cloud/src/common/config.h | 6 +-
cloud/src/common/configbase.cpp | 22 +-
cloud/src/common/configbase.h | 3 +
cloud/src/meta-service/meta_service.cpp | 125 ++-
cloud/src/meta-service/meta_service_job.cpp | 90 +-
cloud/src/recycler/recycler.cpp | 26 +-
cloud/src/recycler/recycler.h | 2 -
cloud/src/recycler/s3_accessor.cpp | 8 +-
cloud/test/meta_service_job_test.cpp | 215 +++++
cloud/test/meta_service_test.cpp | 87 +-
cloud/test/recycler_test.cpp | 61 ++
common/cpp/obj_retry_strategy.cpp | 10 +-
common/cpp/obj_retry_strategy.h | 7 +-
docker/runtime/doris-compose/Dockerfile | 1 +
.../main/java/org/apache/doris/common/Config.java | 39 +
.../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 2 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 13 +-
.../apache/doris/cloud/CacheHotspotManager.java | 492 +++++++++-
.../org/apache/doris/cloud/CloudWarmUpJob.java | 260 ++++-
.../org/apache/doris/cloud/JobWarmUpStats.java | 285 ++++++
.../org/apache/doris/cloud/OnTablesFilter.java | 169 ++++
.../doris/cloud/TableWarmUpWindowedStats.java | 206 ++++
.../doris/cloud/catalog/CloudClusterChecker.java | 13 +-
.../cloud/catalog/CloudInstanceStatusChecker.java | 9 +-
.../apache/doris/cloud/catalog/CloudReplica.java | 125 +--
.../apache/doris/cloud/catalog/CloudTablet.java | 15 +-
.../doris/cloud/system/CloudSystemInfoService.java | 93 +-
.../config/WebServerFactoryCustomizerConfig.java | 3 +
.../org/apache/doris/httpv2/rest/LoadAction.java | 156 +--
.../doris/httpv2/rest/RestBaseController.java | 51 +-
.../httpv2/util/StreamLoadRedirectDrainUtil.java | 154 +++
.../java/org/apache/doris/metric/MetricRepo.java | 203 ++++
.../doris/nereids/parser/LogicalPlanBuilder.java | 15 +-
.../nereids/rules/analysis/SubqueryToApply.java | 25 +-
.../trees/plans/commands/ShowWarmUpCommand.java | 3 +
.../trees/plans/commands/WarmUpClusterCommand.java | 57 +-
.../org/apache/doris/planner/OlapScanNode.java | 26 +-
.../org/apache/doris/planner/OlapTableSink.java | 17 +-
.../apache/doris/service/FrontendServiceImpl.java | 35 +-
.../cloud/CacheHotspotManagerTableFilterTest.java | 1003 ++++++++++++++++++++
.../doris/cloud/CloudWarmUpJobTableFilterTest.java | 461 +++++++++
.../org/apache/doris/cloud/OnTablesFilterTest.java | 141 +++
.../cloud/WarmUpClusterOnTablesParseTest.java | 447 +++++++++
.../org/apache/doris/cloud/WarmUpStatsTest.java | 497 ++++++++++
.../catalog/CloudInstanceStatusCheckerTest.java | 260 +++++
.../cloud/system/CloudSystemInfoServiceTest.java | 15 +
.../apache/doris/httpv2/rest/LoadActionTest.java | 445 ++++++++-
.../doris/httpv2/rest/RestBaseControllerTest.java | 64 ++
.../util/StreamLoadRedirectDrainUtilTest.java | 394 ++++++++
.../java/org/apache/doris/metric/MetricsTest.java | 161 ++++
gensrc/proto/internal_service.proto | 2 +
gensrc/proto/olap_file.proto | 10 +
gensrc/thrift/BackendService.thrift | 1 +
.../regression/util/WarmupMetricsUtils.groovy | 268 ++++++
...est_balance_use_compute_group_properties.groovy | 1 +
.../cloud_p0/balance/test_balance_warm_up.groovy | 1 +
.../test_balance_warm_up_sync_global_config.groovy | 1 +
.../test_balance_warm_up_task_abnormal.groovy | 1 +
.../test_balance_warm_up_use_peer_cache.groovy | 1 +
.../balance/test_peer_read_async_warmup.groovy | 1 +
...on_tables_abnormal_cancel_empty_recovery.groovy | 212 +++++
...ent_on_tables_abnormal_stats_and_failure.groovy | 261 +++++
...warm_up_event_on_tables_canonicalization.groovy | 117 +++
.../test_warm_up_event_on_tables_dynamic.groovy | 217 +++++
...m_up_event_on_tables_error_and_lifecycle.groovy | 387 ++++++++
.../test_warm_up_event_on_tables_include.groovy | 167 ++++
..._warm_up_event_on_tables_include_exclude.groovy | 153 +++
...t_warm_up_event_on_tables_mow_compaction.groovy | 221 +++++
.../test_warm_up_event_on_tables_multi_dst.groovy | 213 +++++
...st_warm_up_event_on_tables_multi_include.groovy | 142 +++
...t_warm_up_event_on_tables_overlap_and_mv.groovy | 332 +++++++
..._warm_up_event_on_tables_show_and_cancel.groovy | 384 ++++++++
.../test_warm_up_event_on_tables_sync_stats.groovy | 300 ++++++
...up_event_on_tables_system_cluster_change.groovy | 168 ++++
...nt_on_tables_system_compaction_sync_wait.groovy | 214 +++++
...m_up_event_on_tables_system_e2e_multi_be.groovy | 269 ++++++
...rm_up_event_on_tables_system_packed_file.groovy | 155 +++
...vent_on_tables_system_restart_and_resize.groovy | 223 +++++
...m_up_event_on_tables_system_schema_index.groovy | 182 ++++
...st_file_cache_warmup_read_metrics_docker.groovy | 457 +++++++++
.../test_non_mow_key_bounds_aggregation.groovy | 177 ++++
.../test_dml_when_one_be_down.groovy | 141 +++
.../scripts/stream_load_redirect_chunked_e2e.py | 140 +++
...test_stream_load_fe_redirect_chunked_e2e.groovy | 117 +++
.../flexible/test_f_no_simdjson.groovy | 166 ----
135 files changed, 15080 insertions(+), 799 deletions(-)
create mode 100644 be/src/cloud/cloud_warmup_metrics.cpp
create mode 100644 be/src/cloud/cloud_warmup_metrics.h
create mode 100644 be/src/http/action/warmup_stats_action.cpp
create mode 100644 be/src/http/action/warmup_stats_action.h
create mode 100644 be/src/util/bvar_windowed_adder.h
create mode 100644 be/test/cloud/cloud_warm_up_manager_filter_test.cpp
create mode 100644 be/test/io/cache/block_file_cache_profile_reporter_test.cpp
create mode 100644 be/test/service/http/file_cache_action_test.cpp
create mode 100644 be/test/util/bvar_windowed_adder_test.cpp
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/cloud/JobWarmUpStats.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/cloud/OnTablesFilter.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/cloud/TableWarmUpWindowedStats.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/httpv2/util/StreamLoadRedirectDrainUtil.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/cloud/CacheHotspotManagerTableFilterTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/cloud/CloudWarmUpJobTableFilterTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/cloud/OnTablesFilterTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/cloud/WarmUpClusterOnTablesParseTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/cloud/WarmUpStatsTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/CloudInstanceStatusCheckerTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/httpv2/rest/RestBaseControllerTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/httpv2/util/StreamLoadRedirectDrainUtilTest.java
create mode 100644
regression-test/framework/src/main/groovy/org/apache/doris/regression/util/WarmupMetricsUtils.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/on_tables/test_warm_up_event_on_tables_abnormal_cancel_empty_recovery.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/on_tables/test_warm_up_event_on_tables_abnormal_stats_and_failure.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/on_tables/test_warm_up_event_on_tables_canonicalization.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/on_tables/test_warm_up_event_on_tables_dynamic.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/on_tables/test_warm_up_event_on_tables_error_and_lifecycle.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/on_tables/test_warm_up_event_on_tables_include.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/on_tables/test_warm_up_event_on_tables_include_exclude.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/on_tables/test_warm_up_event_on_tables_mow_compaction.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/on_tables/test_warm_up_event_on_tables_multi_dst.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/on_tables/test_warm_up_event_on_tables_multi_include.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/on_tables/test_warm_up_event_on_tables_overlap_and_mv.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/on_tables/test_warm_up_event_on_tables_show_and_cancel.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/on_tables/test_warm_up_event_on_tables_sync_stats.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/on_tables/test_warm_up_event_on_tables_system_cluster_change.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/on_tables/test_warm_up_event_on_tables_system_compaction_sync_wait.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/on_tables/test_warm_up_event_on_tables_system_e2e_multi_be.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/on_tables/test_warm_up_event_on_tables_system_packed_file.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/on_tables/test_warm_up_event_on_tables_system_restart_and_resize.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/on_tables/test_warm_up_event_on_tables_system_schema_index.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/warm_up/test_file_cache_warmup_read_metrics_docker.groovy
create mode 100644
regression-test/suites/data_model_p0/duplicate/test_non_mow_key_bounds_aggregation.groovy
create mode 100644
regression-test/suites/fault_injection_p0/test_dml_when_one_be_down.groovy
create mode 100644
regression-test/suites/load_p0/stream_load/scripts/stream_load_redirect_chunked_e2e.py
create mode 100644
regression-test/suites/load_p0/stream_load/test_stream_load_fe_redirect_chunked_e2e.groovy
delete mode 100644
regression-test/suites/unique_with_mow_p0/flexible/test_f_no_simdjson.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
