This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 96fc8dfcff8 [enhancement](time_series) increase the version limit for
the time series table (#51371)
96fc8dfcff8 is described below
commit 96fc8dfcff856d5ae968d99a95b3094998862a1a
Author: Luwei <lu...@selectdb.com>
AuthorDate: Sun Jun 1 09:17:11 2025 +0800
[enhancement](time_series) increase the version limit for the time series
table (#51371)
---
be/src/agent/task_worker_pool.cpp | 3 ++-
be/src/cloud/cloud_delete_task.cpp | 11 +++++++----
be/src/cloud/cloud_rowset_builder.cpp | 8 +++++---
be/src/cloud/cloud_tablet_mgr.cpp | 4 +++-
be/src/common/config.cpp | 2 ++
be/src/common/config.h | 2 ++
be/src/olap/base_tablet.cpp | 8 ++++++++
be/src/olap/base_tablet.h | 2 ++
be/src/olap/olap_server.cpp | 3 ++-
be/src/olap/push_handler.cpp | 9 ++++++---
be/src/olap/rowset_builder.cpp | 10 ++++++----
11 files changed, 45 insertions(+), 17 deletions(-)
diff --git a/be/src/agent/task_worker_pool.cpp
b/be/src/agent/task_worker_pool.cpp
index 4ee3f5b09a3..53b9b85ee1c 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -1884,8 +1884,9 @@ void
PublishVersionWorkerPool::publish_version_callback(const TAgentTaskRequest&
if
(!tablet->tablet_meta()->tablet_schema()->disable_auto_compaction()) {
tablet->published_count.fetch_add(1);
int64_t published_count =
tablet->published_count.load();
+ int32_t max_version_config =
tablet->max_version_config();
if (tablet->exceed_version_limit(
- config::max_tablet_version_num *
+ max_version_config *
config::load_trigger_compaction_version_percent / 100) &&
published_count % 20 == 0) {
auto st = _engine.submit_compaction_task(
diff --git a/be/src/cloud/cloud_delete_task.cpp
b/be/src/cloud/cloud_delete_task.cpp
index 35c48841d38..5698fb632cd 100644
--- a/be/src/cloud/cloud_delete_task.cpp
+++ b/be/src/cloud/cloud_delete_task.cpp
@@ -48,14 +48,17 @@ Status CloudDeleteTask::execute(CloudStorageEngine& engine,
const TPushReq& requ
tablet->last_load_time_ms =
duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count();
// check if version number exceed limit
- if (tablet->fetch_add_approximate_num_rowsets(0) >
config::max_tablet_version_num) {
+
+ int32_t max_version_config = tablet->max_version_config();
+ if (tablet->fetch_add_approximate_num_rowsets(0) > max_version_config) {
LOG_WARNING("tablet exceeds max version num limit")
- .tag("limit", config::max_tablet_version_num)
+ .tag("limit", max_version_config)
.tag("tablet_id", tablet->tablet_id());
return Status::Error<TOO_MANY_VERSION>(
"too many versions, versions={} tablet={}. Please reduce the
frequency of loading "
- "data or adjust the max_tablet_version_num in be.conf to a
larger value.",
- config::max_tablet_version_num, tablet->tablet_id());
+ "data or adjust the max_tablet_version_num or
time_series_max_tablet_version_num "
+ "in be.conf to a larger value.",
+ max_version_config, tablet->tablet_id());
}
// check delete condition if push for delete
diff --git a/be/src/cloud/cloud_rowset_builder.cpp
b/be/src/cloud/cloud_rowset_builder.cpp
index e9a5c3b879d..389b6c7c682 100644
--- a/be/src/cloud/cloud_rowset_builder.cpp
+++ b/be/src/cloud/cloud_rowset_builder.cpp
@@ -90,12 +90,14 @@ Status CloudRowsetBuilder::init() {
Status CloudRowsetBuilder::check_tablet_version_count() {
int64_t version_count =
cloud_tablet()->fetch_add_approximate_num_rowsets(0);
// TODO(plat1ko): load backoff algorithm
- if (version_count > config::max_tablet_version_num) {
+ int32_t max_version_config = cloud_tablet()->max_version_config();
+ if (version_count > max_version_config) {
return Status::Error<TOO_MANY_VERSION>(
"failed to init rowset builder. version count: {}, exceed
limit: {}, "
"tablet: {}. Please reduce the frequency of loading data or
adjust the "
- "max_tablet_version_num in be.conf to a larger value.",
- version_count, config::max_tablet_version_num,
_tablet->tablet_id());
+ "max_tablet_version_num or
time_series_max_tablet_version_numin be.conf to a "
+ "larger value.",
+ version_count, max_version_config, _tablet->tablet_id());
}
return Status::OK();
}
diff --git a/be/src/cloud/cloud_tablet_mgr.cpp
b/be/src/cloud/cloud_tablet_mgr.cpp
index 6334506113a..6c3a671f184 100644
--- a/be/src/cloud/cloud_tablet_mgr.cpp
+++ b/be/src/cloud/cloud_tablet_mgr.cpp
@@ -393,11 +393,13 @@ Status CloudTabletMgr::get_topn_tablets_to_compact(
}
// If tablet has too many rowsets but not be compacted for a long
time, compaction should be performed
// regardless of whether there is a load job recently.
+
+ int32_t max_version_config = t->max_version_config();
return now - t->last_cumu_compaction_failure_time() <
config::min_compaction_failure_interval_ms ||
now - t->last_cumu_no_suitable_version_ms <
config::min_compaction_failure_interval_ms ||
(now - t->last_load_time_ms >
config::cu_compaction_freeze_interval_s * 1000
&& now - t->last_cumu_compaction_success_time_ms <
config::cumu_compaction_interval_s * 1000
- && t->fetch_add_approximate_num_rowsets(0) <
config::max_tablet_version_num / 2);
+ && t->fetch_add_approximate_num_rowsets(0) < max_version_config
/ 2);
};
// We don't schedule tablets that are disabled for compaction
auto disable = [](CloudTablet* t) { return
t->tablet_meta()->tablet_schema()->disable_auto_compaction(); };
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index cd6514f7056..43936fc4e76 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -816,6 +816,8 @@ DEFINE_Int32(query_cache_max_partition_count, "1024");
// This is to avoid too many version num.
DEFINE_mInt32(max_tablet_version_num, "2000");
+DEFINE_mInt32(time_series_max_tablet_version_num, "20000");
+
// Frontend mainly use two thrift sever type: THREAD_POOL, THREADED_SELECTOR.
if fe use THREADED_SELECTOR model for thrift server,
// the thrift_server_type_of_fe should be set THREADED_SELECTOR to make be
thrift client to fe constructed with TFramedTransport
DEFINE_String(thrift_server_type_of_fe, "THREAD_POOL");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 3351716f768..1c1c64ea811 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -858,6 +858,8 @@ DECLARE_Int32(query_cache_max_partition_count);
// This is to avoid too many version num.
DECLARE_mInt32(max_tablet_version_num);
+DECLARE_mInt32(time_series_max_tablet_version_num);
+
// Frontend mainly use two thrift sever type: THREAD_POOL, THREADED_SELECTOR.
if fe use THREADED_SELECTOR model for thrift server,
// the thrift_server_type_of_fe should be set THREADED_SELECTOR to make be
thrift client to fe constructed with TFramedTransport
DECLARE_String(thrift_server_type_of_fe);
diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp
index 2a422ff6091..d28713cf8fc 100644
--- a/be/src/olap/base_tablet.cpp
+++ b/be/src/olap/base_tablet.cpp
@@ -32,6 +32,7 @@
#include "common/logging.h"
#include "common/status.h"
#include "olap/calc_delete_bitmap_executor.h"
+#include "olap/cumulative_compaction_time_series_policy.h"
#include "olap/delete_bitmap_calculator.h"
#include "olap/iterators.h"
#include "olap/memtable.h"
@@ -2096,4 +2097,11 @@ void BaseTablet::get_base_rowset_delete_bitmap_count(
}
}
+int32_t BaseTablet::max_version_config() {
+ int32_t max_version = tablet_meta()->compaction_policy() ==
CUMULATIVE_TIME_SERIES_POLICY
+ ? config::time_series_max_tablet_version_num
+ : config::max_tablet_version_num;
+ return max_version;
+}
+
} // namespace doris
diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h
index 2f49db4f1d4..f0a97b3f094 100644
--- a/be/src/olap/base_tablet.h
+++ b/be/src/olap/base_tablet.h
@@ -82,6 +82,8 @@ public:
// Property encapsulated in TabletMeta
const TabletMetaSharedPtr& tablet_meta() { return _tablet_meta; }
+ int32 max_version_config();
+
// FIXME(plat1ko): It is not appropriate to expose this lock
std::shared_mutex& get_header_lock() { return _meta_lock; }
diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index 37ae5a51cf4..a8f869ff2e3 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -1687,8 +1687,9 @@ void StorageEngine::_process_async_publish() {
continue;
}
if (version != max_version + 1) {
+ int32_t max_version_config = tablet->max_version_config();
// Keep only the most recent versions
- while (tablet_iter->second.size() >
config::max_tablet_version_num) {
+ while (tablet_iter->second.size() > max_version_config) {
need_removed_tasks.emplace_back(tablet, version);
task_iter = tablet_iter->second.erase(task_iter);
version = task_iter->first;
diff --git a/be/src/olap/push_handler.cpp b/be/src/olap/push_handler.cpp
index 57a2d835d26..08b6bac8312 100644
--- a/be/src/olap/push_handler.cpp
+++ b/be/src/olap/push_handler.cpp
@@ -41,6 +41,7 @@
#include "common/logging.h"
#include "common/status.h"
#include "io/hdfs_builder.h"
+#include "olap/cumulative_compaction_time_series_policy.h"
#include "olap/delete_handler.h"
#include "olap/olap_define.h"
#include "olap/rowset/pending_rowset_helper.h"
@@ -165,13 +166,15 @@ Status
PushHandler::_do_streaming_ingestion(TabletSharedPtr tablet, const TPushR
}
}
+ int32_t max_version_config = tablet->max_version_config();
// check if version number exceed limit
- if (tablet->exceed_version_limit(config::max_tablet_version_num)) {
+ if (tablet->exceed_version_limit(max_version_config)) {
return Status::Status::Error<TOO_MANY_VERSION>(
"failed to push data. version count: {}, exceed limit: {},
tablet: {}. Please "
- "reduce the frequency of loading data or adjust the
max_tablet_version_num in "
+ "reduce the frequency of loading data or adjust the
max_tablet_version_num or "
+ "time_series_max_tablet_version_num in "
"be.conf to a larger value.",
- tablet->version_count(), config::max_tablet_version_num,
tablet->tablet_id());
+ tablet->version_count(), max_version_config,
tablet->tablet_id());
}
int version_count = tablet->version_count() +
tablet->stale_version_count();
diff --git a/be/src/olap/rowset_builder.cpp b/be/src/olap/rowset_builder.cpp
index 0839217b172..1f63893730d 100644
--- a/be/src/olap/rowset_builder.cpp
+++ b/be/src/olap/rowset_builder.cpp
@@ -150,9 +150,10 @@ Status RowsetBuilder::check_tablet_version_count() {
bool injection = false;
DBUG_EXECUTE_IF("RowsetBuilder.check_tablet_version_count.too_many_version",
{ injection = true; });
+ int32_t max_version_config = _tablet->max_version_config();
if (injection) {
// do not return if injection
- } else if (!_tablet->exceed_version_limit(config::max_tablet_version_num -
100) ||
+ } else if (!_tablet->exceed_version_limit(max_version_config - 100) ||
GlobalMemoryArbitrator::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) {
return Status::OK();
}
@@ -166,12 +167,13 @@ Status RowsetBuilder::check_tablet_version_count() {
int version_count = tablet()->version_count();
DBUG_EXECUTE_IF("RowsetBuilder.check_tablet_version_count.too_many_version",
{ version_count = INT_MAX; });
- if (version_count > config::max_tablet_version_num) {
+ if (version_count > max_version_config) {
return Status::Error<TOO_MANY_VERSION>(
"failed to init rowset builder. version count: {}, exceed
limit: {}, "
"tablet: {}. Please reduce the frequency of loading data or
adjust the "
- "max_tablet_version_num in be.conf to a larger value.",
- version_count, config::max_tablet_version_num,
_tablet->tablet_id());
+ "max_tablet_version_num or time_series_max_tablet_version_num
in be.conf to a "
+ "larger value.",
+ version_count, max_version_config, _tablet->tablet_id());
}
return Status::OK();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
