This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 6acf137cd25 [enhancement](time_series) increase the version limit for
the time series table (#51371) (#51656)
6acf137cd25 is described below
commit 6acf137cd257e43cf4ddf74cca79e481c3f6f20f
Author: Luwei <[email protected]>
AuthorDate: Tue Jun 17 14:19:49 2025 +0800
[enhancement](time_series) increase the version limit for the time series
table (#51371) (#51656)
pick #51371
---
be/src/agent/task_worker_pool.cpp | 3 ++-
be/src/cloud/cloud_delete_task.cpp | 11 +++++++----
be/src/cloud/cloud_rowset_builder.cpp | 8 +++++---
be/src/cloud/cloud_tablet_mgr.cpp | 4 +++-
be/src/common/config.cpp | 2 ++
be/src/common/config.h | 2 ++
be/src/olap/base_tablet.cpp | 8 ++++++++
be/src/olap/base_tablet.h | 2 ++
be/src/olap/olap_server.cpp | 3 ++-
be/src/olap/push_handler.cpp | 9 ++++++---
be/src/olap/rowset_builder.cpp | 10 ++++++----
11 files changed, 45 insertions(+), 17 deletions(-)
diff --git a/be/src/agent/task_worker_pool.cpp
b/be/src/agent/task_worker_pool.cpp
index 5fe8da388b1..7eb7716e413 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -1893,8 +1893,9 @@ void
PublishVersionWorkerPool::publish_version_callback(const TAgentTaskRequest&
if
(!tablet->tablet_meta()->tablet_schema()->disable_auto_compaction()) {
tablet->published_count.fetch_add(1);
int64_t published_count =
tablet->published_count.load();
+ int32_t max_version_config =
tablet->max_version_config();
if (tablet->exceed_version_limit(
- config::max_tablet_version_num *
+ max_version_config *
config::load_trigger_compaction_version_percent / 100) &&
published_count % 20 == 0) {
auto st = _engine.submit_compaction_task(
diff --git a/be/src/cloud/cloud_delete_task.cpp
b/be/src/cloud/cloud_delete_task.cpp
index 35c48841d38..5698fb632cd 100644
--- a/be/src/cloud/cloud_delete_task.cpp
+++ b/be/src/cloud/cloud_delete_task.cpp
@@ -48,14 +48,17 @@ Status CloudDeleteTask::execute(CloudStorageEngine& engine,
const TPushReq& requ
tablet->last_load_time_ms =
duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count();
// check if version number exceed limit
- if (tablet->fetch_add_approximate_num_rowsets(0) >
config::max_tablet_version_num) {
+
+ int32_t max_version_config = tablet->max_version_config();
+ if (tablet->fetch_add_approximate_num_rowsets(0) > max_version_config) {
LOG_WARNING("tablet exceeds max version num limit")
- .tag("limit", config::max_tablet_version_num)
+ .tag("limit", max_version_config)
.tag("tablet_id", tablet->tablet_id());
return Status::Error<TOO_MANY_VERSION>(
"too many versions, versions={} tablet={}. Please reduce the
frequency of loading "
- "data or adjust the max_tablet_version_num in be.conf to a
larger value.",
- config::max_tablet_version_num, tablet->tablet_id());
+ "data or adjust the max_tablet_version_num or
time_series_max_tablet_version_num "
+ "in be.conf to a larger value.",
+ max_version_config, tablet->tablet_id());
}
// check delete condition if push for delete
diff --git a/be/src/cloud/cloud_rowset_builder.cpp
b/be/src/cloud/cloud_rowset_builder.cpp
index 9466dd10628..8715d90340c 100644
--- a/be/src/cloud/cloud_rowset_builder.cpp
+++ b/be/src/cloud/cloud_rowset_builder.cpp
@@ -89,12 +89,14 @@ Status CloudRowsetBuilder::init() {
Status CloudRowsetBuilder::check_tablet_version_count() {
int version_count = cloud_tablet()->fetch_add_approximate_num_rowsets(0);
// TODO(plat1ko): load backoff algorithm
- if (version_count > config::max_tablet_version_num) {
+ int32_t max_version_config = cloud_tablet()->max_version_config();
+ if (version_count > max_version_config) {
return Status::Error<TOO_MANY_VERSION>(
"failed to init rowset builder. version count: {}, exceed
limit: {}, "
"tablet: {}. Please reduce the frequency of loading data or
adjust the "
- "max_tablet_version_num in be.conf to a larger value.",
- version_count, config::max_tablet_version_num,
_tablet->tablet_id());
+ "max_tablet_version_num or
time_series_max_tablet_version_numin be.conf to a "
+ "larger value.",
+ version_count, max_version_config, _tablet->tablet_id());
}
return Status::OK();
}
diff --git a/be/src/cloud/cloud_tablet_mgr.cpp
b/be/src/cloud/cloud_tablet_mgr.cpp
index 17d1b98a95f..e8d06f72577 100644
--- a/be/src/cloud/cloud_tablet_mgr.cpp
+++ b/be/src/cloud/cloud_tablet_mgr.cpp
@@ -343,11 +343,13 @@ Status CloudTabletMgr::get_topn_tablets_to_compact(
}
// If tablet has too many rowsets but not be compacted for a long
time, compaction should be performed
// regardless of whether there is a load job recently.
+
+ int32_t max_version_config = t->max_version_config();
return now - t->last_cumu_compaction_failure_time() <
config::min_compaction_failure_interval_ms ||
now - t->last_cumu_no_suitable_version_ms <
config::min_compaction_failure_interval_ms ||
(now - t->last_load_time_ms >
config::cu_compaction_freeze_interval_s * 1000
&& now - t->last_cumu_compaction_success_time_ms <
config::cumu_compaction_interval_s * 1000
- && t->fetch_add_approximate_num_rowsets(0) <
config::max_tablet_version_num / 2);
+ && t->fetch_add_approximate_num_rowsets(0) < max_version_config
/ 2);
};
// We don't schedule tablets that are disabled for compaction
auto disable = [](CloudTablet* t) { return
t->tablet_meta()->tablet_schema()->disable_auto_compaction(); };
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 04e04aa4b4b..02cd267cc24 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -824,6 +824,8 @@ DEFINE_Int32(query_cache_max_partition_count, "1024");
// This is to avoid too many version num.
DEFINE_mInt32(max_tablet_version_num, "2000");
+DEFINE_mInt32(time_series_max_tablet_version_num, "20000");
+
// Frontend mainly use two thrift sever type: THREAD_POOL, THREADED_SELECTOR.
if fe use THREADED_SELECTOR model for thrift server,
// the thrift_server_type_of_fe should be set THREADED_SELECTOR to make be
thrift client to fe constructed with TFramedTransport
DEFINE_String(thrift_server_type_of_fe, "THREAD_POOL");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 51a275a55b8..b60e10fe49d 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -868,6 +868,8 @@ DECLARE_Int32(query_cache_max_partition_count);
// This is to avoid too many version num.
DECLARE_mInt32(max_tablet_version_num);
+DECLARE_mInt32(time_series_max_tablet_version_num);
+
// Frontend mainly use two thrift sever type: THREAD_POOL, THREADED_SELECTOR.
if fe use THREADED_SELECTOR model for thrift server,
// the thrift_server_type_of_fe should be set THREADED_SELECTOR to make be
thrift client to fe constructed with TFramedTransport
DECLARE_String(thrift_server_type_of_fe);
diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp
index 8c8e52be30f..1aa74582d84 100644
--- a/be/src/olap/base_tablet.cpp
+++ b/be/src/olap/base_tablet.cpp
@@ -29,6 +29,7 @@
#include "common/logging.h"
#include "common/status.h"
#include "olap/calc_delete_bitmap_executor.h"
+#include "olap/cumulative_compaction_time_series_policy.h"
#include "olap/delete_bitmap_calculator.h"
#include "olap/iterators.h"
#include "olap/memtable.h"
@@ -1736,4 +1737,11 @@ void BaseTablet::get_base_rowset_delete_bitmap_count(
}
}
+int32_t BaseTablet::max_version_config() {
+ int32_t max_version = tablet_meta()->compaction_policy() ==
CUMULATIVE_TIME_SERIES_POLICY
+ ? config::time_series_max_tablet_version_num
+ : config::max_tablet_version_num;
+ return max_version;
+}
+
} // namespace doris
diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h
index 4e9ddaac1b5..4df16de7eb3 100644
--- a/be/src/olap/base_tablet.h
+++ b/be/src/olap/base_tablet.h
@@ -79,6 +79,8 @@ public:
// Property encapsulated in TabletMeta
const TabletMetaSharedPtr& tablet_meta() { return _tablet_meta; }
+ int32 max_version_config();
+
// FIXME(plat1ko): It is not appropriate to expose this lock
std::shared_mutex& get_header_lock() { return _meta_lock; }
diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index 8469503b8ed..df201c7d699 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -1700,8 +1700,9 @@ void StorageEngine::_process_async_publish() {
continue;
}
if (version != max_version + 1) {
+ int32_t max_version_config = tablet->max_version_config();
// Keep only the most recent versions
- while (tablet_iter->second.size() >
config::max_tablet_version_num) {
+ while (tablet_iter->second.size() > max_version_config) {
need_removed_tasks.emplace_back(tablet, version);
task_iter = tablet_iter->second.erase(task_iter);
version = task_iter->first;
diff --git a/be/src/olap/push_handler.cpp b/be/src/olap/push_handler.cpp
index 575b002b2f6..8cfe6a1a7f9 100644
--- a/be/src/olap/push_handler.cpp
+++ b/be/src/olap/push_handler.cpp
@@ -39,6 +39,7 @@
#include "common/config.h"
#include "common/logging.h"
#include "common/status.h"
+#include "olap/cumulative_compaction_time_series_policy.h"
#include "olap/delete_handler.h"
#include "olap/olap_define.h"
#include "olap/rowset/pending_rowset_helper.h"
@@ -160,13 +161,15 @@ Status
PushHandler::_do_streaming_ingestion(TabletSharedPtr tablet, const TPushR
}
}
+ int32_t max_version_config = tablet->max_version_config();
// check if version number exceed limit
- if (tablet->exceed_version_limit(config::max_tablet_version_num)) {
+ if (tablet->exceed_version_limit(max_version_config)) {
return Status::Status::Error<TOO_MANY_VERSION>(
"failed to push data. version count: {}, exceed limit: {},
tablet: {}. Please "
- "reduce the frequency of loading data or adjust the
max_tablet_version_num in "
+ "reduce the frequency of loading data or adjust the
max_tablet_version_num or "
+ "time_series_max_tablet_version_num in "
"be.conf to a larger value.",
- tablet->version_count(), config::max_tablet_version_num,
tablet->tablet_id());
+ tablet->version_count(), max_version_config,
tablet->tablet_id());
}
int version_count = tablet->version_count() +
tablet->stale_version_count();
diff --git a/be/src/olap/rowset_builder.cpp b/be/src/olap/rowset_builder.cpp
index 97e140748cf..e9b518eaae0 100644
--- a/be/src/olap/rowset_builder.cpp
+++ b/be/src/olap/rowset_builder.cpp
@@ -151,9 +151,10 @@ Status RowsetBuilder::check_tablet_version_count() {
bool injection = false;
DBUG_EXECUTE_IF("RowsetBuilder.check_tablet_version_count.too_many_version",
{ injection = true; });
+ int32_t max_version_config = _tablet->max_version_config();
if (injection) {
// do not return if injection
- } else if (!_tablet->exceed_version_limit(config::max_tablet_version_num -
100) ||
+ } else if (!_tablet->exceed_version_limit(max_version_config - 100) ||
GlobalMemoryArbitrator::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) {
return Status::OK();
}
@@ -167,12 +168,13 @@ Status RowsetBuilder::check_tablet_version_count() {
int version_count = tablet()->version_count();
DBUG_EXECUTE_IF("RowsetBuilder.check_tablet_version_count.too_many_version",
{ version_count = INT_MAX; });
- if (version_count > config::max_tablet_version_num) {
+ if (version_count > max_version_config) {
return Status::Error<TOO_MANY_VERSION>(
"failed to init rowset builder. version count: {}, exceed
limit: {}, "
"tablet: {}. Please reduce the frequency of loading data or
adjust the "
- "max_tablet_version_num in be.conf to a larger value.",
- version_count, config::max_tablet_version_num,
_tablet->tablet_id());
+ "max_tablet_version_num or time_series_max_tablet_version_num
in be.conf to a "
+ "larger value.",
+ version_count, max_version_config, _tablet->tablet_id());
}
return Status::OK();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
