This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch variant-sparse
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 4f1dade7bfbab79c9e9649b5b4fc9879ef560d1d
Author: Sun Chenyang <suncheny...@selectdb.com>
AuthorDate: Mon Mar 3 15:24:26 2025 +0800
[enhance](variant) support for down grade (#48518)
### What problem does this PR solve?
Issue Number: close #xxx
Related PR: #xxx
Problem Summary:
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/src/agent/be_exec_version_manager.cpp | 2 +-
be/src/agent/be_exec_version_manager.h | 3 ++-
be/src/vec/data_types/data_type_object.cpp | 24 ++++++++++++++--------
.../main/java/org/apache/doris/common/Config.java | 2 +-
4 files changed, 20 insertions(+), 11 deletions(-)
diff --git a/be/src/agent/be_exec_version_manager.cpp
b/be/src/agent/be_exec_version_manager.cpp
index bfd0745e316..9146bd21479 100644
--- a/be/src/agent/be_exec_version_manager.cpp
+++ b/be/src/agent/be_exec_version_manager.cpp
@@ -113,7 +113,7 @@ void BeExecVersionManager::check_function_compatibility(int
current_be_exec_vers
* b. support const column in serialize/deserialize function: PR #41175
*/
-const int BeExecVersionManager::max_be_exec_version = 8;
+const int BeExecVersionManager::max_be_exec_version = 9;
const int BeExecVersionManager::min_be_exec_version = 0;
std::map<std::string, std::set<int>>
BeExecVersionManager::_function_change_map {};
std::set<std::string> BeExecVersionManager::_function_restrict_map;
diff --git a/be/src/agent/be_exec_version_manager.h
b/be/src/agent/be_exec_version_manager.h
index f4158a40152..14779bd0ce9 100644
--- a/be/src/agent/be_exec_version_manager.h
+++ b/be/src/agent/be_exec_version_manager.h
@@ -34,7 +34,8 @@ constexpr inline int AGGREGATION_2_1_VERSION =
6; // some aggregation changed the data format after this version
constexpr inline int USE_CONST_SERDE =
8; // support const column in serialize/deserialize function: PR #41175
-
+constexpr inline int VARIANT_SPARSE_SERDE =
+ 9; // support variant for serializing/deserializing sparse column
class BeExecVersionManager {
public:
BeExecVersionManager() = delete;
diff --git a/be/src/vec/data_types/data_type_object.cpp
b/be/src/vec/data_types/data_type_object.cpp
index fef6b2cbcef..5bcb9e6985b 100644
--- a/be/src/vec/data_types/data_type_object.cpp
+++ b/be/src/vec/data_types/data_type_object.cpp
@@ -94,8 +94,10 @@ int64_t
DataTypeObject::get_uncompressed_serialized_bytes(const IColumn& column,
// sparse column
// TODO make compability with sparse column
- size +=
ColumnObject::get_sparse_column_type()->get_uncompressed_serialized_bytes(
- *column_object.get_sparse_column(), be_exec_version);
+ if (be_exec_version >= VARIANT_SPARSE_SERDE) {
+ size +=
ColumnObject::get_sparse_column_type()->get_uncompressed_serialized_bytes(
+ *column_object.get_sparse_column(), be_exec_version);
+ }
return size;
}
@@ -149,8 +151,10 @@ char* DataTypeObject::serialize(const IColumn& column,
char* buf, int be_exec_ve
// serialize sparse column
// TODO make compability with sparse column
- buf =
ColumnObject::get_sparse_column_type()->serialize(*column_object.get_sparse_column(),
buf,
- be_exec_version);
+ if (be_exec_version >= VARIANT_SPARSE_SERDE) {
+ buf =
ColumnObject::get_sparse_column_type()->serialize(*column_object.get_sparse_column(),
+ buf,
be_exec_version);
+ }
return buf;
}
@@ -193,10 +197,14 @@ const char* DataTypeObject::deserialize(const char* buf,
MutableColumnPtr* colum
}
// deserialize sparse column
- // TODO make compability with sparse column
- MutableColumnPtr sparse_column =
ColumnObject::get_sparse_column_type()->create_column();
- buf = ColumnObject::get_sparse_column_type()->deserialize(buf,
&sparse_column, be_exec_version);
- column_object->set_sparse_column(std::move(sparse_column));
+ if (be_exec_version >= VARIANT_SPARSE_SERDE) {
+ MutableColumnPtr sparse_column =
ColumnObject::get_sparse_column_type()->create_column();
+ buf = ColumnObject::get_sparse_column_type()->deserialize(buf,
&sparse_column,
+
be_exec_version);
+ column_object->set_sparse_column(std::move(sparse_column));
+ } else {
+
column_object->get_sparse_column()->assume_mutable()->insert_many_defaults(num_rows);
+ }
if (!root_added && column_object->get_subcolumn({})) {
column_object->get_subcolumn({})->insert_many_defaults(num_rows);
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index be0390db584..f3ed61e1c3f 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1955,7 +1955,7 @@ public class Config extends ConfigBase {
* Max data version of backends serialize block.
*/
@ConfField(mutable = false)
- public static int max_be_exec_version = 8;
+ public static int max_be_exec_version = 9;
/**
* Min data version of backends serialize block.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
