This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new ebbfb06162c [Bug](array) fix array column core dump in
get_shrinked_column as not check type (#33295)
ebbfb06162c is described below
commit ebbfb06162c886027bc17c4891081d99223ddfb7
Author: zhangstar333 <87313068+zhangstar...@users.noreply.github.com>
AuthorDate: Mon Apr 8 07:26:42 2024 +0800
[Bug](array) fix array column core dump in get_shrinked_column as not check
type (#33295)
* [Bug](array) fix array column core dump in get_shrinked_column as not
check type
* add function could_shrinked_column
---
be/src/vec/columns/column.h | 6 +++++-
be/src/vec/columns/column_array.cpp | 10 +++++++++-
be/src/vec/columns/column_array.h | 1 +
be/src/vec/columns/column_map.cpp | 10 ++++++----
be/src/vec/columns/column_map.h | 1 +
be/src/vec/columns/column_nullable.cpp | 12 ++++++++++--
be/src/vec/columns/column_nullable.h | 2 ++
be/src/vec/columns/column_string.h | 1 +
be/src/vec/columns/column_struct.cpp | 13 +++++++++++--
be/src/vec/columns/column_struct.h | 1 +
.../data/query_p0/test_array_orderby_limit.out | 3 +++
.../suites/query_p0/test_array_orderby_limit.groovy | 16 ++++++++++++++++
12 files changed, 66 insertions(+), 10 deletions(-)
diff --git a/be/src/vec/columns/column.h b/be/src/vec/columns/column.h
index f6749832363..afb97542e5a 100644
--- a/be/src/vec/columns/column.h
+++ b/be/src/vec/columns/column.h
@@ -127,10 +127,14 @@ public:
// shrink the end zeros for CHAR type or ARRAY<CHAR> type
virtual MutablePtr get_shrinked_column() {
- LOG(FATAL) << "Cannot clone_resized() column " << get_name();
+ LOG(FATAL) << "Cannot get_shrinked_column() column " << get_name();
return nullptr;
}
+ // check the column whether could shrinked
+ // now support only in char type, or the nested type in complex type:
array{char}, struct{char}, map{char}
+ virtual bool could_shrinked_column() { return false; }
+
/// Some columns may require finalization before using of other operations.
virtual void finalize() {}
diff --git a/be/src/vec/columns/column_array.cpp
b/be/src/vec/columns/column_array.cpp
index 86d31c9223b..7251f1e1798 100644
--- a/be/src/vec/columns/column_array.cpp
+++ b/be/src/vec/columns/column_array.cpp
@@ -118,8 +118,16 @@ ColumnArray::ColumnArray(MutableColumnPtr&& nested_column)
: data(std::move(nest
offsets = ColumnOffsets::create();
}
+bool ColumnArray::could_shrinked_column() {
+ return data->could_shrinked_column();
+}
+
MutableColumnPtr ColumnArray::get_shrinked_column() {
- return ColumnArray::create(data->get_shrinked_column(),
offsets->assume_mutable());
+ if (could_shrinked_column()) {
+ return ColumnArray::create(data->get_shrinked_column(),
offsets->assume_mutable());
+ } else {
+ return ColumnArray::create(data->assume_mutable(),
offsets->assume_mutable());
+ }
}
std::string ColumnArray::get_name() const {
diff --git a/be/src/vec/columns/column_array.h
b/be/src/vec/columns/column_array.h
index 79580d36a17..5f6db27c594 100644
--- a/be/src/vec/columns/column_array.h
+++ b/be/src/vec/columns/column_array.h
@@ -119,6 +119,7 @@ public:
}
MutableColumnPtr get_shrinked_column() override;
+ bool could_shrinked_column() override;
/** On the index i there is an offset to the beginning of the i + 1 -th
element. */
using ColumnOffsets = ColumnVector<Offset64>;
diff --git a/be/src/vec/columns/column_map.cpp
b/be/src/vec/columns/column_map.cpp
index c1c668ef07c..bec58bfda7b 100644
--- a/be/src/vec/columns/column_map.cpp
+++ b/be/src/vec/columns/column_map.cpp
@@ -455,18 +455,20 @@ ColumnPtr ColumnMap::replicate(const Offsets& offsets)
const {
return res;
}
+bool ColumnMap::could_shrinked_column() {
+ return keys_column->could_shrinked_column() ||
values_column->could_shrinked_column();
+}
+
MutableColumnPtr ColumnMap::get_shrinked_column() {
MutableColumns new_columns(2);
- if (keys_column->is_column_string() || keys_column->is_column_array() ||
- keys_column->is_column_map() || keys_column->is_column_struct()) {
+ if (keys_column->could_shrinked_column()) {
new_columns[0] = keys_column->get_shrinked_column();
} else {
new_columns[0] = keys_column->get_ptr();
}
- if (values_column->is_column_string() || values_column->is_column_array()
||
- values_column->is_column_map() || values_column->is_column_struct()) {
+ if (values_column->could_shrinked_column()) {
new_columns[1] = values_column->get_shrinked_column();
} else {
new_columns[1] = values_column->get_ptr();
diff --git a/be/src/vec/columns/column_map.h b/be/src/vec/columns/column_map.h
index 3b9c4a100e0..0fe61ccdaeb 100644
--- a/be/src/vec/columns/column_map.h
+++ b/be/src/vec/columns/column_map.h
@@ -113,6 +113,7 @@ public:
void update_hash_with_value(size_t n, SipHash& hash) const override;
MutableColumnPtr get_shrinked_column() override;
+ bool could_shrinked_column() override;
ColumnPtr filter(const Filter& filt, ssize_t result_size_hint) const
override;
size_t filter(const Filter& filter) override;
ColumnPtr permute(const Permutation& perm, size_t limit) const override;
diff --git a/be/src/vec/columns/column_nullable.cpp
b/be/src/vec/columns/column_nullable.cpp
index e20b87af826..dc8853e49af 100644
--- a/be/src/vec/columns/column_nullable.cpp
+++ b/be/src/vec/columns/column_nullable.cpp
@@ -48,9 +48,17 @@ ColumnNullable::ColumnNullable(MutableColumnPtr&&
nested_column_, MutableColumnP
_need_update_has_null = true;
}
+bool ColumnNullable::could_shrinked_column() {
+ return get_nested_column_ptr()->could_shrinked_column();
+}
+
MutableColumnPtr ColumnNullable::get_shrinked_column() {
- return
ColumnNullable::create(get_nested_column_ptr()->get_shrinked_column(),
- get_null_map_column_ptr());
+ if (could_shrinked_column()) {
+ return
ColumnNullable::create(get_nested_column_ptr()->get_shrinked_column(),
+ get_null_map_column_ptr());
+ } else {
+ return ColumnNullable::create(get_nested_column_ptr(),
get_null_map_column_ptr());
+ }
}
void ColumnNullable::update_xxHash_with_value(size_t start, size_t end,
uint64_t& hash,
diff --git a/be/src/vec/columns/column_nullable.h
b/be/src/vec/columns/column_nullable.h
index 2c13de158a8..1bec1165aae 100644
--- a/be/src/vec/columns/column_nullable.h
+++ b/be/src/vec/columns/column_nullable.h
@@ -82,7 +82,9 @@ public:
}
MutableColumnPtr get_shrinked_column() override;
+ bool could_shrinked_column() override;
bool is_variable_length() const override { return
nested_column->is_variable_length(); }
+
const char* get_family_name() const override { return "Nullable"; }
std::string get_name() const override { return "Nullable(" +
nested_column->get_name() + ")"; }
MutableColumnPtr clone_resized(size_t size) const override;
diff --git a/be/src/vec/columns/column_string.h
b/be/src/vec/columns/column_string.h
index 705af04b1f8..04a599f5e9b 100644
--- a/be/src/vec/columns/column_string.h
+++ b/be/src/vec/columns/column_string.h
@@ -120,6 +120,7 @@ public:
MutableColumnPtr clone_resized(size_t to_size) const override;
MutableColumnPtr get_shrinked_column() override;
+ bool could_shrinked_column() override { return true; }
Field operator[](size_t n) const override {
assert(n < size());
diff --git a/be/src/vec/columns/column_struct.cpp
b/be/src/vec/columns/column_struct.cpp
index 290452a1c8b..b1a415de272 100644
--- a/be/src/vec/columns/column_struct.cpp
+++ b/be/src/vec/columns/column_struct.cpp
@@ -295,13 +295,22 @@ ColumnPtr ColumnStruct::replicate(const Offsets& offsets)
const {
return ColumnStruct::create(new_columns);
}
+bool ColumnStruct::could_shrinked_column() {
+ const size_t tuple_size = columns.size();
+ for (size_t i = 0; i < tuple_size; ++i) {
+ if (columns[i]->could_shrinked_column()) {
+ return true;
+ }
+ }
+ return false;
+}
+
MutableColumnPtr ColumnStruct::get_shrinked_column() {
const size_t tuple_size = columns.size();
MutableColumns new_columns(tuple_size);
for (size_t i = 0; i < tuple_size; ++i) {
- if (columns[i]->is_column_string() || columns[i]->is_column_array() ||
- columns[i]->is_column_map() || columns[i]->is_column_struct()) {
+ if (columns[i]->could_shrinked_column()) {
new_columns[i] = columns[i]->get_shrinked_column();
} else {
new_columns[i] = columns[i]->get_ptr();
diff --git a/be/src/vec/columns/column_struct.h
b/be/src/vec/columns/column_struct.h
index d91d3800648..ce4b5f461d7 100644
--- a/be/src/vec/columns/column_struct.h
+++ b/be/src/vec/columns/column_struct.h
@@ -157,6 +157,7 @@ public:
int compare_at(size_t n, size_t m, const IColumn& rhs_, int
nan_direction_hint) const override;
MutableColumnPtr get_shrinked_column() override;
+ bool could_shrinked_column() override;
void reserve(size_t n) override;
void resize(size_t n) override;
diff --git a/regression-test/data/query_p0/test_array_orderby_limit.out
b/regression-test/data/query_p0/test_array_orderby_limit.out
index abcea7af965..d06cad836aa 100644
--- a/regression-test/data/query_p0/test_array_orderby_limit.out
+++ b/regression-test/data/query_p0/test_array_orderby_limit.out
@@ -2,3 +2,6 @@
-- !select --
100 [["abc"]]
+-- !select_2 --
+a {"codes": [123, 456], "props": {"key1":["char1", "char2"]}}
+
diff --git a/regression-test/suites/query_p0/test_array_orderby_limit.groovy
b/regression-test/suites/query_p0/test_array_orderby_limit.groovy
index c573b1c7986..2fa3bb9c5ed 100644
--- a/regression-test/suites/query_p0/test_array_orderby_limit.groovy
+++ b/regression-test/suites/query_p0/test_array_orderby_limit.groovy
@@ -45,4 +45,20 @@ suite("test_array_char_orderby", "query") {
}
qt_select """ select * from ${testTable} order by k1 limit 1 """
+
+ sql "DROP TABLE IF EXISTS unpart_tbl_parquet_struct_3;"
+ sql """
+ CREATE TABLE unpart_tbl_parquet_struct_3 (
+ `col1` CHAR,
+ `col20` STRUCT<codes:ARRAY<INT>,props:MAP<STRING, ARRAY<CHAR(16)>>>
+ )ENGINE=OLAP
+ DUPLICATE KEY(`col1`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`col1`) BUCKETS 5
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+ sql """ insert into unpart_tbl_parquet_struct_3 values
('a',STRUCT(ARRAY(123, 456), MAP('key1', ARRAY('char1', 'char2'))) ); """
+ qt_select_2 """ select * from unpart_tbl_parquet_struct_3;"""
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
