This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 8928bf91222 branch-3.0: [fix](schema-change) fix array/map/struct in
schema-change not-null to null will make core #45305 (#45481)
8928bf91222 is described below
commit 8928bf9122293593616c73ffe61727daa985226e
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Dec 17 17:06:24 2024 +0800
branch-3.0: [fix](schema-change) fix array/map/struct in schema-change
not-null to null will make core #45305 (#45481)
Cherry-picked from #45305
Co-authored-by: amory <wangqian...@selectdb.com>
---
be/src/olap/rowset/segment_v2/column_reader.cpp | 42 +++++++++--
.../ddl/create_nestedtypes_with_schemachange.out | 84 ++++++++++++++++++++++
.../create_nestedtypes_with_schemachange.groovy | 57 +++++++++++++++
3 files changed, 177 insertions(+), 6 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp
b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 66e24fc105e..f44885325e4 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -860,8 +860,18 @@ Status MapFileColumnIterator::next_batch(size_t* n,
vectorized::MutableColumnPtr
size_t num_read = *n;
auto null_map_ptr =
static_cast<vectorized::ColumnNullable&>(*dst).get_null_map_column_ptr();
- bool null_signs_has_null = false;
- RETURN_IF_ERROR(_null_iterator->next_batch(&num_read, null_map_ptr,
&null_signs_has_null));
+ // in not-null to null linked-schemachange mode,
+ // actually we do not change dat data include meta in footer,
+ // so may dst from changed meta which is nullable but old data is not
nullable,
+ // if so, we should set null_map to all null by default
+ if (_null_iterator) {
+ bool null_signs_has_null = false;
+ RETURN_IF_ERROR(
+ _null_iterator->next_batch(&num_read, null_map_ptr,
&null_signs_has_null));
+ } else {
+ auto& null_map =
assert_cast<vectorized::ColumnUInt8&>(*null_map_ptr);
+ null_map.insert_many_vals(0, num_read);
+ }
DCHECK(num_read == *n);
}
return Status::OK();
@@ -921,8 +931,18 @@ Status StructFileColumnIterator::next_batch(size_t* n,
vectorized::MutableColumn
size_t num_read = *n;
auto null_map_ptr =
static_cast<vectorized::ColumnNullable&>(*dst).get_null_map_column_ptr();
- bool null_signs_has_null = false;
- RETURN_IF_ERROR(_null_iterator->next_batch(&num_read, null_map_ptr,
&null_signs_has_null));
+ // in not-null to null linked-schemachange mode,
+ // actually we do not change dat data include meta in footer,
+ // so may dst from changed meta which is nullable but old data is not
nullable,
+ // if so, we should set null_map to all null by default
+ if (_null_iterator) {
+ bool null_signs_has_null = false;
+ RETURN_IF_ERROR(
+ _null_iterator->next_batch(&num_read, null_map_ptr,
&null_signs_has_null));
+ } else {
+ auto& null_map =
assert_cast<vectorized::ColumnUInt8&>(*null_map_ptr);
+ null_map.insert_many_vals(0, num_read);
+ }
DCHECK(num_read == *n);
}
@@ -1075,8 +1095,18 @@ Status ArrayFileColumnIterator::next_batch(size_t* n,
vectorized::MutableColumnP
auto null_map_ptr =
static_cast<vectorized::ColumnNullable&>(*dst).get_null_map_column_ptr();
size_t num_read = *n;
- bool null_signs_has_null = false;
- RETURN_IF_ERROR(_null_iterator->next_batch(&num_read, null_map_ptr,
&null_signs_has_null));
+ // in not-null to null linked-schemachange mode,
+ // actually we do not change dat data include meta in footer,
+ // so may dst from changed meta which is nullable but old data is not
nullable,
+ // if so, we should set null_map to all null by default
+ if (_null_iterator) {
+ bool null_signs_has_null = false;
+ RETURN_IF_ERROR(
+ _null_iterator->next_batch(&num_read, null_map_ptr,
&null_signs_has_null));
+ } else {
+ auto& null_map =
assert_cast<vectorized::ColumnUInt8&>(*null_map_ptr);
+ null_map.insert_many_vals(0, num_read);
+ }
DCHECK(num_read == *n);
}
diff --git
a/regression-test/data/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.out
b/regression-test/data/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.out
index af88eb6b9bb..5b811095fcb 100644
---
a/regression-test/data/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.out
+++
b/regression-test/data/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.out
@@ -44,3 +44,87 @@ col3 array<int> Yes false \N NONE
col4 map<int,int> Yes false \N NONE
col5 struct<f1:int> Yes false \N NONE
+-- !sql_before --
+1 2 [1, 2] {1:2} {"f1":1} {"a":[1,2,3]}
+
+-- !sql --
+col0 bigint No true \N
+col2 int No false \N NONE
+col3 array<int> Yes false \N NONE
+col4 map<int,int> Yes false \N NONE
+col5 struct<f1:int> Yes false \N NONE
+col6 variant Yes false \N NONE
+
+-- !sql_after --
+1 2 [1, 2] {1:2} {"f1":1} {"a":[1,2,3]}
+
+-- !sql_before --
+1 2 [1, 2] {1:2} {"f1":1} {"a":[1,2,3]}
+
+-- !sql --
+col0 bigint No true \N
+col2 int No false \N NONE
+col3 array<int> Yes false \N NONE
+col4 map<int,int> No false \N NONE
+col5 struct<f1:int> No false \N NONE
+col6 variant No false \N NONE
+
+-- !sql_after --
+1 2 [1, 2] {1:2} {"f1":1} {"a":[1,2,3]}
+
+-- !sql_before --
+1 2 [1, 2] {1:2} {"f1":1} {"a":[1,2,3]}
+
+-- !sql --
+col0 bigint No true \N
+col2 int No false \N NONE
+col3 array<int> Yes false \N NONE
+col4 map<int,int> Yes false \N NONE
+col5 struct<f1:int> Yes false \N NONE
+col6 variant Yes false \N NONE
+
+-- !sql_after --
+1 2 [1, 2] {1:2} {"f1":1} {"a":[1,2,3]}
+
+-- !sql_before --
+1 2 [1, 2] {1:2} {"f1":1} {"a":[1,2,3]}
+
+-- !sql --
+col0 bigint No true \N
+col2 int No false \N NONE
+col3 array<int> No false \N NONE
+col4 map<int,int> Yes false \N NONE
+col5 struct<f1:int> No false \N NONE
+col6 variant No false \N NONE
+
+-- !sql_after --
+1 2 [1, 2] {1:2} {"f1":1} {"a":[1,2,3]}
+
+-- !sql_before --
+1 2 [1, 2] {1:2} {"f1":1} {"a":[1,2,3]}
+
+-- !sql --
+col0 bigint No true \N
+col2 int No false \N NONE
+col3 array<int> Yes false \N NONE
+col4 map<int,int> Yes false \N NONE
+col5 struct<f1:int> Yes false \N NONE
+col6 variant Yes false \N NONE
+
+-- !sql_after --
+1 2 [1, 2] {1:2} {"f1":1} {"a":[1,2,3]}
+
+-- !sql_before --
+1 2 [1, 2] {1:2} {"f1":1} {"a":[1,2,3]}
+
+-- !sql --
+col0 bigint No true \N
+col2 int No false \N NONE
+col3 array<int> No false \N NONE
+col4 map<int,int> No false \N NONE
+col5 struct<f1:int> Yes false \N NONE
+col6 variant No false \N NONE
+
+-- !sql_after --
+1 2 [1, 2] {1:2} {"f1":1} {"a":[1,2,3]}
+
diff --git
a/regression-test/suites/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.groovy
b/regression-test/suites/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.groovy
index ec0f163821d..62b9ab3bb56 100644
---
a/regression-test/suites/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.groovy
+++
b/regression-test/suites/datatype_p0/nested_types/ddl/create_nestedtypes_with_schemachange.groovy
@@ -62,4 +62,61 @@ suite("create_nestedtypes_with_schemachange", "p0") {
// struct with other type
create_nested_table_and_schema_change.call("test_struct_schemachange_1",
"STRUCT<f1: varchar(1)>", "col5", "errCode = 2");
+ def create_nested_table_and_schema_change_null_trans = {testTablex,
nested_type, column_name, notNull2Null ->
+ def null_define = "NULL"
+ if (notNull2Null) {
+ null_define = "NOT NULL"
+ }
+ // create basic type
+ sql "DROP TABLE IF EXISTS $testTablex"
+ sql """ CREATE TABLE IF NOT EXISTS $testTablex (
+ col0 BIGINT NOT NULL,
+ col2 int NOT NULL,
+ col3 array<int> $null_define,
+ col4 map<int, int> $null_define,
+ col5 struct<f1: int> $null_define,
+ col6 variant $null_define
+ )
+ /* mow */
+ UNIQUE KEY(col0) DISTRIBUTED BY HASH(col0) BUCKETS 4
PROPERTIES (
+ "enable_unique_key_merge_on_write" = "true",
+ "replication_num" = "1",
+ 'light_schema_change' = 'true',
'disable_auto_compaction'='true'
+ ); """
+ // insert data
+ sql """ INSERT INTO $testTablex VALUES (1, 2, array(1, 2), map(1, 2),
named_struct('f1', 1), '{"a": [1,2,3]}')"""
+ // select
+ qt_sql_before "select * from $testTablex"
+
+ if (notNull2Null) {
+ sql "ALTER TABLE $testTablex MODIFY COLUMN $column_name
$nested_type NULL"
+ waitForSchemaChangeDone {
+ sql """ SHOW ALTER TABLE COLUMN WHERE IndexName='$testTablex'
ORDER BY createtime DESC LIMIT 1 """
+ time 600
+ }
+ } else {
+ // schema change from null to non-nullable is not supported
+ test {
+ sql "ALTER TABLE $testTablex MODIFY COLUMN $column_name
$nested_type NOT NULL"
+ exception "Can not change from nullable to non-nullable"
+ }
+ }
+ // desc table
+ qt_sql "DESC $testTablex"
+ qt_sql_after "select * from $testTablex"
+ }
+
+ // array
+
create_nested_table_and_schema_change_null_trans.call("test_array_schemachange_null",
"ARRAY<INT>", "col3", false)
+
create_nested_table_and_schema_change_null_trans.call("test_array_schemachange_null1",
"ARRAY<INT>", "col3", true)
+ // map
+
create_nested_table_and_schema_change_null_trans.call("test_map_schemachange_null",
"Map<INT, INT>", "col4", false)
+
create_nested_table_and_schema_change_null_trans.call("test_map_schemachange_null1",
"Map<INT, INT>", "col4", true)
+ // struct
+
create_nested_table_and_schema_change_null_trans.call("test_struct_schemachange_null",
"struct<f1: int>", "col5", false)
+
create_nested_table_and_schema_change_null_trans.call("test_struct_schemachange_null1",
"struct<f1: int>", "col5", true)
+ // variant
+ //
create_nested_table_and_schema_change_null_trans.call("test_v_schemachange_null",
"variant", "col6", false)
+ //
create_nested_table_and_schema_change_null_trans.call("test_v_schemachange_null1",
"variant", "col6", true)
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
