This is an automated email from the ASF dual-hosted git repository.
jianliangqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new a118eb923bf [fix](build index) fix core when build index for a new
column which without data (#32550)
a118eb923bf is described below
commit a118eb923bfe6f7321768be9389c137e0341b34f
Author: qiye <jianliang5...@gmail.com>
AuthorDate: Fri Mar 22 14:05:15 2024 +0800
[fix](build index) fix core when build index for a new column which without
data (#32550)
Co-authored-by: Luennng <luen...@gmail.com>
Co-authored-by: Tanya-W <tanya1218w@163,com>
---
be/src/olap/task/index_builder.cpp | 26 ++++--
.../test_index_change_on_new_column.out | 7 ++
.../test_index_change_on_new_column.groovy | 97 ++++++++++++++++++++++
3 files changed, 123 insertions(+), 7 deletions(-)
diff --git a/be/src/olap/task/index_builder.cpp
b/be/src/olap/task/index_builder.cpp
index 813d8232e41..ab881c7a842 100644
--- a/be/src/olap/task/index_builder.cpp
+++ b/be/src/olap/task/index_builder.cpp
@@ -122,6 +122,13 @@ Status IndexBuilder::update_inverted_index_info() {
TabletIndex index;
index.init_from_thrift(t_inverted_index,
*input_rs_tablet_schema);
auto column_uid = index.col_unique_ids()[0];
+ if (column_uid < 0) {
+ LOG(WARNING) << "referenced column was missing. "
+ << "[column=" << t_inverted_index.columns[0]
+ << " referenced_column=" << column_uid << "]";
+ output_rs_tablet_schema->append_index(index);
+ continue;
+ }
const TabletColumn& col =
output_rs_tablet_schema->column_by_uid(column_uid);
const TabletIndex* exist_index =
output_rs_tablet_schema->get_inverted_index(col);
if (exist_index && exist_index->index_id() !=
index.index_id()) {
@@ -361,6 +368,12 @@ Status
IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta
inverted_index_writer_signs.emplace_back(writer_sign);
}
}
+
+ if (return_columns.empty()) {
+ // no columns to read
+ break;
+ }
+
_inverted_index_file_writers.emplace(seg_ptr->id(),
std::move(inverted_index_file_writer));
@@ -379,7 +392,7 @@ Status
IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta
return
Status::Error<ErrorCode::ROWSET_READER_INIT>(res.to_string());
}
- std::shared_ptr<vectorized::Block> block =
std::make_shared<vectorized::Block>(
+ auto block = vectorized::Block::create_unique(
output_rowset_schema->create_block(return_columns));
while (true) {
auto status = iter->next_batch(block.get());
@@ -448,12 +461,6 @@ Status
IndexBuilder::_write_inverted_index_data(TabletSchemaSPtr tablet_schema,
for (auto i = 0; i < _alter_inverted_indexes.size(); ++i) {
auto inverted_index = _alter_inverted_indexes[i];
auto index_id = inverted_index.index_id;
- auto converted_result = _olap_data_convertor->convert_column_data(i);
- if (!converted_result.first.ok()) {
- LOG(WARNING) << "failed to convert block, errcode: " <<
converted_result.first;
- return converted_result.first;
- }
-
auto column_name = inverted_index.columns[0];
auto column_idx = tablet_schema->field_index(column_name);
if (column_idx < 0) {
@@ -464,6 +471,11 @@ Status
IndexBuilder::_write_inverted_index_data(TabletSchemaSPtr tablet_schema,
auto column = tablet_schema->column(column_idx);
auto writer_sign = std::make_pair(segment_idx, index_id);
std::unique_ptr<Field> field(FieldFactory::create(column));
+ auto converted_result = _olap_data_convertor->convert_column_data(i);
+ if (converted_result.first != Status::OK()) {
+ LOG(WARNING) << "failed to convert block, errcode: " <<
converted_result.first;
+ return converted_result.first;
+ }
const auto* ptr = (const uint8_t*)converted_result.second->get_data();
if (converted_result.second->get_nullmap()) {
RETURN_IF_ERROR(_add_nullable(column_name, writer_sign,
field.get(),
diff --git
a/regression-test/data/inverted_index_p0/index_change/test_index_change_on_new_column.out
b/regression-test/data/inverted_index_p0/index_change/test_index_change_on_new_column.out
new file mode 100644
index 00000000000..6fc02ad3fde
--- /dev/null
+++
b/regression-test/data/inverted_index_p0/index_change/test_index_change_on_new_column.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select1 --
+1 hello world \N
+
+-- !select2 --
+1 hello world \N
+
diff --git
a/regression-test/suites/inverted_index_p0/index_change/test_index_change_on_new_column.groovy
b/regression-test/suites/inverted_index_p0/index_change/test_index_change_on_new_column.groovy
new file mode 100644
index 00000000000..b4ca879569b
--- /dev/null
+++
b/regression-test/suites/inverted_index_p0/index_change/test_index_change_on_new_column.groovy
@@ -0,0 +1,97 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite("test_index_change_on_new_column") {
+ def timeout = 60000
+ def delta_time = 1000
+ def alter_res = "null"
+ def useTime = 0
+
+ def wait_for_latest_op_on_table_finish = { table_name, OpTimeout ->
+ for(int t = delta_time; t <= OpTimeout; t += delta_time){
+ alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName =
"${table_name}" ORDER BY CreateTime DESC LIMIT 1;"""
+ alter_res = alter_res.toString()
+ if(alter_res.contains("FINISHED")) {
+ sleep(3000) // wait change table state to normal
+ logger.info(table_name + " latest alter job finished, detail:
" + alter_res)
+ break
+ }
+ useTime = t
+ sleep(delta_time)
+ }
+ assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish
timeout")
+ }
+
+ def wait_for_build_index_on_partition_finish = { table_name, OpTimeout ->
+ for(int t = delta_time; t <= OpTimeout; t += delta_time){
+ alter_res = sql """SHOW BUILD INDEX WHERE TableName =
"${table_name}";"""
+ def expected_finished_num = alter_res.size();
+ def finished_num = 0;
+ for (int i = 0; i < expected_finished_num; i++) {
+ logger.info(table_name + " build index job state: " +
alter_res[i][7] + i)
+ if (alter_res[i][7] == "FINISHED") {
+ ++finished_num;
+ }
+ }
+ if (finished_num == expected_finished_num) {
+ logger.info(table_name + " all build index jobs finished,
detail: " + alter_res)
+ break
+ }
+ useTime = t
+ sleep(delta_time)
+ }
+ assertTrue(useTime <= OpTimeout,
"wait_for_latest_build_index_on_partition_finish timeout")
+ }
+
+ def tableName = "test_index_change_on_new_column"
+
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `id` INT COMMENT "",
+ `s` STRING COMMENT ""
+ )
+ DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`)
+ PROPERTIES ( "replication_num" = "1" );
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES
+ (1, 'hello world')
+ """
+
+ // add new column
+ sql """ alter table ${tableName} add column s1 varchar(50) default null
after s; """
+
+ qt_select1 """ SELECT * FROM ${tableName}; """
+
+ // create inverted index on new column
+ sql """ alter table ${tableName} add index idx_s1(s1) USING INVERTED """
+ wait_for_latest_op_on_table_finish(tableName, timeout)
+
+ // build inverted index on new column
+ sql """ build index idx_s1 on ${tableName} """
+ wait_for_build_index_on_partition_finish(tableName, timeout)
+
+ def show_result = sql "show index from ${tableName}"
+ logger.info("show index from " + tableName + " result: " + show_result)
+ assertEquals(show_result.size(), 1)
+ assertEquals(show_result[0][2], "idx_s1")
+
+ qt_select2 """ SELECT * FROM ${tableName}; """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
