This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 6c3b35288e1 [fix](build index) fix core when build index for a new
column which without data (#27276)
6c3b35288e1 is described below
commit 6c3b35288e16176792acfa04dcfa31e9b354e3bb
Author: YueW <45946325+tany...@users.noreply.github.com>
AuthorDate: Wed Nov 22 14:10:15 2023 +0800
[fix](build index) fix core when build index for a new column which without
data (#27276)
---
be/src/olap/task/index_builder.cpp | 19 +++--
.../test_index_change_on_new_column.out | 7 ++
.../test_index_change_on_new_column.groovy | 97 ++++++++++++++++++++++
3 files changed, 115 insertions(+), 8 deletions(-)
diff --git a/be/src/olap/task/index_builder.cpp
b/be/src/olap/task/index_builder.cpp
index b26187ebb39..6019857d229 100644
--- a/be/src/olap/task/index_builder.cpp
+++ b/be/src/olap/task/index_builder.cpp
@@ -198,6 +198,11 @@ Status
IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta
}
}
+ if (return_columns.empty()) {
+ // no columns to read
+ break;
+ }
+
// create iterator for each segment
StorageReadOptions read_options;
OlapReaderStatistics stats;
@@ -212,8 +217,7 @@ Status
IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta
<< "]: " << res.to_string();
return
Status::Error<ErrorCode::ROWSET_READER_INIT>(res.to_string());
}
-
- std::shared_ptr<vectorized::Block> block =
std::make_shared<vectorized::Block>(
+ auto block = vectorized::Block::create_unique(
output_rowset_schema->create_block(return_columns));
while (true) {
auto st = iter->next_batch(block.get());
@@ -264,12 +268,6 @@ Status
IndexBuilder::_write_inverted_index_data(TabletSchemaSPtr tablet_schema,
for (auto i = 0; i < _alter_inverted_indexes.size(); ++i) {
auto inverted_index = _alter_inverted_indexes[i];
auto index_id = inverted_index.index_id;
- auto converted_result = _olap_data_convertor->convert_column_data(i);
- if (converted_result.first != Status::OK()) {
- LOG(WARNING) << "failed to convert block, errcode: " <<
converted_result.first;
- return converted_result.first;
- }
-
auto column_name = inverted_index.columns[0];
auto column_idx = tablet_schema->field_index(column_name);
if (column_idx < 0) {
@@ -280,6 +278,11 @@ Status
IndexBuilder::_write_inverted_index_data(TabletSchemaSPtr tablet_schema,
auto column = tablet_schema->column(column_idx);
auto writer_sign = std::make_pair(segment_idx, index_id);
std::unique_ptr<Field> field(FieldFactory::create(column));
+ auto converted_result = _olap_data_convertor->convert_column_data(i);
+ if (converted_result.first != Status::OK()) {
+ LOG(WARNING) << "failed to convert block, errcode: " <<
converted_result.first;
+ return converted_result.first;
+ }
const auto* ptr = (const uint8_t*)converted_result.second->get_data();
if (converted_result.second->get_nullmap()) {
RETURN_IF_ERROR(_add_nullable(column_name, writer_sign,
field.get(),
diff --git
a/regression-test/data/inverted_index_p0/index_change/test_index_change_on_new_column.out
b/regression-test/data/inverted_index_p0/index_change/test_index_change_on_new_column.out
new file mode 100644
index 00000000000..6fc02ad3fde
--- /dev/null
+++
b/regression-test/data/inverted_index_p0/index_change/test_index_change_on_new_column.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select1 --
+1 hello world \N
+
+-- !select2 --
+1 hello world \N
+
diff --git
a/regression-test/suites/inverted_index_p0/index_change/test_index_change_on_new_column.groovy
b/regression-test/suites/inverted_index_p0/index_change/test_index_change_on_new_column.groovy
new file mode 100644
index 00000000000..b4ca879569b
--- /dev/null
+++
b/regression-test/suites/inverted_index_p0/index_change/test_index_change_on_new_column.groovy
@@ -0,0 +1,97 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite("test_index_change_on_new_column") {
+ def timeout = 60000
+ def delta_time = 1000
+ def alter_res = "null"
+ def useTime = 0
+
+ def wait_for_latest_op_on_table_finish = { table_name, OpTimeout ->
+ for(int t = delta_time; t <= OpTimeout; t += delta_time){
+ alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName =
"${table_name}" ORDER BY CreateTime DESC LIMIT 1;"""
+ alter_res = alter_res.toString()
+ if(alter_res.contains("FINISHED")) {
+ sleep(3000) // wait change table state to normal
+ logger.info(table_name + " latest alter job finished, detail:
" + alter_res)
+ break
+ }
+ useTime = t
+ sleep(delta_time)
+ }
+ assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish
timeout")
+ }
+
+ def wait_for_build_index_on_partition_finish = { table_name, OpTimeout ->
+ for(int t = delta_time; t <= OpTimeout; t += delta_time){
+ alter_res = sql """SHOW BUILD INDEX WHERE TableName =
"${table_name}";"""
+ def expected_finished_num = alter_res.size();
+ def finished_num = 0;
+ for (int i = 0; i < expected_finished_num; i++) {
+ logger.info(table_name + " build index job state: " +
alter_res[i][7] + i)
+ if (alter_res[i][7] == "FINISHED") {
+ ++finished_num;
+ }
+ }
+ if (finished_num == expected_finished_num) {
+ logger.info(table_name + " all build index jobs finished,
detail: " + alter_res)
+ break
+ }
+ useTime = t
+ sleep(delta_time)
+ }
+ assertTrue(useTime <= OpTimeout,
"wait_for_latest_build_index_on_partition_finish timeout")
+ }
+
+ def tableName = "test_index_change_on_new_column"
+
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `id` INT COMMENT "",
+ `s` STRING COMMENT ""
+ )
+ DUPLICATE KEY(`id`) DISTRIBUTED BY HASH(`id`)
+ PROPERTIES ( "replication_num" = "1" );
+ """
+
+ sql """ INSERT INTO ${tableName} VALUES
+ (1, 'hello world')
+ """
+
+ // add new column
+ sql """ alter table ${tableName} add column s1 varchar(50) default null
after s; """
+
+ qt_select1 """ SELECT * FROM ${tableName}; """
+
+ // create inverted index on new column
+ sql """ alter table ${tableName} add index idx_s1(s1) USING INVERTED """
+ wait_for_latest_op_on_table_finish(tableName, timeout)
+
+ // build inverted index on new column
+ sql """ build index idx_s1 on ${tableName} """
+ wait_for_build_index_on_partition_finish(tableName, timeout)
+
+ def show_result = sql "show index from ${tableName}"
+ logger.info("show index from " + tableName + " result: " + show_result)
+ assertEquals(show_result.size(), 1)
+ assertEquals(show_result[0][2], "idx_s1")
+
+ qt_select2 """ SELECT * FROM ${tableName}; """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
