This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 6df51d0b11c branch-3.1: [fix](index) remove unused update index #55514
(#55704)
6df51d0b11c is described below
commit 6df51d0b11cb5a657790a676134950b17437a205
Author: Sun Chenyang <[email protected]>
AuthorDate: Sat Sep 6 00:44:40 2025 +0800
branch-3.1: [fix](index) remove unused update index #55514 (#55704)
pick from master #55514
---
be/src/olap/tablet_schema.cpp | 20 ----
be/src/olap/tablet_schema.h | 2 -
be/src/vec/common/schema_util.cpp | 24 +++--
be/test/olap/tablet_schema_multi_index_test.cpp | 107 ---------------------
be/test/vec/common/schema_util_test.cpp | 55 -----------
.../suites/variant_p0/with_index/var_index.groovy | 96 ++++++++++++++++++
6 files changed, 110 insertions(+), 194 deletions(-)
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index e20c0bbda02..0aa95b1bbeb 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -964,26 +964,6 @@ void TabletSchema::append_index(TabletIndex&& index) {
}
}
-void TabletSchema::update_index(const TabletColumn& col, const IndexType&
index_type,
- std::vector<TabletIndex>&& indexes) {
- int32_t col_unique_id = col.is_extracted_column() ? col.parent_unique_id()
: col.unique_id();
- const std::string& suffix_path = escape_for_path_name(col.suffix_path());
- IndexKey key(index_type, col_unique_id, suffix_path);
- auto iter = _col_id_suffix_to_index.find(key);
- if (iter != _col_id_suffix_to_index.end()) {
- if (iter->second.size() == indexes.size()) {
- for (size_t i = 0; i < iter->second.size(); ++i) {
- int32_t pos = iter->second[i];
- if (pos >= 0 && pos < _indexes.size()) {
- _indexes[pos] =
std::make_shared<TabletIndex>(std::move(indexes[i]));
- }
- }
- }
- }
- LOG(WARNING) << " failed to update_index: " << index_type << " " <<
col_unique_id << " "
- << suffix_path;
-}
-
void TabletSchema::replace_column(size_t pos, TabletColumn new_col) {
CHECK_LT(pos, num_columns()) << " outof range";
_cols[pos] = std::make_shared<TabletColumn>(std::move(new_col));
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index afcbaa0627f..ac8da118734 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -383,8 +383,6 @@ public:
void to_schema_pb(TabletSchemaPB* tablet_meta_pb) const;
void append_column(TabletColumn column, ColumnType col_type =
ColumnType::NORMAL);
void append_index(TabletIndex&& index);
- void update_index(const TabletColumn& column, const IndexType& index_type,
- std::vector<TabletIndex>&& indexs);
void remove_index(int64_t index_id);
void clear_index();
// Must make sure the row column is always the last column
diff --git a/be/src/vec/common/schema_util.cpp
b/be/src/vec/common/schema_util.cpp
index 3dc6c4f69b6..68066cc840d 100644
--- a/be/src/vec/common/schema_util.cpp
+++ b/be/src/vec/common/schema_util.cpp
@@ -509,21 +509,25 @@ void inherit_column_attributes(const TabletColumn&
source, TabletColumn& target,
}
// 2. inverted index
- std::vector<TabletIndex> indexes_to_update;
+ TabletIndexes indexes_to_add;
auto source_indexes =
(*target_schema)->inverted_indexs(source.unique_id());
- for (const auto& source_index_meta : source_indexes) {
- TabletIndex index_info = *source_index_meta;
-
index_info.set_escaped_escaped_index_suffix_path(target.path_info_ptr()->get_path());
- indexes_to_update.emplace_back(std::move(index_info));
+ // if target is variant type, we need to inherit all indexes
+ // because this schema is a read schema from fe
+ if (target.is_variant_type()) {
+ for (auto& index : source_indexes) {
+ auto index_info = std::make_shared<TabletIndex>(*index);
+
index_info->set_escaped_escaped_index_suffix_path(target.path_info_ptr()->get_path());
+ indexes_to_add.emplace_back(std::move(index_info));
+ }
+ } else {
+ inherit_index(source_indexes, indexes_to_add, target);
}
auto target_indexes = (*target_schema)
->inverted_indexs(target.parent_unique_id(),
target.path_info_ptr()->get_path());
- if (!target_indexes.empty()) {
- (*target_schema)->update_index(target, IndexType::INVERTED,
std::move(indexes_to_update));
- } else {
- for (auto& index_info : indexes_to_update) {
- (*target_schema)->append_index(std::move(index_info));
+ if (target_indexes.empty()) {
+ for (auto& index_info : indexes_to_add) {
+ (*target_schema)->append_index(std::move(*index_info));
}
}
diff --git a/be/test/olap/tablet_schema_multi_index_test.cpp
b/be/test/olap/tablet_schema_multi_index_test.cpp
index 949af0b48b1..62dc2257ce1 100644
--- a/be/test/olap/tablet_schema_multi_index_test.cpp
+++ b/be/test/olap/tablet_schema_multi_index_test.cpp
@@ -116,113 +116,6 @@ TEST_F(TabletSchemaMultiIndexTest,
AppendIndexWithExtractedColumn) {
ASSERT_EQ(schema->_col_id_suffix_to_index[key][0], 0);
}
-TEST_F(TabletSchemaMultiIndexTest, UpdateIndex) {
- // Setup initial index
- TabletIndex index;
- index._index_id = 100;
- index._index_type = IndexType::INVERTED;
- index._col_unique_ids = {1};
- schema->append_index(std::move(index));
-
- // Create updated index
- std::vector<TabletIndex> updates;
- TabletIndex updated_index;
- updated_index._index_id = 101;
- updated_index._index_type = IndexType::INVERTED;
- updated_index._col_unique_ids = {1};
- std::map<std::string, std::string> propertie = {{"updated_key",
"updated_value"}};
- updated_index._properties = propertie;
- updates.push_back(updated_index);
-
- TabletColumn col1;
- col1._unique_id = 1;
-
- // Update the index
- schema->update_index(col1, IndexType::INVERTED, std::move(updates));
-
- // Verify the index was updated
- ASSERT_EQ(schema->_indexes[0]->_index_id, 101);
- ASSERT_EQ(schema->_indexes[0]->_properties, propertie);
-}
-
-TEST_F(TabletSchemaMultiIndexTest, UpdateIndexWithMultipleIndexes) {
- // Setup two indexes for same column
- TabletIndex index1;
- index1._index_id = 100;
- index1._index_type = IndexType::INVERTED;
- index1._col_unique_ids = {1};
- schema->append_index(std::move(index1));
-
- TabletIndex index2;
- index2._index_id = 101;
- index2._index_type = IndexType::INVERTED;
- index2._col_unique_ids = {1};
- schema->append_index(std::move(index2));
-
- // Create two updates
- std::vector<TabletIndex> updates;
- TabletIndex updated1;
- updated1._index_id = 102;
- updated1._index_type = IndexType::INVERTED;
- updated1._col_unique_ids = {1};
- std::map<std::string, std::string> properties1 = {{"updated_key1",
"updated_value"}};
- updated1._properties = properties1;
- updates.push_back(updated1);
-
- TabletIndex updated2;
- updated2._index_id = 103;
- updated2._index_type = IndexType::INVERTED;
- updated2._col_unique_ids = {1};
- std::map<std::string, std::string> properties2 = {{"updated_key2",
"updated_value"}};
- updated2._properties = properties2;
- updates.push_back(updated2);
-
- TabletColumn col1;
- col1._unique_id = 1;
-
- // Update the indexes
- schema->update_index(col1, IndexType::INVERTED, std::move(updates));
-
- // Verify both indexes were updated
- ASSERT_EQ(schema->_indexes[0]->_index_id, 102);
- ASSERT_EQ(schema->_indexes[1]->_index_id, 103);
- ASSERT_EQ(schema->_indexes[0]->_properties, properties1);
- ASSERT_EQ(schema->_indexes[1]->_properties, properties2);
-}
-
-TEST_F(TabletSchemaMultiIndexTest, UpdateIndexWithMismatchedCount) {
- // Setup one index
- TabletIndex index;
- index._index_id = 100;
- index._index_type = IndexType::INVERTED;
- index._col_unique_ids = {1};
- schema->append_index(std::move(index));
-
- // Try to update with two indexes (should fail)
- std::vector<TabletIndex> updates;
- TabletIndex updated1;
- updated1._index_id = 101;
- updated1._index_type = IndexType::INVERTED;
- updated1._col_unique_ids = {1};
- std::map<std::string, std::string> properties1 = {{"updated_key1",
"updated_value"}};
- updated1._properties = properties1;
- updates.push_back(updated1);
-
- TabletIndex updated2;
- updated2._index_id = 102;
- updated2._index_type = IndexType::INVERTED;
- updated2._col_unique_ids = {1};
- updates.push_back(updated2);
-
- TabletColumn col1;
- col1._unique_id = 1;
-
- schema->update_index(col1, IndexType::INVERTED, std::move(updates));
-
- // Verify the index was NOT updated
- ASSERT_NE(schema->_indexes[0]->_properties, properties1);
-}
-
TEST_F(TabletSchemaMultiIndexTest, RemoveIndex) {
// Setup two indexes
TabletIndex index1;
diff --git a/be/test/vec/common/schema_util_test.cpp
b/be/test/vec/common/schema_util_test.cpp
index 3fa9037c72e..5dcdf53df06 100644
--- a/be/test/vec/common/schema_util_test.cpp
+++ b/be/test/vec/common/schema_util_test.cpp
@@ -308,61 +308,6 @@ TEST_F(SchemaUtilTest, test_multiple_index_inheritance) {
}
}
-TEST_F(SchemaUtilTest, test_index_update_logic) {
- TabletSchemaPB schema_pb;
- schema_pb.set_keys_type(KeysType::DUP_KEYS);
-
schema_pb.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V2);
-
- construct_column(schema_pb.add_column(), schema_pb.add_index(), 10000,
"v1_index_orig1", 1,
- "VARIANT", "v1", IndexType::INVERTED);
- construct_column(schema_pb.add_column(), schema_pb.add_index(), 10001,
"v1_index_orig2", 1,
- "VARIANT", "v1", IndexType::INVERTED);
-
- TabletSchemaSPtr tablet_schema = std::make_shared<TabletSchema>();
- tablet_schema->init_from_pb(schema_pb);
- std::vector<TabletColumn> subcolumns;
-
- construct_subcolumn(tablet_schema, FieldType::OLAP_FIELD_TYPE_STRING, 1,
"v1.name",
- &subcolumns);
- vectorized::schema_util::inherit_column_attributes(tablet_schema);
-
- const auto& subcol = subcolumns[0];
- auto initial_indexes = tablet_schema->inverted_indexs(subcol);
- ASSERT_EQ(initial_indexes.size(), 2);
- EXPECT_EQ(initial_indexes[0]->index_name(), "v1_index_orig1");
- EXPECT_EQ(initial_indexes[1]->index_name(), "v1_index_orig2");
-
- std::vector<TabletIndex> updated_indexes;
- TabletIndexPB tablet_index_pb1;
- tablet_index_pb1.set_index_id(10002);
- tablet_index_pb1.set_index_name("v1_index_updated1");
- tablet_index_pb1.set_index_type(IndexType::INVERTED);
- tablet_index_pb1.add_col_unique_id(1);
- TabletIndex tablet_index1;
- tablet_index1.init_from_pb(tablet_index_pb1);
- updated_indexes.emplace_back(std::move(tablet_index1));
-
- TabletIndexPB tablet_index_pb2;
- tablet_index_pb2.set_index_id(10003);
- tablet_index_pb2.set_index_name("v1_index_updated2");
- tablet_index_pb2.set_index_type(IndexType::INVERTED);
- tablet_index_pb2.add_col_unique_id(1);
- TabletIndex tablet_index2;
- tablet_index2.init_from_pb(tablet_index_pb2);
- updated_indexes.emplace_back(std::move(tablet_index2));
-
- tablet_schema->update_index(tablet_schema->column(1), IndexType::INVERTED,
- std::move(updated_indexes));
-
- vectorized::schema_util::inherit_column_attributes(tablet_schema);
- auto updated_subcol_indexes = tablet_schema->inverted_indexs(subcol);
-
- EXPECT_EQ(updated_subcol_indexes.size(), 2);
- EXPECT_EQ(updated_subcol_indexes[0]->index_name(), "v1_index_updated1");
- EXPECT_EQ(updated_subcol_indexes[1]->index_name(), "v1_index_updated2");
- EXPECT_EQ(updated_subcol_indexes[0]->get_index_suffix(), "v1%2Ename");
-}
-
static std::unordered_map<std::string, int>
construct_column_map_with_random_values(
auto& column_map, int key_size, int value_size, const std::string&
prefix) {
std::unordered_map<std::string, int> key_value_counts;
diff --git a/regression-test/suites/variant_p0/with_index/var_index.groovy
b/regression-test/suites/variant_p0/with_index/var_index.groovy
index 4bac3f6ac1c..00dcfe8497d 100644
--- a/regression-test/suites/variant_p0/with_index/var_index.groovy
+++ b/regression-test/suites/variant_p0/with_index/var_index.groovy
@@ -144,4 +144,100 @@ suite("regression_test_variant_var_index", "p0,
nonConcurrent"){
}
}
+
+ sql """ DROP TABLE IF EXISTS ${table_name} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${table_name} (
+ k bigint,
+ v variant<properties("variant_max_subcolumns_count" = "0")>,
+ INDEX idx_var(v) USING INVERTED PROPERTIES("parser" = "english")
COMMENT '',
+ INDEX idx_var2(v) USING INVERTED
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 1
+ properties("replication_num" = "1", "disable_auto_compaction" = "true");
+ """
+
+ sql """insert into var_index values(1, '{"name": "张三", "age": 18}')"""
+ sql """ select * from var_index """
+
+ def timeout = 60000
+ def delta_time = 1000
+ def alter_res = "null"
+ def useTime = 0
+
+ def wait_for_latest_op_on_table_finish = { table, OpTimeout ->
+ for(int t = delta_time; t <= OpTimeout; t += delta_time){
+ alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName =
"${table}" ORDER BY CreateTime DESC LIMIT 1;"""
+ alter_res = alter_res.toString()
+ if(alter_res.contains("FINISHED")) {
+ sleep(3000) // wait change table state to normal
+ logger.info(table + " latest alter job finished, detail: " +
alter_res)
+ break
+ }
+ useTime = t
+ sleep(delta_time)
+ }
+ assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish
timeout")
+ }
+
+ def get_indeces_count = {
+ def tablets = sql_return_maparray """ show tablets from ${table_name};
"""
+
+ def backendId_to_backendIP = [:]
+ def backendId_to_backendHttpPort = [:]
+ getBackendIpHttpPort(backendId_to_backendIP,
backendId_to_backendHttpPort);
+
+ String tablet_id = tablets[0].TabletId
+ String backend_id = tablets[0].BackendId
+ String ip = backendId_to_backendIP.get(backend_id)
+ String port = backendId_to_backendHttpPort.get(backend_id)
+ def (code, out, err) = http_client("GET",
String.format("http://%s:%s/api/show_nested_index_file?tablet_id=%s";, ip, port,
tablet_id))
+ logger.info("Run show_nested_index_file_on_tablet: code=" + code + ",
out=" + out + ", err=" + err)
+ if (parseJson(out.trim()).status == "E-6003") {
+ return 0
+ }
+ def rowset_count = parseJson(out.trim()).rowsets.size();
+ def indices_count = 0
+ for (def rowset in parseJson(out.trim()).rowsets) {
+ for (int i = 0; i < rowset.segments.size(); i++) {
+ def segment = rowset.segments[i]
+ assertEquals(i, segment.segment_id)
+ if (segment.indices != null) {
+ indices_count += segment.indices.size()
+ }
+ }
+ }
+ return indices_count
+ }
+
+ assertEquals(3, get_indeces_count())
+
+ sql """ drop index idx_var2 on ${table_name} """
+ wait_for_latest_op_on_table_finish(table_name, timeout)
+ sql """ insert into ${table_name} values(2, '{"name": "李四", "age": 20}')
"""
+ sql """ select * from ${table_name} """
+ if (isCloudMode()) {
+ assertEquals(4, get_indeces_count())
+ } else {
+ assertEquals(5, get_indeces_count())
+ }
+
+
+ sql """ insert into ${table_name} values(2, '{"name": "李四", "age": 20}')
"""
+ sql """ insert into ${table_name} values(2, '{"name": "李四", "age": 20}')
"""
+ sql """ insert into ${table_name} values(2, '{"name": "李四", "age": 20}')
"""
+ sql """ select * from ${table_name} """
+ if (isCloudMode()) {
+ assertEquals(10, get_indeces_count())
+ } else {
+ assertEquals(11, get_indeces_count())
+ }
+
+ sql """ drop index idx_var on ${table_name} """
+ wait_for_latest_op_on_table_finish(table_name, timeout)
+ sql """ insert into ${table_name} values(2, '{"name": "李四", "age": 20}')
"""
+ sql """ select * from ${table_name} """
+ trigger_and_wait_compaction(table_name, "full")
+ assertEquals(0, get_indeces_count())
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
