This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit bdbe4afbdc764429f521aad26539bc7bef8311b8 Author: YueW <45946325+tany...@users.noreply.github.com> AuthorDate: Fri Jul 21 23:02:39 2023 +0800 [opt](inverted index) support the same column create different type index (#21972) --- be/src/olap/rowset/segment_v2/segment_iterator.cpp | 1 + .../apache/doris/alter/SchemaChangeHandler.java | 1 + .../org/apache/doris/analysis/CreateTableStmt.java | 9 +- .../doris/catalog/MaterializedIndexMeta.java | 4 + .../inverted_index_p0/test_add_drop_index.groovy | 2 +- .../inverted_index_p0/test_create_index_1.groovy | 182 +++++++++++++++++++++ .../inverted_index_p0/test_create_index_2.groovy | 176 ++++++++++++++++++++ 7 files changed, 371 insertions(+), 4 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index 1c6bfbadb0..59a44a0d2d 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -794,6 +794,7 @@ bool SegmentIterator::_downgrade_without_index(Status res, bool need_remaining) // such as: where A = '' and B = ',' // the predicate of A and B need downgrade without index query. // above case can downgrade without index query + LOG(INFO) << "will downgrade without index to evaluate predicate, because of res: " << res; return true; } return false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index ff5d54f8a5..e949c5c2c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -2676,6 +2676,7 @@ public class SchemaChangeHandler extends AlterHandler { } } currentIndexMeta.setMaxColUniqueId(maxColUniqueId); + currentIndexMeta.setIndexes(indexes); } olapTable.setIndexes(indexes); olapTable.rebuildFullSchema(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java index 556bc03a03..ef8bec00f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java @@ -17,6 +17,7 @@ package org.apache.doris.analysis; +import org.apache.doris.analysis.IndexDef.IndexType; import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DistributionInfo; @@ -31,6 +32,7 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeConstants; import org.apache.doris.common.FeNameFormat; +import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.util.AutoBucketUtils; import org.apache.doris.common.util.ParseUtil; @@ -588,7 +590,7 @@ public class CreateTableStmt extends DdlStmt { if (CollectionUtils.isNotEmpty(indexDefs)) { Set<String> distinct = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - Set<List<String>> distinctCol = new HashSet<>(); + Set<Pair<IndexType, List<String>>> distinctCol = new HashSet<>(); for (IndexDef indexDef : indexDefs) { indexDef.analyze(); @@ -613,13 +615,14 @@ public class CreateTableStmt extends DdlStmt { indexDef.getColumns(), indexDef.getIndexType(), indexDef.getProperties(), indexDef.getComment())); distinct.add(indexDef.getIndexName()); - distinctCol.add(indexDef.getColumns().stream().map(String::toUpperCase).collect(Collectors.toList())); + distinctCol.add(Pair.of(indexDef.getIndexType(), + indexDef.getColumns().stream().map(String::toUpperCase).collect(Collectors.toList()))); } if (distinct.size() != indexes.size()) { throw new AnalysisException("index name must be unique."); } if (distinctCol.size() != indexes.size()) { - throw new AnalysisException("same index columns have multiple index name is not allowed."); + throw new AnalysisException("same index columns have multiple same type index is not allowed."); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java index 12cab9f7ec..06f8aa6318 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java @@ -137,6 +137,10 @@ public class MaterializedIndexMeta implements Writable, GsonPostProcessable { return indexes != null ? indexes : Lists.newArrayList(); } + public void setIndexes(List<Index> newIndexes) { + this.indexes = newIndexes; + } + public List<Column> getSchema() { return getSchema(true); } diff --git a/regression-test/suites/inverted_index_p0/test_add_drop_index.groovy b/regression-test/suites/inverted_index_p0/test_add_drop_index.groovy index 243f05ec2d..9bdff40418 100644 --- a/regression-test/suites/inverted_index_p0/test_add_drop_index.groovy +++ b/regression-test/suites/inverted_index_p0/test_add_drop_index.groovy @@ -85,7 +85,7 @@ suite("test_add_drop_index", "inverted_index"){ logger.info("create same duplicate with different name index, result: " + ex) } assertEquals(create_dup_index_result, "fail") - // case1.3 create duplicate different index for one colume with same name + // case1.3 create duplicate different index for one colume with different name sql "create index age_idx_diff on ${indexTbName1}(`age`) using bitmap" wait_for_latest_op_on_table_finish(indexTbName1, timeout) show_result = sql "show index from ${indexTbName1}" diff --git a/regression-test/suites/inverted_index_p0/test_create_index_1.groovy b/regression-test/suites/inverted_index_p0/test_create_index_1.groovy new file mode 100644 index 0000000000..49bd5033dd --- /dev/null +++ b/regression-test/suites/inverted_index_p0/test_create_index_1.groovy @@ -0,0 +1,182 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +suite("test_create_index_1", "inverted_index"){ + // prepare test table + def timeout = 60000 + def delta_time = 1000 + def alter_res = "null" + def useTime = 0 + def wait_for_latest_op_on_table_finish = { table_name, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${table_name}" ORDER BY CreateTime DESC LIMIT 1;""" + alter_res = alter_res.toString() + if(alter_res.contains("FINISHED")) { + sleep(3000) // wait change table state to normal + logger.info(table_name + " latest alter job finished, detail: " + alter_res) + break + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") + } + + def indexTbName1 = "test_create_index_1" + + sql "DROP TABLE IF EXISTS ${indexTbName1}" + // case 1: create table with index + // case 1.1: create duplicate same index for one colume with same name + def create_dup_index_result = "fail" + try { + sql """ + CREATE TABLE IF NOT EXISTS ${indexTbName1} ( + name varchar(50), + age int NOT NULL, + grade int NOT NULL, + registDate datetime NULL, + studentInfo char(100), + tearchComment string, + INDEX age_idx(age) USING INVERTED COMMENT 'age index', + INDEX age_idx(age) USING INVERTED COMMENT 'age index' + ) + DUPLICATE KEY(`name`) + DISTRIBUTED BY HASH(`name`) BUCKETS 10 + properties("replication_num" = "1"); + """ + create_dup_index_result = "success" + } catch(Exception ex) { + logger.info("create duplicate same index for one colume with same name, result: " + ex) + } + assertEquals(create_dup_index_result, "fail") + + // case 1.2: create duplicate same index for one colume with different name + try { + sql """ + CREATE TABLE IF NOT EXISTS ${indexTbName1} ( + name varchar(50), + age int NOT NULL, + grade int NOT NULL, + registDate datetime NULL, + studentInfo char(100), + tearchComment string, + INDEX age_idx_1(age) USING INVERTED COMMENT 'age index', + INDEX age_idx_2(age) USING INVERTED COMMENT 'age index' + ) + DUPLICATE KEY(`name`) + DISTRIBUTED BY HASH(`name`) BUCKETS 10 + properties("replication_num" = "1"); + """ + create_dup_index_result = "success" + } catch(Exception ex) { + logger.info("create duplicate same index for one colume with different name, result: " + ex) + } + assertEquals(create_dup_index_result, "fail") + + // case 1.3: create duplicate different index for one colume with same name + try { + sql """ + CREATE TABLE IF NOT EXISTS ${indexTbName1} ( + name varchar(50), + age int NOT NULL, + grade int NOT NULL, + registDate datetime NULL, + studentInfo char(100), + tearchComment string, + INDEX age_idx(age) USING BITMAP COMMENT 'age index', + INDEX age_idx(age) USING INVERTED COMMENT 'age index' + ) + DUPLICATE KEY(`name`) + DISTRIBUTED BY HASH(`name`) BUCKETS 10 + properties("replication_num" = "1"); + """ + create_dup_index_result = "success" + } catch(Exception ex) { + logger.info("create duplicate different index for one colume with same name, result: " + ex) + } + assertEquals(create_dup_index_result, "fail") + + // case 1.4: create duplicate different index for one colume with different name + sql """ + CREATE TABLE IF NOT EXISTS ${indexTbName1} ( + name varchar(50), + age int NOT NULL, + grade int NOT NULL, + registDate datetime NULL, + studentInfo char(100), + tearchComment string, + INDEX age_idx_1(age) USING BITMAP COMMENT 'age index', + INDEX age_idx_2(age) USING INVERTED COMMENT 'age index' + ) + DUPLICATE KEY(`name`) + DISTRIBUTED BY HASH(`name`) BUCKETS 10 + properties("replication_num" = "1"); + """ + + def show_result = sql "show index from ${indexTbName1}" + logger.info("show index from " + indexTbName1 + " result: " + show_result) + assertEquals(show_result.size(), 2) + assertEquals(show_result[0][2], "age_idx_1") + assertEquals(show_result[1][2], "age_idx_2") + + // drop index + sql "drop index age_idx_1 on ${indexTbName1}" + wait_for_latest_op_on_table_finish(indexTbName1, timeout) + sql "drop index age_idx_2 on ${indexTbName1}" + show_result = sql "show index from ${indexTbName1}" + assertEquals(show_result.size(), 0) + + // case 2: alter add index + sql "create index age_idx on ${indexTbName1}(age) using inverted" + show_result = sql "show index from ${indexTbName1}" + logger.info("show index from " + indexTbName1 + " result: " + show_result) + assertEquals(show_result[0][2], "age_idx") + // case 2.1: create duplicate same index for one colume with same name + try { + sql "create index age_idx on ${indexTbName1}(`age`) using inverted" + create_dup_index_result = "success" + } catch(Exception ex) { + logger.info("create duplicate same index for one colume with same name, result: " + ex) + } + + // case 2.2: create duplicate same index for one colume with different name + try { + sql "create index age_idx_2 on ${indexTbName1}(`age`) using inverted" + create_dup_index_result = "success" + } catch(Exception ex) { + logger.info("create duplicate same index for one colume with different name, result: " + ex) + } + + // case 2.3: create duplicate different index for one colume with same name + try { + sql "create index age_idx on ${indexTbName1}(`age`) using bitmap" + create_dup_index_result = "success" + } catch(Exception ex) { + logger.info("create duplicate different index for one colume with same name, result: " + ex) + } + + // 2.4: create duplicate different index for one colume with different name + sql "create index age_idx_2 on ${indexTbName1}(`age`) using bitmap" + wait_for_latest_op_on_table_finish(indexTbName1, timeout) + + show_result = sql "show index from ${indexTbName1}" + logger.info("show index from " + indexTbName1 + " result: " + show_result) + assertEquals(show_result.size(), 2) + assertEquals(show_result[0][2], "age_idx") + assertEquals(show_result[1][2], "age_idx_2") +} diff --git a/regression-test/suites/inverted_index_p0/test_create_index_2.groovy b/regression-test/suites/inverted_index_p0/test_create_index_2.groovy new file mode 100644 index 0000000000..a9aa616971 --- /dev/null +++ b/regression-test/suites/inverted_index_p0/test_create_index_2.groovy @@ -0,0 +1,176 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +suite("test_create_index_2", "inverted_index"){ + // prepare test table + def timeout = 60000 + def delta_time = 1000 + def alter_res = "null" + def useTime = 0 + def wait_for_latest_op_on_table_finish = { table_name, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${table_name}" ORDER BY CreateTime DESC LIMIT 1;""" + alter_res = alter_res.toString() + if(alter_res.contains("FINISHED")) { + sleep(3000) // wait change table state to normal + logger.info(table_name + " latest alter job finished, detail: " + alter_res) + break + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") + } + + def indexTbName1 = "test_create_index_2" + + sql "DROP TABLE IF EXISTS ${indexTbName1}" + // case 1: create table with index + // case 1.1: create duplicate same index for one colume with same name + def create_dup_index_result = "fail" + try { + sql """ + CREATE TABLE IF NOT EXISTS ${indexTbName1} ( + id INT DEFAULT '10', + name VARCHAR(32) DEFAULT '', + INDEX name_idx(name) USING INVERTED PROPERTIES("parser" = "english") COMMENT 'name index', + INDEX name_idx(name) USING INVERTED PROPERTIES("parser" = "english") COMMENT 'name index' + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES("replication_num" = "1"); + """ + create_dup_index_result = "success" + } catch(Exception ex) { + logger.info("create duplicate same index for one colume with same name, result: " + ex) + } + assertEquals(create_dup_index_result, "fail") + + // case 1.2: create duplicate same index for one colume with different name + try { + sql """ + CREATE TABLE IF NOT EXISTS ${indexTbName1} ( + id INT DEFAULT '10', + name VARCHAR(32) DEFAULT '', + INDEX name_idx_1(name) USING INVERTED PROPERTIES("parser" = "english") COMMENT 'name index', + INDEX name_idx_2(name) USING INVERTED PROPERTIES("parser" = "english") COMMENT 'name index' + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES("replication_num" = "1"); + """ + create_dup_index_result = "success" + } catch(Exception ex) { + logger.info("create duplicate same index for one colume with different name, result: " + ex) + } + assertEquals(create_dup_index_result, "fail") + + // case 1.3: create duplicate different index for one colume with same name + try { + sql """ + CREATE TABLE IF NOT EXISTS ${indexTbName1} ( + id INT DEFAULT '10', + name VARCHAR(32) DEFAULT '', + INDEX name_idx(name) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256") COMMENT 'name index', + INDEX name_idx(name) USING INVERTED PROPERTIES("parser" = "english") COMMENT 'name index' + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES("replication_num" = "1"); + """ + create_dup_index_result = "success" + } catch(Exception ex) { + logger.info("create duplicate different index for one colume with same name, result: " + ex) + } + assertEquals(create_dup_index_result, "fail") + + // case 1.4: create duplicate different index for one colume with different name + sql """ + CREATE TABLE IF NOT EXISTS ${indexTbName1} ( + id INT DEFAULT '10', + name VARCHAR(32) DEFAULT '', + INDEX name_idx_1(name) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256") COMMENT 'name index', + INDEX name_idx_2(name) USING INVERTED PROPERTIES("parser" = "english") COMMENT 'name index' + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES("replication_num" = "1"); + """ + + def show_result = sql "show index from ${indexTbName1}" + logger.info("show index from " + indexTbName1 + " result: " + show_result) + assertEquals(show_result.size(), 2) + assertEquals(show_result[0][2], "name_idx_1") + assertEquals(show_result[1][2], "name_idx_2") + + // drop index + sql "drop index name_idx_1 on ${indexTbName1}" + wait_for_latest_op_on_table_finish(indexTbName1, timeout) + sql "drop index name_idx_2 on ${indexTbName1}" + show_result = sql "show index from ${indexTbName1}" + assertEquals(show_result.size(), 0) + + // case 2: alter add index + sql """ + create index name_idx on ${indexTbName1}(name) using inverted properties("parser" = "english") comment 'name index'; + """ + show_result = sql "show index from ${indexTbName1}" + logger.info("show index from " + indexTbName1 + " result: " + show_result) + assertEquals(show_result[0][2], "name_idx") + // case 2.1: create duplicate same index for one colume with same name + try { + sql """ + create index name_idx on ${indexTbName1}(name) using inverted properties("parser" = "english") comment 'name index'; + """ + create_dup_index_result = "success" + } catch(Exception ex) { + logger.info("create duplicate same index for one colume with same name, result: " + ex) + } + + // case 2.2: create duplicate same index for one colume with different name + try { + sql """ + create index name_idx_2 on ${indexTbName1}(name) using inverted properties("parser" = "english") comment 'name index'; + """ + create_dup_index_result = "success" + } catch(Exception ex) { + logger.info("create duplicate same index for one colume with different name, result: " + ex) + } + + // case 2.3: create duplicate different index for one colume with same name + try { + sql """ + create index name_idx on ${indexTbName1}(name) using ngram_bf properties("gram_size"="3", "bf_size"="256") comment 'name index'; + """ + create_dup_index_result = "success" + } catch(Exception ex) { + logger.info("create duplicate different index for one colume with same name, result: " + ex) + } + + // 2.4: create duplicate different index for one colume with different name + sql """ + create index name_idx_2 on ${indexTbName1}(name) using ngram_bf properties("gram_size"="3", "bf_size"="256") comment 'name index'; + """ + wait_for_latest_op_on_table_finish(indexTbName1, timeout) + + show_result = sql "show index from ${indexTbName1}" + logger.info("show index from " + indexTbName1 + " result: " + show_result) + assertEquals(show_result.size(), 2) + assertEquals(show_result[0][2], "name_idx") + assertEquals(show_result[1][2], "name_idx_2") +} --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org