This is an automated email from the ASF dual-hosted git repository.

kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git

commit bdbe4afbdc764429f521aad26539bc7bef8311b8
Author: YueW <45946325+tany...@users.noreply.github.com>
AuthorDate: Fri Jul 21 23:02:39 2023 +0800

    [opt](inverted index) support the same column create different type index 
(#21972)
---
 be/src/olap/rowset/segment_v2/segment_iterator.cpp |   1 +
 .../apache/doris/alter/SchemaChangeHandler.java    |   1 +
 .../org/apache/doris/analysis/CreateTableStmt.java |   9 +-
 .../doris/catalog/MaterializedIndexMeta.java       |   4 +
 .../inverted_index_p0/test_add_drop_index.groovy   |   2 +-
 .../inverted_index_p0/test_create_index_1.groovy   | 182 +++++++++++++++++++++
 .../inverted_index_p0/test_create_index_2.groovy   | 176 ++++++++++++++++++++
 7 files changed, 371 insertions(+), 4 deletions(-)

diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp 
b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 1c6bfbadb0..59a44a0d2d 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -794,6 +794,7 @@ bool SegmentIterator::_downgrade_without_index(Status res, 
bool need_remaining)
         //    such as: where A = '' and B = ','
         //    the predicate of A and B need downgrade without index query.
         // above case can downgrade without index query
+        LOG(INFO) << "will downgrade without index to evaluate predicate, 
because of res: " << res;
         return true;
     }
     return false;
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java 
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index ff5d54f8a5..e949c5c2c2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -2676,6 +2676,7 @@ public class SchemaChangeHandler extends AlterHandler {
                 }
             }
             currentIndexMeta.setMaxColUniqueId(maxColUniqueId);
+            currentIndexMeta.setIndexes(indexes);
         }
         olapTable.setIndexes(indexes);
         olapTable.rebuildFullSchema();
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java 
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
index 556bc03a03..ef8bec00f3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
@@ -17,6 +17,7 @@
 
 package org.apache.doris.analysis;
 
+import org.apache.doris.analysis.IndexDef.IndexType;
 import org.apache.doris.catalog.AggregateType;
 import org.apache.doris.catalog.Column;
 import org.apache.doris.catalog.DistributionInfo;
@@ -31,6 +32,7 @@ import org.apache.doris.common.ErrorCode;
 import org.apache.doris.common.ErrorReport;
 import org.apache.doris.common.FeConstants;
 import org.apache.doris.common.FeNameFormat;
+import org.apache.doris.common.Pair;
 import org.apache.doris.common.UserException;
 import org.apache.doris.common.util.AutoBucketUtils;
 import org.apache.doris.common.util.ParseUtil;
@@ -588,7 +590,7 @@ public class CreateTableStmt extends DdlStmt {
 
         if (CollectionUtils.isNotEmpty(indexDefs)) {
             Set<String> distinct = new 
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
-            Set<List<String>> distinctCol = new HashSet<>();
+            Set<Pair<IndexType, List<String>>> distinctCol = new HashSet<>();
 
             for (IndexDef indexDef : indexDefs) {
                 indexDef.analyze();
@@ -613,13 +615,14 @@ public class CreateTableStmt extends DdlStmt {
                         indexDef.getColumns(), indexDef.getIndexType(),
                         indexDef.getProperties(), indexDef.getComment()));
                 distinct.add(indexDef.getIndexName());
-                
distinctCol.add(indexDef.getColumns().stream().map(String::toUpperCase).collect(Collectors.toList()));
+                distinctCol.add(Pair.of(indexDef.getIndexType(),
+                        
indexDef.getColumns().stream().map(String::toUpperCase).collect(Collectors.toList())));
             }
             if (distinct.size() != indexes.size()) {
                 throw new AnalysisException("index name must be unique.");
             }
             if (distinctCol.size() != indexes.size()) {
-                throw new AnalysisException("same index columns have multiple 
index name is not allowed.");
+                throw new AnalysisException("same index columns have multiple 
same type index is not allowed.");
             }
         }
     }
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java 
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java
index 12cab9f7ec..06f8aa6318 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndexMeta.java
@@ -137,6 +137,10 @@ public class MaterializedIndexMeta implements Writable, 
GsonPostProcessable {
         return indexes != null ? indexes : Lists.newArrayList();
     }
 
+    public void setIndexes(List<Index> newIndexes) {
+        this.indexes = newIndexes;
+    }
+
     public List<Column> getSchema() {
         return getSchema(true);
     }
diff --git 
a/regression-test/suites/inverted_index_p0/test_add_drop_index.groovy 
b/regression-test/suites/inverted_index_p0/test_add_drop_index.groovy
index 243f05ec2d..9bdff40418 100644
--- a/regression-test/suites/inverted_index_p0/test_add_drop_index.groovy
+++ b/regression-test/suites/inverted_index_p0/test_add_drop_index.groovy
@@ -85,7 +85,7 @@ suite("test_add_drop_index", "inverted_index"){
         logger.info("create same duplicate with different name index,  result: 
" + ex)
     }
     assertEquals(create_dup_index_result, "fail")
-    // case1.3 create duplicate different index for one colume with same name
+    // case1.3 create duplicate different index for one colume with different 
name
     sql "create index age_idx_diff on ${indexTbName1}(`age`) using bitmap"
     wait_for_latest_op_on_table_finish(indexTbName1, timeout)
     show_result = sql "show index from ${indexTbName1}"
diff --git 
a/regression-test/suites/inverted_index_p0/test_create_index_1.groovy 
b/regression-test/suites/inverted_index_p0/test_create_index_1.groovy
new file mode 100644
index 0000000000..49bd5033dd
--- /dev/null
+++ b/regression-test/suites/inverted_index_p0/test_create_index_1.groovy
@@ -0,0 +1,182 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+suite("test_create_index_1", "inverted_index"){
+    // prepare test table
+    def timeout = 60000
+    def delta_time = 1000
+    def alter_res = "null"
+    def useTime = 0
+    def wait_for_latest_op_on_table_finish = { table_name, OpTimeout ->
+        for(int t = delta_time; t <= OpTimeout; t += delta_time){
+            alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = 
"${table_name}" ORDER BY CreateTime DESC LIMIT 1;"""
+            alter_res = alter_res.toString()
+            if(alter_res.contains("FINISHED")) {
+                sleep(3000) // wait change table state to normal
+                logger.info(table_name + " latest alter job finished, detail: 
" + alter_res)
+                break
+            }
+            useTime = t
+            sleep(delta_time)
+        }
+        assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish 
timeout")
+    }
+
+    def indexTbName1 = "test_create_index_1"
+
+    sql "DROP TABLE IF EXISTS ${indexTbName1}"
+    // case 1: create table with index
+    // case 1.1: create duplicate same index for one colume with same name
+    def create_dup_index_result = "fail"
+    try {
+        sql """
+                CREATE TABLE IF NOT EXISTS ${indexTbName1} (
+                    name varchar(50),
+                    age int NOT NULL,
+                    grade int NOT NULL,
+                    registDate datetime NULL,
+                    studentInfo char(100),
+                    tearchComment string,
+                    INDEX age_idx(age) USING INVERTED COMMENT 'age index',
+                    INDEX age_idx(age) USING INVERTED COMMENT 'age index'
+                )
+                DUPLICATE KEY(`name`)
+                DISTRIBUTED BY HASH(`name`) BUCKETS 10
+                properties("replication_num" = "1");
+        """
+        create_dup_index_result = "success"
+    } catch(Exception ex) {
+        logger.info("create duplicate same index for one colume with same 
name, result: " + ex)
+    }
+    assertEquals(create_dup_index_result, "fail")
+
+    // case 1.2: create duplicate same index for one colume with different name
+    try {
+        sql """
+                CREATE TABLE IF NOT EXISTS ${indexTbName1} (
+                    name varchar(50),
+                    age int NOT NULL,
+                    grade int NOT NULL,
+                    registDate datetime NULL,
+                    studentInfo char(100),
+                    tearchComment string,
+                    INDEX age_idx_1(age) USING INVERTED COMMENT 'age index',
+                    INDEX age_idx_2(age) USING INVERTED COMMENT 'age index'
+                )
+                DUPLICATE KEY(`name`)
+                DISTRIBUTED BY HASH(`name`) BUCKETS 10
+                properties("replication_num" = "1");
+        """
+        create_dup_index_result = "success"
+    } catch(Exception ex) {
+        logger.info("create duplicate same index for one colume with different 
name, result: " + ex)
+    }
+    assertEquals(create_dup_index_result, "fail")
+
+    // case 1.3: create duplicate different index for one colume with same name
+    try {
+        sql """
+                CREATE TABLE IF NOT EXISTS ${indexTbName1} (
+                    name varchar(50),
+                    age int NOT NULL,
+                    grade int NOT NULL,
+                    registDate datetime NULL,
+                    studentInfo char(100),
+                    tearchComment string,
+                    INDEX age_idx(age) USING BITMAP COMMENT 'age index',
+                    INDEX age_idx(age) USING INVERTED COMMENT 'age index'
+                )
+                DUPLICATE KEY(`name`)
+                DISTRIBUTED BY HASH(`name`) BUCKETS 10
+                properties("replication_num" = "1");
+        """
+        create_dup_index_result = "success"
+    } catch(Exception ex) {
+        logger.info("create duplicate different index for one colume with same 
name, result: " + ex)
+    }
+    assertEquals(create_dup_index_result, "fail")
+
+    // case 1.4: create duplicate different index for one colume with 
different name
+    sql """
+            CREATE TABLE IF NOT EXISTS ${indexTbName1} (
+                name varchar(50),
+                age int NOT NULL,
+                grade int NOT NULL,
+                registDate datetime NULL,
+                studentInfo char(100),
+                tearchComment string,
+                INDEX age_idx_1(age) USING BITMAP COMMENT 'age index',
+                INDEX age_idx_2(age) USING INVERTED COMMENT 'age index'
+            )
+            DUPLICATE KEY(`name`)
+            DISTRIBUTED BY HASH(`name`) BUCKETS 10
+            properties("replication_num" = "1");
+    """
+
+    def show_result = sql "show index from ${indexTbName1}"
+    logger.info("show index from " + indexTbName1 + " result: " + show_result)
+    assertEquals(show_result.size(), 2)
+    assertEquals(show_result[0][2], "age_idx_1")
+    assertEquals(show_result[1][2], "age_idx_2")
+    
+    // drop index
+    sql "drop index age_idx_1 on ${indexTbName1}"
+    wait_for_latest_op_on_table_finish(indexTbName1, timeout)
+    sql "drop index age_idx_2 on ${indexTbName1}"
+    show_result = sql "show index from ${indexTbName1}"
+    assertEquals(show_result.size(), 0)
+
+    // case 2: alter add index
+    sql "create index age_idx on ${indexTbName1}(age) using inverted"
+    show_result = sql "show index from ${indexTbName1}"
+    logger.info("show index from " + indexTbName1 + " result: " + show_result)
+    assertEquals(show_result[0][2], "age_idx")
+    // case 2.1: create duplicate same index for one colume with same name
+    try {
+        sql "create index age_idx on ${indexTbName1}(`age`) using inverted"
+        create_dup_index_result = "success"
+    } catch(Exception ex) {
+        logger.info("create duplicate same index for one colume with same 
name, result: " + ex)
+    }
+
+    // case 2.2: create duplicate same index for one colume with different name
+    try {
+        sql "create index age_idx_2 on ${indexTbName1}(`age`) using inverted"
+        create_dup_index_result = "success"
+    } catch(Exception ex) {
+        logger.info("create duplicate same index for one colume with different 
name, result: " + ex)
+    }
+
+    // case 2.3: create duplicate different index for one colume with same name
+    try {
+        sql "create index age_idx on ${indexTbName1}(`age`) using bitmap"
+        create_dup_index_result = "success"
+    } catch(Exception ex) {
+        logger.info("create duplicate different index for one colume with same 
name, result: " + ex)
+    }
+
+    // 2.4: create duplicate different index for one colume with different name
+    sql "create index age_idx_2 on ${indexTbName1}(`age`) using bitmap"
+    wait_for_latest_op_on_table_finish(indexTbName1, timeout)
+
+    show_result = sql "show index from ${indexTbName1}"
+    logger.info("show index from " + indexTbName1 + " result: " + show_result)
+    assertEquals(show_result.size(), 2)
+    assertEquals(show_result[0][2], "age_idx")
+    assertEquals(show_result[1][2], "age_idx_2")
+}
diff --git 
a/regression-test/suites/inverted_index_p0/test_create_index_2.groovy 
b/regression-test/suites/inverted_index_p0/test_create_index_2.groovy
new file mode 100644
index 0000000000..a9aa616971
--- /dev/null
+++ b/regression-test/suites/inverted_index_p0/test_create_index_2.groovy
@@ -0,0 +1,176 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+suite("test_create_index_2", "inverted_index"){
+    // prepare test table
+    def timeout = 60000
+    def delta_time = 1000
+    def alter_res = "null"
+    def useTime = 0
+    def wait_for_latest_op_on_table_finish = { table_name, OpTimeout ->
+        for(int t = delta_time; t <= OpTimeout; t += delta_time){
+            alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = 
"${table_name}" ORDER BY CreateTime DESC LIMIT 1;"""
+            alter_res = alter_res.toString()
+            if(alter_res.contains("FINISHED")) {
+                sleep(3000) // wait change table state to normal
+                logger.info(table_name + " latest alter job finished, detail: 
" + alter_res)
+                break
+            }
+            useTime = t
+            sleep(delta_time)
+        }
+        assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish 
timeout")
+    }
+
+    def indexTbName1 = "test_create_index_2"
+
+    sql "DROP TABLE IF EXISTS ${indexTbName1}"
+    // case 1: create table with index
+    // case 1.1: create duplicate same index for one colume with same name
+    def create_dup_index_result = "fail"
+    try {
+        sql """
+                CREATE TABLE IF NOT EXISTS ${indexTbName1} (
+                    id INT DEFAULT '10',
+                    name VARCHAR(32) DEFAULT '',
+                    INDEX name_idx(name) USING INVERTED PROPERTIES("parser" = 
"english") COMMENT 'name index',
+                    INDEX name_idx(name) USING INVERTED PROPERTIES("parser" = 
"english") COMMENT 'name index'
+                )
+                DUPLICATE KEY(id)
+                DISTRIBUTED BY HASH(id) BUCKETS 10
+                PROPERTIES("replication_num" = "1");
+        """
+        create_dup_index_result = "success"
+    } catch(Exception ex) {
+        logger.info("create duplicate same index for one colume with same 
name, result: " + ex)
+    }
+    assertEquals(create_dup_index_result, "fail")
+
+    // case 1.2: create duplicate same index for one colume with different name
+    try {
+        sql """
+                CREATE TABLE IF NOT EXISTS ${indexTbName1} (
+                    id INT DEFAULT '10',
+                    name VARCHAR(32) DEFAULT '',
+                    INDEX name_idx_1(name) USING INVERTED PROPERTIES("parser" 
= "english") COMMENT 'name index',
+                    INDEX name_idx_2(name) USING INVERTED PROPERTIES("parser" 
= "english") COMMENT 'name index'
+                )
+                DUPLICATE KEY(id)
+                DISTRIBUTED BY HASH(id) BUCKETS 10
+                PROPERTIES("replication_num" = "1");
+        """
+        create_dup_index_result = "success"
+    } catch(Exception ex) {
+        logger.info("create duplicate same index for one colume with different 
name, result: " + ex)
+    }
+    assertEquals(create_dup_index_result, "fail")
+
+    // case 1.3: create duplicate different index for one colume with same name
+    try {
+        sql """
+                CREATE TABLE IF NOT EXISTS ${indexTbName1} (
+                    id INT DEFAULT '10',
+                    name VARCHAR(32) DEFAULT '',
+                    INDEX name_idx(name) USING NGRAM_BF 
PROPERTIES("gram_size"="3", "bf_size"="256") COMMENT 'name index',
+                    INDEX name_idx(name) USING INVERTED PROPERTIES("parser" = 
"english") COMMENT 'name index'
+                )
+                DUPLICATE KEY(id)
+                DISTRIBUTED BY HASH(id) BUCKETS 10
+                PROPERTIES("replication_num" = "1");
+        """
+        create_dup_index_result = "success"
+    } catch(Exception ex) {
+        logger.info("create duplicate different index for one colume with same 
name, result: " + ex)
+    }
+    assertEquals(create_dup_index_result, "fail")
+
+    // case 1.4: create duplicate different index for one colume with 
different name
+    sql """
+            CREATE TABLE IF NOT EXISTS ${indexTbName1} (
+                    id INT DEFAULT '10',
+                    name VARCHAR(32) DEFAULT '',
+                    INDEX name_idx_1(name) USING NGRAM_BF 
PROPERTIES("gram_size"="3", "bf_size"="256") COMMENT 'name index',
+                    INDEX name_idx_2(name) USING INVERTED PROPERTIES("parser" 
= "english") COMMENT 'name index'
+                )
+                DUPLICATE KEY(id)
+                DISTRIBUTED BY HASH(id) BUCKETS 10
+                PROPERTIES("replication_num" = "1");
+    """
+
+    def show_result = sql "show index from ${indexTbName1}"
+    logger.info("show index from " + indexTbName1 + " result: " + show_result)
+    assertEquals(show_result.size(), 2)
+    assertEquals(show_result[0][2], "name_idx_1")
+    assertEquals(show_result[1][2], "name_idx_2")
+    
+    // drop index
+    sql "drop index name_idx_1 on ${indexTbName1}"
+    wait_for_latest_op_on_table_finish(indexTbName1, timeout)
+    sql "drop index name_idx_2 on ${indexTbName1}"
+    show_result = sql "show index from ${indexTbName1}"
+    assertEquals(show_result.size(), 0)
+
+    // case 2: alter add index
+    sql """
+        create index name_idx on ${indexTbName1}(name) using inverted 
properties("parser" = "english") comment 'name index';
+    """
+    show_result = sql "show index from ${indexTbName1}"
+    logger.info("show index from " + indexTbName1 + " result: " + show_result)
+    assertEquals(show_result[0][2], "name_idx")
+    // case 2.1: create duplicate same index for one colume with same name
+    try {
+        sql """
+            create index name_idx on ${indexTbName1}(name) using inverted 
properties("parser" = "english") comment 'name index';
+        """
+        create_dup_index_result = "success"
+    } catch(Exception ex) {
+        logger.info("create duplicate same index for one colume with same 
name, result: " + ex)
+    }
+
+    // case 2.2: create duplicate same index for one colume with different name
+    try {
+        sql """
+            create index name_idx_2 on ${indexTbName1}(name) using inverted 
properties("parser" = "english") comment 'name index';
+        """
+        create_dup_index_result = "success"
+    } catch(Exception ex) {
+        logger.info("create duplicate same index for one colume with different 
name, result: " + ex)
+    }
+
+    // case 2.3: create duplicate different index for one colume with same name
+    try {
+        sql """
+            create index name_idx on ${indexTbName1}(name) using ngram_bf 
properties("gram_size"="3", "bf_size"="256") comment 'name index';
+        """
+        create_dup_index_result = "success"
+    } catch(Exception ex) {
+        logger.info("create duplicate different index for one colume with same 
name, result: " + ex)
+    }
+
+    // 2.4: create duplicate different index for one colume with different name
+    sql """
+        create index name_idx_2 on ${indexTbName1}(name) using ngram_bf 
properties("gram_size"="3", "bf_size"="256") comment 'name index';
+    """
+    wait_for_latest_op_on_table_finish(indexTbName1, timeout)
+
+    show_result = sql "show index from ${indexTbName1}"
+    logger.info("show index from " + indexTbName1 + " result: " + show_result)
+    assertEquals(show_result.size(), 2)
+    assertEquals(show_result[0][2], "name_idx")
+    assertEquals(show_result[1][2], "name_idx_2")
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Reply via email to