This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push: new f6562e53910 [feature](index) Replace BITMAP INDEX with INVERTED INDEX #30950 (#31003) f6562e53910 is described below commit f6562e53910eae091e735866baa4f0c6f4d5adf3 Author: Kang <kxiao.ti...@gmail.com> AuthorDate: Tue Feb 13 15:11:59 2024 +0800 [feature](index) Replace BITMAP INDEX with INVERTED INDEX #30950 (#31003) --- .../community/design/doris_storage_optimization.md | 6 +- docs/en/docs/admin-manual/query-profile.md | 8 +- docs/en/docs/advanced/alter-table/schema-change.md | 3 +- docs/en/docs/advanced/cold-hot-separation.md | 1 - docs/en/docs/data-table/best-practice.md | 3 +- docs/en/docs/data-table/index/bitmap-index.md | 83 --------------------- docs/en/docs/data-table/index/bloomfilter.md | 2 +- docs/en/docs/data-table/index/index-overview.md | 2 +- docs/en/docs/data-table/index/inverted-index.md | 4 +- .../data-table/index/ngram-bloomfilter-index.md | 2 +- .../Alter/ALTER-TABLE-BITMAP.md | 86 --------------------- .../Create/CREATE-INDEX.md | 9 +-- .../Create/CREATE-TABLE.md | 10 +-- .../Data-Definition-Statements/Drop/DROP-INDEX.md | 2 +- .../sql-reference/Show-Statements/SHOW-INDEX.md | 2 +- docs/sidebars.json | 4 +- .../docs/advanced/alter-table/schema-change.md | 3 +- docs/zh-CN/docs/advanced/cold-hot-separation.md | 1 - docs/zh-CN/docs/data-table/best-practice.md | 3 +- docs/zh-CN/docs/data-table/index/bitmap-index.md | 85 --------------------- docs/zh-CN/docs/data-table/index/bloomfilter.md | 2 +- docs/zh-CN/docs/data-table/index/index-overview.md | 2 +- docs/zh-CN/docs/data-table/index/inverted-index.md | 4 +- .../data-table/index/ngram-bloomfilter-index.md | 2 +- .../Alter/ALTER-TABLE-BITMAP.md | 87 ---------------------- .../Create/CREATE-INDEX.md | 9 +-- .../Create/CREATE-TABLE.md | 10 +-- fe/fe-core/src/main/cup/sql_parser.cup | 2 +- .../java/org/apache/doris/analysis/IndexDef.java | 24 ++---- .../doris/analysis/CreateIndexClauseTest.java | 4 +- .../org/apache/doris/analysis/IndexDefTest.java | 10 +-- .../persist/TableAddOrDropColumnsInfoTest.java | 2 +- .../decimalv2/test_decimalv2_common.groovy | 4 +- .../suites/datatype_p0/scalar_types/load.groovy | 55 -------------- .../sql/unique2_bitmap_index_q01_bool.sql | 1 - .../sql/unique2_bitmap_index_q02_tinyint.sql | 1 - .../sql/unique2_bitmap_index_q03_smallint.sql | 1 - .../sql/unique2_bitmap_index_q04_int.sql | 1 - .../sql/unique2_bitmap_index_q05_bigint.sql | 1 - .../sql/unique2_bitmap_index_q06_largeint.sql | 1 - .../sql/unique2_bitmap_index_q07_float.sql | 1 - .../sql/unique2_bitmap_index_q08_double.sql | 1 - .../sql/unique2_bitmap_index_q09_decimal.sql | 1 - .../sql/unique2_bitmap_index_q10_decimalv3.sql | 1 - .../sql/unique2_bitmap_index_q11_date.sql | 1 - .../sql/unique2_bitmap_index_q12_datetime.sql | 1 - .../sql/unique2_bitmap_index_q13_datev2.sql | 1 - .../sql/unique2_bitmap_index_q14_datetimev2.sql | 1 - .../sql/unique2_bitmap_index_q15_char.sql | 1 - .../sql/unique2_bitmap_index_q16_varchar.sql | 1 - .../sql/unique2_bitmap_index_q17_string.sql | 1 - .../suites/index_p0/test_bitmap_index.groovy | 16 +--- .../suites/index_p0/test_index_meta.groovy | 10 +-- .../inverted_index_p0/test_add_drop_index.groovy | 12 +-- .../inverted_index_p0/test_bitmap_index.groovy | 8 +- .../inverted_index_p0/test_create_index_1.groovy | 23 +++--- .../inverted_index_p0/test_inverted_index.groovy | 2 +- .../load_p0/broker_load/ddl/mow_tbl_basic.sql | 5 -- .../load_p0/broker_load/ddl/uniq_tbl_basic.sql | 8 +- .../load_p0/broker_load/test_seq_load.groovy | 8 +- .../routine_load/ddl/mow_tbl_basic_create.sql | 5 -- .../routine_load/ddl/uniq_tbl_basic_create.sql | 8 +- .../stream_load/ddl/mow_tbl_basic_create.sql | 5 -- .../stream_load/ddl/uniq_tbl_basic_create.sql | 8 +- .../ddl/uniq_tbl_basic_create_sequence.sql | 8 +- .../suites/load_p0/tvf/ddl/mow_tbl_basic_tvf.sql | 5 -- .../suites/load_p0/tvf/ddl/uniq_tbl_basic_tvf.sql | 8 +- .../test_segcompaction_unique_keys_index.groovy | 2 +- 68 files changed, 96 insertions(+), 598 deletions(-) diff --git a/docs/en/community/design/doris_storage_optimization.md b/docs/en/community/design/doris_storage_optimization.md index 4cfb5866278..3172e0b60b0 100644 --- a/docs/en/community/design/doris_storage_optimization.md +++ b/docs/en/community/design/doris_storage_optimization.md @@ -113,7 +113,7 @@ We generate a sparse index of short key every N rows (configurable) with the con ### Column's other indexes ### -The format design supports the subsequent expansion of other index information, such as bitmap index, spatial index, etc. It only needs to write the required data to the existing column data, and add the corresponding metadata fields to FileFooterPB. +The format design supports the subsequent expansion of other index information, such as inverted index, spatial index, etc. It only needs to write the required data to the existing column data, and add the corresponding metadata fields to FileFooterPB. ### Metadata Definition ### SegmentFooterPB is defined as: @@ -210,7 +210,7 @@ Relevant issues: 1. Read the magic of the file and judge the type and version of the file. 2. Read FileFooterPB and check sum 3. Read short key index and data ordinal index information of corresponding columns according to required columns -4. Use start key and end key, locate the row number to be read through short key index, then determine the row ranges to be read through ordinal index, and filter the row ranges to be read through statistics, bitmap index and so on. +4. Use start key and end key, locate the row number to be read through short key index, then determine the row ranges to be read through ordinal index, and filter the row ranges to be read through statistics, inverted index and so on. 5. Then read row data through ordinal index according to row ranges Relevant issues: @@ -232,4 +232,4 @@ It implements a scalable compression framework, supports a variety of compressio ## TODO ## 1. How to implement nested types? How to locate line numbers in nested types? -2. How to optimize the downstream bitmap and column statistics caused by ScanRange splitting? +2. How to optimize the downstream inverted index and column statistics caused by ScanRange splitting? diff --git a/docs/en/docs/admin-manual/query-profile.md b/docs/en/docs/admin-manual/query-profile.md index f46061419c6..e8c89faaa4d 100644 --- a/docs/en/docs/admin-manual/query-profile.md +++ b/docs/en/docs/admin-manual/query-profile.md @@ -221,7 +221,7 @@ OLAP_SCAN_NODE (id=0):(Active: 1.2ms,% non-child: 0.00%) - ScanTime: 39.24us # The time returned from ScanNode to the upper node. - ShowHintsTime_V1: 0ns # V2 has no meaning. Read part of the data in V1 to perform ScanRange segmentation. SegmentIterator: - - BitmapIndexFilterTimer: 779ns # Use bitmap index to filter data time-consuming. + - InvertedIndexFilterTimer: 779ns # Use inverted index to filter data time-consuming. - BlockLoadTime: 415.925us # SegmentReader(V1) or SegmentIterator(V2) gets the time of the block. - BlockSeekCount: 12 # The number of block seeks when reading Segment. - BlockSeekTime: 222.556us # It takes time to block seek when reading Segment. @@ -234,7 +234,7 @@ OLAP_SCAN_NODE (id=0):(Active: 1.2ms,% non-child: 0.00%) - NumSegmentFiltered: 0 # When generating Segment Iterator, the number of Segments that are completely filtered out through column statistics and query conditions. - NumSegmentTotal: 6 # Query the number of all segments involved. - RawRowsRead: 7 # The number of raw rows read in the storage engine. See below for details. - - RowsBitmapIndexFiltered: 0 # Only in V2, the number of rows filtered by the Bitmap index. + - RowsInvertedIndexFiltered: 0 # Only in V2, the number of rows filtered by the Inverted index. - RowsBloomFilterFiltered: 0 # Only in V2, the number of rows filtered by BloomFilter index. - RowsKeyRangeFiltered: 0 # In V2 only, the number of rows filtered out by SortkeyIndex index. - RowsStatsFiltered: 0 # In V2, the number of rows filtered by the ZoneMap index, including the deletion condition. V1 also contains the number of rows filtered by BloomFilter. @@ -248,8 +248,8 @@ OLAP_SCAN_NODE (id=0):(Active: 1.2ms,% non-child: 0.00%) The predicate push down and index usage can be inferred from the related indicators of the number of data rows in the profile. The following only describes the profile in the reading process of segment V2 format data. In segment V1 format, the meaning of these indicators is slightly different. - When reading a segment V2, if the query has key_ranges (the query range composed of prefix keys), first filter the data through the SortkeyIndex index, and the number of filtered rows is recorded in `RowsKeyRangeFiltered`. - - After that, use the Bitmap index to perform precise filtering on the columns containing the bitmap index in the query condition, and the number of filtered rows is recorded in `RowsBitmapIndexFiltered`. - - After that, according to the equivalent (eq, in, is) condition in the query condition, use the BloomFilter index to filter the data and record it in `RowsBloomFilterFiltered`. The value of `RowsBloomFilterFiltered` is the difference between the total number of rows of the Segment (not the number of rows filtered by the Bitmap index) and the number of remaining rows after BloomFilter, so the data filtered by BloomFilter may overlap with the data filtered by Bitmap. + - After that, use the Inverted index to perform precise filtering on the columns containing the inverted index in the query condition, and the number of filtered rows is recorded in `RowsInvertedIndexFiltered`. + - After that, according to the equivalent (eq, in, is) condition in the query condition, use the BloomFilter index to filter the data and record it in `RowsBloomFilterFiltered`. The value of `RowsBloomFilterFiltered` is the difference between the total number of rows of the Segment (not the number of rows filtered by the Inverted index) and the number of remaining rows after BloomFilter, so the data filtered by BloomFilter may overlap with the data filtered by inverted index. - After that, use the ZoneMap index to filter the data according to the query conditions and delete conditions and record it in `RowsStatsFiltered`. - `RowsConditionsFiltered` is the number of rows filtered by various indexes, including the values of `RowsBloomFilterFiltered` and `RowsStatsFiltered`. - So far, the Init phase is completed, and the number of rows filtered by the condition to be deleted in the Next phase is recorded in `RowsDelFiltered`. Therefore, the number of rows actually filtered by the delete condition are recorded in `RowsStatsFiltered` and `RowsDelFiltered` respectively. diff --git a/docs/en/docs/advanced/alter-table/schema-change.md b/docs/en/docs/advanced/alter-table/schema-change.md index 31a335114ae..a8a98a79f7a 100644 --- a/docs/en/docs/advanced/alter-table/schema-change.md +++ b/docs/en/docs/advanced/alter-table/schema-change.md @@ -31,8 +31,7 @@ Users can modify the schema of existing tables through the Schema Change operati * Add and delete columns * Modify column type * Adjust column order -* Add and modify Bloom Filter -* Add and delete bitmap index +* Add and delete index This document mainly describes how to create a Schema Change job, as well as some considerations and frequently asked questions about Schema Change. ## Glossary diff --git a/docs/en/docs/advanced/cold-hot-separation.md b/docs/en/docs/advanced/cold-hot-separation.md index fadd0ed8c9c..9ec83dc1396 100644 --- a/docs/en/docs/advanced/cold-hot-separation.md +++ b/docs/en/docs/advanced/cold-hot-separation.md @@ -167,7 +167,6 @@ The supported schema change types after data cooling are as follows: * Modify column type * Adjust column order * Add and modify Bloom Filter -* Add and delete bitmap index ## cold data Garbage collection The garbage data of cold data refers to the data that is not used by any Replica. Object storage may have garbage data generated by the following situations: diff --git a/docs/en/docs/data-table/best-practice.md b/docs/en/docs/data-table/best-practice.md index 2dda2d91a26..5c4333bd7b8 100644 --- a/docs/en/docs/data-table/best-practice.md +++ b/docs/en/docs/data-table/best-practice.md @@ -177,7 +177,6 @@ Users can modify the Schema of an existing table through the Schema Change opera - Adding and deleting columns - Modify column types - Reorder columns -- Adding or modifying Bloom Filter -- Adding or removing bitmap index +- Adding or removing index For details, please refer to [Schema Change](../advanced/alter-table/schema-change.md) \ No newline at end of file diff --git a/docs/en/docs/data-table/index/bitmap-index.md b/docs/en/docs/data-table/index/bitmap-index.md deleted file mode 100644 index e743f2e94b9..00000000000 --- a/docs/en/docs/data-table/index/bitmap-index.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -{ - "title": "Bitmap Index", - "language": "en" -} ---- - -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> - -# Bitmap Index -Users can speed up queries by creating a bitmap index -This document focuses on how to create an index job, as well as some considerations and frequently asked questions when creating an index. - -## Glossary -* bitmap index: a fast data structure that speeds up queries - -## Basic Principles -Creating and dropping index is essentially a schema change job. For details, please refer to -[Schema Change](../../advanced/alter-table/schema-change.md). - -## Syntax -### Create index - -Create a bitmap index for siteid on table1 - -```sql -CREATE INDEX [IF NOT EXISTS] index_name ON table1 (siteid) USING BITMAP COMMENT 'balabala'; -``` - -### View index - -Display the lower index of the specified table_name - -```sql -SHOW INDEX FROM example_db.table_name; -``` - -### Delete index - -Delete the lower index of the specified table_name - -```sql -DROP INDEX [IF EXISTS] index_name ON [db_name.]table_name; -``` - -## Notice -* Currently only index of bitmap type is supported. -* The bitmap index is only created on a single column. -* Bitmap indexes can be applied to all columns of the `Duplicate` , `Uniq` data model and key columns of the `Aggregate` models. -* The data types supported by bitmap indexes are as follows: - * `TINYINT` - * `SMALLINT` - * `INT` - * `BIGINT` - * `CHAR` - * `VARCHAR` - * `DATE` - * `DATETIME` - * `LARGEINT` - * `DECIMAL` - * `BOOL` -* The bitmap index takes effect only in segmentV2. The table's storage format will be converted to V2 automatically when creating index. - -### More Help - -For more detailed syntax and best practices for using bitmap indexes, please refer to the [CREARE INDEX](../../sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-INDEX.md) / [SHOW INDEX](../../sql-manual/sql-reference/Show-Statements/SHOW-INDEX.md) / [DROP INDEX](../../sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-INDEX.md) command manual. You can also enter HELP CREATE INDEX / HELP SHOW INDEX / HELP DROP INDEX on the MySql client command line. diff --git a/docs/en/docs/data-table/index/bloomfilter.md b/docs/en/docs/data-table/index/bloomfilter.md index 4a6f783fd43..83368c79fd4 100644 --- a/docs/en/docs/data-table/index/bloomfilter.md +++ b/docs/en/docs/data-table/index/bloomfilter.md @@ -121,7 +121,7 @@ You can consider establishing a Bloom Filter index for a column when the followi 1. First, BloomFilter is suitable for non-prefix filtering. 2. The query will be filtered according to the high frequency of the column, and most of the query conditions are in and = filtering. -3. Unlike Bitmap, BloomFilter is suitable for high cardinality columns. Such as UserID. Because if it is created on a low-cardinality column, such as a "gender" column, each Block will almost contain all values, causing the BloomFilter index to lose its meaning. +3. BloomFilter is suitable for high cardinality columns. Such as UserID. Because if it is created on a low-cardinality column, such as a "gender" column, each Block will almost contain all values, causing the BloomFilter index to lose its meaning. ### **Doris BloomFilter Use Precautions** diff --git a/docs/en/docs/data-table/index/index-overview.md b/docs/en/docs/data-table/index/index-overview.md index 274f0d7d4ff..0e56e11307a 100644 --- a/docs/en/docs/data-table/index/index-overview.md +++ b/docs/en/docs/data-table/index/index-overview.md @@ -30,7 +30,7 @@ Indexes are used to help quickly filter or find data. Doris currently supports two main types of indexes: 1. built-in smart indexes, including prefix indexes and ZoneMap indexes. -2. User-created secondary indexes, including the [inverted index](./inverted-index.md), [bloomfilter index](./bloomfilter.md)、[ngram bloomfilter index](./ngram-bloomfilter-index.md) and [bitmap index](./bitmap-index.md). +2. User-created secondary indexes, including the [inverted index](./inverted-index.md), [bloomfilter index](./bloomfilter.md) and [ngram bloomfilter index](./ngram-bloomfilter-index.md). The ZoneMap index is the index information automatically maintained for each column in the column storage format, including Min/Max, the number of Null values, and so on. This index is transparent to the user. diff --git a/docs/en/docs/data-table/index/inverted-index.md b/docs/en/docs/data-table/index/inverted-index.md index 75a8f6a3b1b..9f3aa8a5ba4 100644 --- a/docs/en/docs/data-table/index/inverted-index.md +++ b/docs/en/docs/data-table/index/inverted-index.md @@ -24,7 +24,7 @@ specific language governing permissions and limitations under the License. --> -# [Experimental] Inverted Index +# Inverted Index <version since="2.0.0"> @@ -53,7 +53,7 @@ The features for inverted index is as follows: - MATCH_ALL matches all keywords, MATCH_ANY matches any keywords - support fulltext on array of text field - support english, chinese and mixed unicode word parser -- accelerate normal equal, range query, replacing bitmap index in the future +- accelerate normal equal, range query, replacing bitmap index - suport =, !=, >, >=, <, <= on text, numeric, datetime types - suport =, !=, >, >=, <, <= on array of text, numeric, datetime types - complete suport for logic combination diff --git a/docs/en/docs/data-table/index/ngram-bloomfilter-index.md b/docs/en/docs/data-table/index/ngram-bloomfilter-index.md index 1bb499a3317..4242431f7d8 100644 --- a/docs/en/docs/data-table/index/ngram-bloomfilter-index.md +++ b/docs/en/docs/data-table/index/ngram-bloomfilter-index.md @@ -24,7 +24,7 @@ specific language governing permissions and limitations under the License. --> -# [Experimental] NGram BloomFilter Index +# NGram BloomFilter Index <version since="2.0.0"> </version> diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-TABLE-BITMAP.md b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-TABLE-BITMAP.md deleted file mode 100644 index 4ce8f42c798..00000000000 --- a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-TABLE-BITMAP.md +++ /dev/null @@ -1,86 +0,0 @@ ---- -{ - "title": "ALTER-TABLE-BITMAP", - "language": "en" -} ---- - -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> - -## ALTER-TABLE-BITMAP - -### Name - -ALTER TABLE BITMAP - -### Description - -This statement is used to perform a bitmap index operation on an existing table. - -grammar: - -```sql -ALTER TABLE [database.]table alter_clause; -``` - -The alter_clause of bitmap index supports the following modification methods - -1. Create a bitmap index - - Syntax: - -```sql -ADD INDEX [IF NOT EXISTS] index_name (column [, ...],) [USING BITMAP] [COMMENT 'balabala']; -``` - -Notice: - -- Currently only supports bitmap indexes -- BITMAP indexes are only created on a single column - -2. Delete the index - -Syntax: - -```sql -DROP INDEX [IF EXISTS] index_name; -``` - -### Example - -1. Create a bitmap index for siteid on table1 - -```sql -ALTER TABLE table1 ADD INDEX [IF NOT EXISTS] index_name (siteid) [USING BITMAP] COMMENT 'balabala'; -``` - -2. Delete the bitmap index of the siteid column on table1 - -```sql -ALTER TABLE table1 DROP INDEX [IF EXISTS] index_name; -``` - -### Keywords - -```text -ALTER, TABLE, BITMAP, INDEX, ALTER TABLE -``` - -### Best Practice diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-INDEX.md b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-INDEX.md index 124293c687e..016034b52e0 100644 --- a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-INDEX.md +++ b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-INDEX.md @@ -36,18 +36,17 @@ This statement is used to create an index grammar: ```sql -CREATE INDEX [IF NOT EXISTS] index_name ON table_name (column [, ...],) [USING BITMAP] [COMMENT 'balabala']; +CREATE INDEX [IF NOT EXISTS] index_name ON table_name (column [, ...],) [USING INVERTED] [COMMENT 'balabala']; ```` Notice: -- Currently only supports bitmap indexes -- BITMAP indexes are only created on a single column +- INVERTED indexes are only created on a single column ### Example -1. Create a bitmap index for siteid on table1 +1. Create a inverted index for siteid on table1 ```sql - CREATE INDEX [IF NOT EXISTS] index_name ON table1 (siteid) USING BITMAP COMMENT 'balabala'; + CREATE INDEX [IF NOT EXISTS] index_name ON table1 (siteid) USING INVERTED COMMENT 'balabala'; ```` diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md index 97e9eed0d3f..83bcf3079fe 100644 --- a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md +++ b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md @@ -165,14 +165,14 @@ Index list definition: Index definition: ```sql - INDEX index_name (col_name) [USING BITMAP] COMMENT'xxxxxx' + INDEX index_name (col_name) [USING INVERTED] COMMENT'xxxxxx' ``` Example: ```sql - INDEX idx1 (k1) USING BITMAP COMMENT "This is a bitmap index1", - INDEX idx2 (k2) USING BITMAP COMMENT "This is a bitmap index2", + INDEX idx1 (k1) USING INVERTED COMMENT "This is a inverted index1", + INDEX idx2 (k2) USING INVERTED COMMENT "This is a inverted index2", ... ``` @@ -582,7 +582,7 @@ Set table properties. The following attributes are currently supported: ); ``` -7. Create a table with bitmap index and bloom filter index +7. Create a table with inverted index and bloom filter index ```sql CREATE TABLE example_db.table_hash @@ -591,7 +591,7 @@ Set table properties. The following attributes are currently supported: k2 DECIMAL(10, 2) DEFAULT "10.5", v1 CHAR(10) REPLACE, v2 INT SUM, - INDEX k1_idx (k1) USING BITMAP COMMENT'my first index' + INDEX k1_idx (k1) USING INVERTED COMMENT'my first index' ) AGGREGATE KEY(k1, k2) DISTRIBUTED BY HASH(k1) BUCKETS 32 diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-INDEX.md b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-INDEX.md index ae4088f7301..d4276a1e82b 100644 --- a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-INDEX.md +++ b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-INDEX.md @@ -32,7 +32,7 @@ DROP INDEX ### Description -This statement is used to delete the index of the specified name from a table. Currently, only bitmap indexes are supported. +This statement is used to delete the index of the specified name from a table. grammar: ```sql diff --git a/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-INDEX.md b/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-INDEX.md index cf9d4052602..b55fb64b9b8 100644 --- a/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-INDEX.md +++ b/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-INDEX.md @@ -32,7 +32,7 @@ SHOW INDEX ### Description - This statement is used to display information about indexes in a table. Currently, only bitmap indexes are supported. + This statement is used to display information about indexes in a table. grammar: diff --git a/docs/sidebars.json b/docs/sidebars.json index 3f9e3cc67f6..da837cfe94e 100644 --- a/docs/sidebars.json +++ b/docs/sidebars.json @@ -60,8 +60,7 @@ "data-table/index/index-overview", "data-table/index/inverted-index", "data-table/index/bloomfilter", - "data-table/index/ngram-bloomfilter-index", - "data-table/index/bitmap-index" + "data-table/index/ngram-bloomfilter-index" ] } ] @@ -939,7 +938,6 @@ "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-DATABASE", "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-TABLE-COLUMN", "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-TABLE-PARTITION", - "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-TABLE-BITMAP", "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-TABLE-ROLLUP", "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-TABLE-RENAME", "sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-TABLE-REPLACE", diff --git a/docs/zh-CN/docs/advanced/alter-table/schema-change.md b/docs/zh-CN/docs/advanced/alter-table/schema-change.md index 3a3e993f86d..4b324a8f73c 100644 --- a/docs/zh-CN/docs/advanced/alter-table/schema-change.md +++ b/docs/zh-CN/docs/advanced/alter-table/schema-change.md @@ -31,8 +31,7 @@ under the License. - 增加、删除列 - 修改列类型 - 调整列顺序 -- 增加、修改 Bloom Filter -- 增加、删除 bitmap index +- 增加、删除索引 本文档主要介绍如何创建 Schema Change 作业,以及进行 Schema Change 的一些注意事项和常见问题。 diff --git a/docs/zh-CN/docs/advanced/cold-hot-separation.md b/docs/zh-CN/docs/advanced/cold-hot-separation.md index c81c45dc717..72241cedcbc 100644 --- a/docs/zh-CN/docs/advanced/cold-hot-separation.md +++ b/docs/zh-CN/docs/advanced/cold-hot-separation.md @@ -167,7 +167,6 @@ ALTER TABLE create_table_partition MODIFY PARTITION (*) SET("storage_policy"="te - 修改列类型 - 调整列顺序 - 增加、修改 Bloom Filter -- 增加、删除 bitmap index ## 冷数据的垃圾回收 冷数据的垃圾数据是指没有被任何Replica使用的数据,对象存储上可能会有如下情况产生的垃圾数据: diff --git a/docs/zh-CN/docs/data-table/best-practice.md b/docs/zh-CN/docs/data-table/best-practice.md index fc71426af16..1597d3b56b9 100644 --- a/docs/zh-CN/docs/data-table/best-practice.md +++ b/docs/zh-CN/docs/data-table/best-practice.md @@ -176,7 +176,6 @@ ALTER TABLE session_data ADD ROLLUP rollup_browser(browser,province,ip,url) DUPL - 增加、删除列 - 修改列类型 - 调整列顺序 -- 增加、修改 Bloom Filter -- 增加、删除 bitmap index +- 增加、删除索引 具体请参照 [Schema 变更](../advanced/alter-table/schema-change.md) diff --git a/docs/zh-CN/docs/data-table/index/bitmap-index.md b/docs/zh-CN/docs/data-table/index/bitmap-index.md deleted file mode 100644 index 91d8803d77e..00000000000 --- a/docs/zh-CN/docs/data-table/index/bitmap-index.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -{ - "title": "Bitmap 索引", - "language": "zh-CN" -} ---- - -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> - -# Bitmap 索引 - -用户可以通过创建bitmap index 加速查询 本文档主要介绍如何创建 index 作业,以及创建 index 的一些注意事项和常见问题。 - -## 名词解释 - -- bitmap index:位图索引,是一种快速数据结构,能够加快查询速度 - -## 原理介绍 - -创建和删除本质上是一个 schema change 的作业,具体细节可以参照 [Schema Change](../../advanced/alter-table/schema-change.md)。 - -## 语法 - -### 创建索引 - -在table1 上为siteid 创建bitmap 索引 - -```sql -CREATE INDEX [IF NOT EXISTS] index_name ON table1 (siteid) USING BITMAP COMMENT 'balabala'; -``` - -### 查看索引 - -展示指定 table_name 的下索引 - -```sql -SHOW INDEX FROM example_db.table_name; -``` - -### 删除索引 - -删除指定 table_name 的下索引 - -```sql -DROP INDEX [IF EXISTS] index_name ON [db_name.]table_name; -``` - -## 注意事项 - -- bitmap 索引仅在单列上创建。 -- bitmap 索引能够应用在 `Duplicate`、`Uniq` 数据模型的所有列和 `Aggregate`模型的key列上。 -- bitmap 索引支持的数据类型如下: - - `TINYINT` - - `SMALLINT` - - `INT` - - `BIGINT` - - `CHAR` - - `VARCHAR` - - `DATE` - - `DATETIME` - - `LARGEINT` - - `DECIMAL` - - `BOOL` -- bitmap索引仅在 Segment V2 下生效。当创建 index 时,表的存储格式将默认转换为 V2 格式。 - -## 更多帮助 - -关于 bitmap索引 使用的更多详细语法及最佳实践,请参阅 [CREARE INDEX](../../sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-INDEX.md) / [SHOW INDEX](../../sql-manual/sql-reference/Show-Statements/SHOW-INDEX.md) / [DROP INDEX](../../sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-INDEX.md) 命令手册,你也可以在 MySql 客户端命令行下输入 `HELP CREATE INDEX` / `HELP SHOW INDEX` / `HELP DROP INDEX`。 diff --git a/docs/zh-CN/docs/data-table/index/bloomfilter.md b/docs/zh-CN/docs/data-table/index/bloomfilter.md index ef8c4a5e4e7..2390101a43b 100644 --- a/docs/zh-CN/docs/data-table/index/bloomfilter.md +++ b/docs/zh-CN/docs/data-table/index/bloomfilter.md @@ -118,7 +118,7 @@ ALTER TABLE <db.table_name> SET ("bloom_filter_columns" = "k1,k3"); 1. 首先BloomFilter适用于非前缀过滤。 2. 查询会根据该列高频过滤,而且查询条件大多是 in 和 = 过滤。 -3. 不同于Bitmap, BloomFilter适用于高基数列。比如UserID。因为如果创建在低基数的列上,比如 “性别” 列,则每个Block几乎都会包含所有取值,导致BloomFilter索引失去意义。 +3. BloomFilter适用于高基数列。比如UserID。因为如果创建在低基数的列上,比如 “性别” 列,则每个Block几乎都会包含所有取值,导致BloomFilter索引失去意义。 ## **Doris BloomFilter使用注意事项** diff --git a/docs/zh-CN/docs/data-table/index/index-overview.md b/docs/zh-CN/docs/data-table/index/index-overview.md index a91d68e31e2..6ac01b76d63 100644 --- a/docs/zh-CN/docs/data-table/index/index-overview.md +++ b/docs/zh-CN/docs/data-table/index/index-overview.md @@ -29,7 +29,7 @@ under the License. 目前 Doris 主要支持两类索引: 1. 内建的智能索引,包括前缀索引和 ZoneMap 索引。 -2. 用户手动创建的二级索引,包括 [倒排索引](./inverted-index.md)、 [bloomfilter索引](./bloomfilter.md)、 [ngram bloomfilter索引](./ngram-bloomfilter-index.md) 和[bitmap索引](./bitmap-index.md)。 +2. 用户手动创建的二级索引,包括 [倒排索引](./inverted-index.md)、 [bloomfilter索引](./bloomfilter.md) 和 [ngram bloomfilter索引](./ngram-bloomfilter-index.md) 。 其中 ZoneMap 索引是在列存格式上,对每一列自动维护的索引信息,包括 Min/Max,Null 值个数等等。这种索引对用户透明。 diff --git a/docs/zh-CN/docs/data-table/index/inverted-index.md b/docs/zh-CN/docs/data-table/index/inverted-index.md index 2f4c3f85bfe..e2c1c516ef6 100644 --- a/docs/zh-CN/docs/data-table/index/inverted-index.md +++ b/docs/zh-CN/docs/data-table/index/inverted-index.md @@ -24,7 +24,7 @@ specific language governing permissions and limitations under the License. --> -# [Experimental] 倒排索引 +# 倒排索引 <version since="2.0.0"> @@ -53,7 +53,7 @@ Doris倒排索引的功能简要介绍如下: - 支持字符串全文检索,包括同时匹配多个关键字MATCH_ALL、匹配任意一个关键字MATCH_ANY、匹配短语词组MATCH_PHRASE - 支持字符串数组类型的全文检索 - 支持英文、中文以及Unicode多语言分词 -- 加速普通等值、范围查询,覆盖bitmap索引的功能,未来会代替bitmap索引 +- 加速普通等值、范围查询,覆盖bitmap索引的功能,可代替bitmap索引 - 支持字符串、数值、日期时间类型的 =, !=, >, >=, <, <= 快速过滤 - 支持字符串、数字、日期时间数组类型的 =, !=, >, >=, <, <= - 支持完善的逻辑组合 diff --git a/docs/zh-CN/docs/data-table/index/ngram-bloomfilter-index.md b/docs/zh-CN/docs/data-table/index/ngram-bloomfilter-index.md index 0549db8d626..f5e1780c102 100644 --- a/docs/zh-CN/docs/data-table/index/ngram-bloomfilter-index.md +++ b/docs/zh-CN/docs/data-table/index/ngram-bloomfilter-index.md @@ -24,7 +24,7 @@ specific language governing permissions and limitations under the License. --> -# [Experimental] NGram BloomFilter索引及使用使用场景 +# NGram BloomFilter索引及使用使用场景 <version since="2.0.0"> </version> diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-TABLE-BITMAP.md b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-TABLE-BITMAP.md deleted file mode 100644 index 9a960f2f750..00000000000 --- a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-TABLE-BITMAP.md +++ /dev/null @@ -1,87 +0,0 @@ ---- -{ - "title": "ALTER-TABLE-BITMAP", - "language": "zh-CN" -} ---- - -<!-- -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---> - -## ALTER-TABLE-BITMAP - -### Name - -ALTER TABLE BITMAP - -### Description - -该语句用于对已有 table 进行 bitmap index 操作。 - -语法: - -```sql -ALTER TABLE [database.]table alter_clause; -``` - -bitmap index 的 alter_clause 支持如下几种修改方式 - -1. 创建bitmap 索引 - -语法: - -```sql -ADD INDEX [IF NOT EXISTS] index_name (column [, ...],) [USING BITMAP] [COMMENT 'balabala']; -``` - -注意: - -- 目前仅支持bitmap 索引 -- BITMAP 索引仅在单列上创建 - -2. 删除索引 - -语法: - -```sql -DROP INDEX [IF EXISTS] index_name; -``` - -### Example - -1. 在table1 上为siteid 创建bitmap 索引 - -```sql -ALTER TABLE table1 ADD INDEX [IF NOT EXISTS] index_name (siteid) [USING BITMAP] COMMENT 'balabala'; -``` - -2. 删除table1 上的siteid列的bitmap 索引 - -```sql -ALTER TABLE table1 DROP INDEX [IF EXISTS] index_name; -``` - -### Keywords - -```text -ALTER, TABLE, BITMAP, INDEX, ALTER TABLE -``` - -### Best Practice - diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-INDEX.md b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-INDEX.md index eb161c36627..5a87a10329f 100644 --- a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-INDEX.md +++ b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-INDEX.md @@ -36,18 +36,17 @@ CREATE INDEX 语法: ```sql -CREATE INDEX [IF NOT EXISTS] index_name ON table_name (column [, ...],) [USING BITMAP] [COMMENT'balabala']; +CREATE INDEX [IF NOT EXISTS] index_name ON table_name (column [, ...],) [USING INVERTED] [COMMENT'balabala']; ``` 注意: -- 目前只支持bitmap 索引 -- BITMAP 索引仅在单列上创建 +- 倒排索引仅在单列上创建 ### Example -1. 在table1 上为siteid 创建bitmap 索引 +1. 在table1 上为siteid 创建倒排索引 ```sql - CREATE INDEX [IF NOT EXISTS] index_name ON table1 (siteid) USING BITMAP COMMENT 'balabala'; + CREATE INDEX [IF NOT EXISTS] index_name ON table1 (siteid) USING INVERTED COMMENT 'balabala'; ``` diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md index 600859009d4..c34c7f709e9 100644 --- a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md +++ b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md @@ -153,14 +153,14 @@ distribution_desc 索引定义: ```sql - INDEX index_name (col_name) [USING BITMAP] COMMENT 'xxxxxx' + INDEX index_name (col_name) [USING INVERTED] COMMENT 'xxxxxx' ``` 示例: ```sql - INDEX idx1 (k1) USING BITMAP COMMENT "This is a bitmap index1", - INDEX idx2 (k2) USING BITMAP COMMENT "This is a bitmap index2", + INDEX idx1 (k1) USING INVERTED COMMENT "This is a inverted index1", + INDEX idx2 (k2) USING INVERTED COMMENT "This is a inverted index2", ... ``` @@ -564,7 +564,7 @@ UNIQUE KEY(k1, k2) ); ``` -7. 创建一个带有 bitmap 索引以及 bloom filter 索引的表 +7. 创建一个带有倒排索引以及 bloom filter 索引的表 ```sql CREATE TABLE example_db.table_hash @@ -573,7 +573,7 @@ UNIQUE KEY(k1, k2) k2 DECIMAL(10, 2) DEFAULT "10.5", v1 CHAR(10) REPLACE, v2 INT SUM, - INDEX k1_idx (k1) USING BITMAP COMMENT 'my first index' + INDEX k1_idx (k1) USING INVERTED COMMENT 'my first index' ) AGGREGATE KEY(k1, k2) DISTRIBUTED BY HASH(k1) BUCKETS 32 diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index 9e0ae035bde..3f279c2d2dd 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -3651,7 +3651,7 @@ opt_index_type ::= :} | KW_USING KW_BITMAP {: - RESULT = IndexDef.IndexType.BITMAP; + RESULT = IndexDef.IndexType.INVERTED; :} | KW_USING KW_NGRAM_BF {: diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java index a609944dc25..18d0f22a38c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java @@ -55,7 +55,7 @@ public class IndexDef { this.ifNotExists = ifNotExists; this.columns = columns; if (indexType == null) { - this.indexType = IndexType.BITMAP; + this.indexType = IndexType.INVERTED; } else { this.indexType = indexType; } @@ -222,18 +222,14 @@ public class IndexDef { if (!(colType.isDateType() || colType.isDecimalV2Type() || colType.isDecimalV3Type() || colType.isFixedPointType() || colType.isStringType() || colType == PrimitiveType.BOOLEAN)) { throw new AnalysisException(colType + " is not supported in " + indexType.toString() + " index. " - + "invalid column: " + indexColName); - } else if (indexType == IndexType.INVERTED - && ((keysType == KeysType.AGG_KEYS && !column.isKey()) - || (keysType == KeysType.UNIQUE_KEYS && !enableUniqueKeyMergeOnWrite))) { - throw new AnalysisException(indexType.toString() - + " index only used in columns of DUP_KEYS table" - + " or UNIQUE_KEYS table with merge_on_write enabled" - + " or key columns of AGG_KEYS table. invalid column: " + indexColName); - } else if (keysType == KeysType.AGG_KEYS && !column.isKey() && indexType != IndexType.INVERTED) { + + "invalid index: " + indexName); + } + if (!column.isKey() + && ((keysType == KeysType.UNIQUE_KEYS && !enableUniqueKeyMergeOnWrite) + || keysType == KeysType.AGG_KEYS)) { throw new AnalysisException(indexType.toString() - + " index only used in columns of DUP_KEYS/UNIQUE_KEYS table or key columns of" - + " AGG_KEYS table. invalid column: " + indexColName); + + " index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table." + + " invalid index: " + indexName); } if (indexType == IndexType.INVERTED) { @@ -243,10 +239,6 @@ public class IndexDef { && colType != PrimitiveType.STRING) { throw new AnalysisException(colType + " is not supported in ngram_bf index. " + "invalid column: " + indexColName); - } else if ((keysType == KeysType.AGG_KEYS && !column.isKey())) { - throw new AnalysisException( - "ngram_bf index only used in columns of DUP_KEYS/UNIQUE_KEYS table or key columns of" - + " AGG_KEYS table. invalid column: " + indexColName); } if (properties.size() != 2) { throw new AnalysisException("ngram_bf index should have gram_size and bf_size properties"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateIndexClauseTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateIndexClauseTest.java index 88aae9c4e24..7341b8e7bcf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateIndexClauseTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateIndexClauseTest.java @@ -37,10 +37,10 @@ public class CreateIndexClauseTest { public void testNormal() throws AnalysisException { CreateIndexClause clause = new CreateIndexClause( new TableName(InternalCatalog.INTERNAL_CATALOG_NAME, "db", "table"), - new IndexDef("index1", false, Lists.newArrayList("col1"), IndexDef.IndexType.BITMAP, null, "balabala"), + new IndexDef("index1", false, Lists.newArrayList("col1"), IndexDef.IndexType.INVERTED, null, "balabala"), false); clause.analyze(analyzer); - Assert.assertEquals("CREATE INDEX index1 ON `db`.`table` (`col1`) USING BITMAP COMMENT 'balabala'", + Assert.assertEquals("CREATE INDEX index1 ON `db`.`table` (`col1`) USING INVERTED COMMENT 'balabala'", clause.toSql()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java index b1072e41fa5..a4eeec679b2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/IndexDefTest.java @@ -29,7 +29,7 @@ public class IndexDefTest { @Before public void setUp() throws Exception { - def = new IndexDef("index1", false, Lists.newArrayList("col1"), IndexDef.IndexType.BITMAP, null, "balabala"); + def = new IndexDef("index1", false, Lists.newArrayList("col1"), IndexDef.IndexType.INVERTED, null, "balabala"); } @Test @@ -46,7 +46,7 @@ public class IndexDefTest { + "x1xxxxxxxxxxxxxxxxxindex1xxxxxxxxxxxxxxxxxindex1xxxxxxxxxxxxxxxxxindex1xxxxxxxxxxxxx" + "xxxxindex1xxxxxxxxxxxxxxxxxindex1xxxxxxxxxxxxxxxxxindex1xxxxxxxxxxxxxxxxxindex1xxxxx" + "xxxxxxxxxxxxindex1xxxxxxxxxxxxxxxxx", false, - Lists.newArrayList("col1"), IndexDef.IndexType.BITMAP, null, + Lists.newArrayList("col1"), IndexDef.IndexType.INVERTED, null, "balabala"); def.analyze(); Assert.fail("No exception throws."); @@ -54,7 +54,7 @@ public class IndexDefTest { Assert.assertTrue(e instanceof AnalysisException); } try { - def = new IndexDef("", false, Lists.newArrayList("col1"), IndexDef.IndexType.BITMAP, null, "balabala"); + def = new IndexDef("", false, Lists.newArrayList("col1"), IndexDef.IndexType.INVERTED, null, "balabala"); def.analyze(); Assert.fail("No exception throws."); } catch (AnalysisException e) { @@ -64,8 +64,8 @@ public class IndexDefTest { @Test public void toSql() { - Assert.assertEquals("INDEX index1 (`col1`) USING BITMAP COMMENT 'balabala'", def.toSql()); - Assert.assertEquals("INDEX index1 ON table1 (`col1`) USING BITMAP COMMENT 'balabala'", + Assert.assertEquals("INDEX index1 (`col1`) USING INVERTED COMMENT 'balabala'", def.toSql()); + Assert.assertEquals("INDEX index1 ON table1 (`col1`) USING INVERTED COMMENT 'balabala'", def.toSql("table1")); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java index 1c78f24cbf3..0e04bddd681 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/TableAddOrDropColumnsInfoTest.java @@ -65,7 +65,7 @@ public class TableAddOrDropColumnsInfoTest { indexSchemaMap.put(tableId, fullSchema); List<Index> indexes = Lists.newArrayList( - new Index(0, "index", Lists.newArrayList("testCol1"), IndexDef.IndexType.BITMAP, null, "xxxxxx")); + new Index(0, "index", Lists.newArrayList("testCol1"), IndexDef.IndexType.INVERTED, null, "xxxxxx")); TableAddOrDropColumnsInfo tableAddOrDropColumnsInfo1 = new TableAddOrDropColumnsInfo("", dbId, tableId, indexSchemaMap, indexes, jobId); diff --git a/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_common.groovy b/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_common.groovy index 64ad8ba8f08..23f7a8c2d79 100644 --- a/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_common.groovy +++ b/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_common.groovy @@ -60,8 +60,8 @@ suite("test_decimalv2_common", "nonConcurrent") { `decimal_key2` decimalv2(16, 5) NULL COMMENT "", `decimal_value1` decimalv2(8, 5) NULL COMMENT "", `decimal_value2` decimalv2(16, 5) NULL COMMENT "", - INDEX `idx_key1` (`decimal_value1`) USING BITMAP, - INDEX `idx_key2` (`decimal_value2`) USING BITMAP + INDEX `idx_key1` (`decimal_key1`) USING BITMAP, + INDEX `idx_key2` (`decimal_key2`) USING BITMAP ) ENGINE=OLAP UNIQUE KEY(`decimal_key1`, `decimal_key2`) COMMENT "OLAP" diff --git a/regression-test/suites/datatype_p0/scalar_types/load.groovy b/regression-test/suites/datatype_p0/scalar_types/load.groovy index 054958bf5b9..0c5c81576a3 100644 --- a/regression-test/suites/datatype_p0/scalar_types/load.groovy +++ b/regression-test/suites/datatype_p0/scalar_types/load.groovy @@ -306,61 +306,6 @@ suite("test_scalar_types_load", "p0") { `c_char`, `c_varchar`, `c_string` FROM tbl_scalar_types_dup""" - // define unique key table2 disable mow - testTable = "tbl_scalar_types_unique2_bitmapindex" - sql "DROP TABLE IF EXISTS ${testTable}" - sql """ - CREATE TABLE IF NOT EXISTS ${testTable} ( - `c_datetimev2` datetimev2(0) NULL, - `c_bigint` bigint(20) NULL, - `c_decimalv3` decimalv3(20, 3) NULL, - `c_bool` boolean NULL, - `c_tinyint` tinyint(4) NULL, - `c_smallint` smallint(6) NULL, - `c_int` int(11) NULL, - `c_largeint` largeint(40) NULL, - `c_float` float NULL, - `c_double` double NULL, - `c_decimal` decimal(20, 3) NULL, - `c_date` date NULL, - `c_datetime` datetime NULL, - `c_datev2` datev2 NULL, - `c_char` char(15) NULL, - `c_varchar` varchar(100) NULL, - `c_string` text NULL, - INDEX idx_c_bool (c_bool) USING BITMAP, - INDEX idx_c_tinyint (c_tinyint) USING BITMAP, - INDEX idx_c_smallint (c_smallint) USING BITMAP, - INDEX idx_c_int (c_int) USING BITMAP, - INDEX idx_c_bigint (c_bigint) USING BITMAP, - INDEX idx_c_largeint (c_largeint) USING BITMAP, - INDEX idx_c_decimal (c_decimal) USING BITMAP, - INDEX idx_c_decimalv3 (c_decimalv3) USING BITMAP, - INDEX idx_c_date (c_date) USING BITMAP, - INDEX idx_c_datetime (c_datetime) USING BITMAP, - INDEX idx_c_datev2 (c_datev2) USING BITMAP, - INDEX idx_c_datetimev2 (c_datetimev2) USING BITMAP, - INDEX idx_c_char (c_char) USING BITMAP, - INDEX idx_c_varchar (c_varchar) USING BITMAP, - INDEX idx_c_string (c_string) USING BITMAP - ) ENGINE=OLAP - UNIQUE KEY(`c_datetimev2`, `c_bigint`, `c_decimalv3`) - COMMENT 'OLAP' - DISTRIBUTED BY HASH(`c_bigint`) BUCKETS 10 - PROPERTIES("replication_num" = "1", "enable_unique_key_merge_on_write" = "false"); - """ - - // insert data into unique key table1 2 times - sql """INSERT INTO ${testTable} SELECT `c_datetimev2`, `c_bigint`, `c_decimalv3`, - `c_bool`, `c_tinyint`, `c_smallint`, `c_int`, `c_largeint`, - `c_float`, `c_double`, `c_decimal`, `c_date`, `c_datetime`, `c_datev2`, - `c_char`, `c_varchar`, `c_string` FROM tbl_scalar_types_dup""" - sql """INSERT INTO ${testTable} SELECT `c_datetimev2`, `c_bigint`, `c_decimalv3`, - `c_bool`, `c_tinyint`, `c_smallint`, `c_int`, `c_largeint`, - `c_float`, `c_double`, `c_decimal`, `c_date`, `c_datetime`, `c_datev2`, - `c_char`, `c_varchar`, `c_string` FROM tbl_scalar_types_dup""" - - // define dup key table with index testTable = "tbl_scalar_types_dup_inverted_index" sql "DROP TABLE IF EXISTS ${testTable}" diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q01_bool.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q01_bool.sql deleted file mode 100644 index 3e384181688..00000000000 --- a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q01_bool.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT count() FROM tbl_scalar_types_unique2_bitmapindex WHERE c_bool = 1; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q02_tinyint.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q02_tinyint.sql deleted file mode 100644 index ed75ebca457..00000000000 --- a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q02_tinyint.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT count() FROM tbl_scalar_types_unique2_bitmapindex WHERE c_tinyint = 14; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q03_smallint.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q03_smallint.sql deleted file mode 100644 index faa2159f0dd..00000000000 --- a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q03_smallint.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_smallint = 12321 ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q04_int.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q04_int.sql deleted file mode 100644 index db82b79873a..00000000000 --- a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q04_int.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_int = -16441 ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q05_bigint.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q05_bigint.sql deleted file mode 100644 index 9ef4811f62a..00000000000 --- a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q05_bigint.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_bigint = 859063837 ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q06_largeint.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q06_largeint.sql deleted file mode 100644 index 2a6b48064c2..00000000000 --- a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q06_largeint.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_largeint = 1524137702 ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q07_float.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q07_float.sql deleted file mode 100644 index 1ae3d82f151..00000000000 --- a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q07_float.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_float = -13023.051 ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q08_double.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q08_double.sql deleted file mode 100644 index 8c21bc08b79..00000000000 --- a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q08_double.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_double = -679666086.055166 ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q09_decimal.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q09_decimal.sql deleted file mode 100644 index 9fd1e832bed..00000000000 --- a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q09_decimal.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_decimal = 5355625357307892.981 ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q10_decimalv3.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q10_decimalv3.sql deleted file mode 100644 index ee676941451..00000000000 --- a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q10_decimalv3.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_decimalv3 = 89599956401481176.347 ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q11_date.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q11_date.sql deleted file mode 100644 index 7f5d383765f..00000000000 --- a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q11_date.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_date = '2022-10-09' ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q12_datetime.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q12_datetime.sql deleted file mode 100644 index a1d7728c621..00000000000 --- a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q12_datetime.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_datetime = '2022-07-31 12:07:00' ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q13_datev2.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q13_datev2.sql deleted file mode 100644 index eee7b111084..00000000000 --- a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q13_datev2.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_datev2 = '2022-03-06' ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q14_datetimev2.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q14_datetimev2.sql deleted file mode 100644 index 83b86dadbf2..00000000000 --- a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q14_datetimev2.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_datetimev2 = '2022-11-18 06:05:56' ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q15_char.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q15_char.sql deleted file mode 100644 index 67486dde69f..00000000000 --- a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q15_char.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_char = '202.171.74.217' ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q16_varchar.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q16_varchar.sql deleted file mode 100644 index 677bc5614a8..00000000000 --- a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q16_varchar.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_varchar = 'mod...@feedmix.biz' ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q17_string.sql b/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q17_string.sql deleted file mode 100644 index b02fb1d72e8..00000000000 --- a/regression-test/suites/datatype_p0/scalar_types/sql/unique2_bitmap_index_q17_string.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT * FROM tbl_scalar_types_unique2_bitmapindex WHERE c_string = 'Reinke Junction 63' ORDER BY c_bigint, c_largeint; diff --git a/regression-test/suites/index_p0/test_bitmap_index.groovy b/regression-test/suites/index_p0/test_bitmap_index.groovy index 61849135e1d..a5cbe31504f 100644 --- a/regression-test/suites/index_p0/test_bitmap_index.groovy +++ b/regression-test/suites/index_p0/test_bitmap_index.groovy @@ -175,7 +175,7 @@ suite("test_bitmap_index") { } test{ sql "ALTER TABLE ${tbName2} ADD INDEX index16 (v1) USING BITMAP;" - exception "errCode = 2, detailMessage = BITMAP index only used in columns of DUP_KEYS/UNIQUE_KEYS table" + exception "errCode = 2, detailMessage = INVERTED index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table. invalid index: index16" } sql "insert into ${tbName2} values(1,1,1,1,'1','1','2022-05-31','2022-05-31 10:00:00',1,1.0,1,'2022-05-31','2022-05-31 10:00:00.111111','2022-05-31 10:00:00.111111','2022-05-31 10:00:00.111111',1);" @@ -254,12 +254,7 @@ suite("test_bitmap_index") { ADD INDEX index8 (k8) USING BITMAP, ADD INDEX index9 (k9) USING BITMAP, ADD INDEX index10 (k10) USING BITMAP, - ADD INDEX index11 (k11) USING BITMAP, - ADD INDEX index12 (k12) USING BITMAP, - ADD INDEX index13 (k13) USING BITMAP, - ADD INDEX index14 (k14) USING BITMAP, - ADD INDEX index15 (k15) USING BITMAP, - ADD INDEX index16 (v1) USING BITMAP; + ADD INDEX index11 (k11) USING BITMAP; """ max_try_secs = 60 while (max_try_secs--) { @@ -280,7 +275,7 @@ suite("test_bitmap_index") { qt_sql "desc ${tbName3};" show_result = sql "show index from ${tbName3}" logger.info("show index from " + tbName3 + " result: " + show_result) - assertEquals(show_result.size(), 16) + assertEquals(show_result.size(), 11) assertEquals(show_result[0][2], "index1") assertEquals(show_result[1][2], "index2") assertEquals(show_result[2][2], "index3") @@ -292,11 +287,6 @@ suite("test_bitmap_index") { assertEquals(show_result[8][2], "index9") assertEquals(show_result[9][2], "index10") assertEquals(show_result[10][2], "index11") - assertEquals(show_result[11][2], "index12") - assertEquals(show_result[12][2], "index13") - assertEquals(show_result[13][2], "index14") - assertEquals(show_result[14][2], "index15") - assertEquals(show_result[15][2], "index16") qt_sql "select * from ${tbName3};" sql "DROP INDEX IF EXISTS index1 ON ${tbName3};" diff --git a/regression-test/suites/index_p0/test_index_meta.groovy b/regression-test/suites/index_p0/test_index_meta.groovy index abec7ce2cd6..e8c3c271c8e 100644 --- a/regression-test/suites/index_p0/test_index_meta.groovy +++ b/regression-test/suites/index_p0/test_index_meta.groovy @@ -47,7 +47,7 @@ suite("index_meta", "p0") { `id` INT NULL, `name` STRING NULL, `description` STRING NULL, - INDEX idx_id (`id`) USING BITMAP COMMENT 'index for id', + INDEX idx_id (`id`) USING INVERTED COMMENT 'index for id', INDEX idx_name (`name`) USING INVERTED PROPERTIES("parser"="none") COMMENT 'index for name' ) DUPLICATE KEY(`id`) @@ -64,7 +64,7 @@ suite("index_meta", "p0") { assertEquals(show_result.size(), 2) assertEquals(show_result[0][2], "idx_id") assertEquals(show_result[0][4], "id") - assertEquals(show_result[0][10], "BITMAP") + assertEquals(show_result[0][10], "INVERTED") assertEquals(show_result[0][11], "index for id") assertEquals(show_result[0][12], "") assertEquals(show_result[1][2], "idx_name") @@ -83,7 +83,7 @@ suite("index_meta", "p0") { assertEquals(show_result.size(), 3) assertEquals(show_result[0][2], "idx_id") assertEquals(show_result[0][4], "id") - assertEquals(show_result[0][10], "BITMAP") + assertEquals(show_result[0][10], "INVERTED") assertEquals(show_result[0][11], "index for id") assertEquals(show_result[0][12], "") assertEquals(show_result[1][2], "idx_name") @@ -107,7 +107,7 @@ suite("index_meta", "p0") { assertEquals(show_result.size(), 2) assertEquals(show_result[0][2], "idx_id") assertEquals(show_result[0][4], "id") - assertEquals(show_result[0][10], "BITMAP") + assertEquals(show_result[0][10], "INVERTED") assertEquals(show_result[0][11], "index for id") assertEquals(show_result[0][12], "") assertEquals(show_result[1][2], "idx_desc") @@ -126,7 +126,7 @@ suite("index_meta", "p0") { assertEquals(show_result.size(), 3) assertEquals(show_result[0][2], "idx_id") assertEquals(show_result[0][4], "id") - assertEquals(show_result[0][10], "BITMAP") + assertEquals(show_result[0][10], "INVERTED") assertEquals(show_result[0][11], "index for id") assertEquals(show_result[0][12], "") assertEquals(show_result[1][2], "idx_desc") diff --git a/regression-test/suites/inverted_index_p0/test_add_drop_index.groovy b/regression-test/suites/inverted_index_p0/test_add_drop_index.groovy index 9bdff40418d..540539dbf2f 100644 --- a/regression-test/suites/inverted_index_p0/test_add_drop_index.groovy +++ b/regression-test/suites/inverted_index_p0/test_add_drop_index.groovy @@ -61,7 +61,7 @@ suite("test_add_drop_index", "inverted_index"){ // case1: create index for int colume // case1.0 create index - sql "create index age_idx on ${indexTbName1}(age) using inverted" + sql "create index age_idx on ${indexTbName1}(tearchComment) using inverted" wait_for_latest_op_on_table_finish(indexTbName1, timeout) def show_result = sql "show index from ${indexTbName1}" @@ -71,7 +71,7 @@ suite("test_add_drop_index", "inverted_index"){ // case1.1 create duplicate same index for one colume with same name def create_dup_index_result = "fail" try { - sql "create index age_idx on ${indexTbName1}(`age`) using inverted" + sql "create index age_idx on ${indexTbName1}(`tearchComment`) using inverted" create_dup_index_result = "success" } catch(Exception ex) { logger.info("create same duplicate and same name index, result: " + ex) @@ -79,14 +79,14 @@ suite("test_add_drop_index", "inverted_index"){ assertEquals(create_dup_index_result, "fail") // case1.2 create duplicate same index for one colume with different name try { - sql "create index age_idx_diff on ${indexTbName1}(`age`) using inverted" + sql "create index age_idx_diff on ${indexTbName1}(`tearchComment`) using inverted" create_dup_index_result = "success" } catch(Exception ex) { logger.info("create same duplicate with different name index, result: " + ex) } assertEquals(create_dup_index_result, "fail") // case1.3 create duplicate different index for one colume with different name - sql "create index age_idx_diff on ${indexTbName1}(`age`) using bitmap" + sql "create index age_idx_diff on ${indexTbName1}(`tearchComment`) using NGRAM_BF" wait_for_latest_op_on_table_finish(indexTbName1, timeout) show_result = sql "show index from ${indexTbName1}" logger.info("show index from " + indexTbName1 + " result: " + show_result) @@ -124,10 +124,10 @@ suite("test_add_drop_index", "inverted_index"){ // case3: create string inverted index for int colume def create_string_index_on_int_colume_result = "fail" try { - syntax_error = sql "create index age_idx on ${indexTbName1}(`age`) USING INVERTED PROPERTIES("parser"="standard")" + syntax_error = sql "create index age_idx on ${indexTbName1}(`tearchComment`) USING INVERTED PROPERTIES("parser"="standard")" create_string_index_on_int_colume_result = "success" } catch(Exception ex) { - logger.info("sql: create index age_idx on" + indexTbName1 + "(`age`) USING INVERTED PROPERTIES(\"parser\"=\"standard\")") + logger.info("sql: create index age_idx on" + indexTbName1 + "(`tearchComment`) USING INVERTED PROPERTIES(\"parser\"=\"standard\")") logger.info("create string inverted index for int colume, result: " + ex) } assertEquals(create_string_index_on_int_colume_result, "fail") diff --git a/regression-test/suites/inverted_index_p0/test_bitmap_index.groovy b/regression-test/suites/inverted_index_p0/test_bitmap_index.groovy index 0b98fdc17b3..c6bbc208924 100644 --- a/regression-test/suites/inverted_index_p0/test_bitmap_index.groovy +++ b/regression-test/suites/inverted_index_p0/test_bitmap_index.groovy @@ -155,7 +155,7 @@ suite("test_bitmap_index", "inverted_index") { } test{ sql "ALTER TABLE ${tbName2} ADD INDEX index12 (v1) USING BITMAP;" - exception "errCode = 2, detailMessage = BITMAP index only used in columns of DUP_KEYS/UNIQUE_KEYS table or key columns of AGG_KEYS table. invalid column: v1" + exception "errCode = 2, detailMessage = INVERTED index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table. invalid index: index12" } sql "insert into ${tbName2} values(1,1,1,1,'1','1','2022-05-31','2022-05-31 10:00:00',1,1.0,1,1);" @@ -226,8 +226,7 @@ suite("test_bitmap_index", "inverted_index") { ADD INDEX index8 (k8) USING BITMAP, ADD INDEX index9 (k9) USING BITMAP, ADD INDEX index10 (k10) USING BITMAP, - ADD INDEX index11 (k11) USING BITMAP, - ADD INDEX index12 (v1) USING BITMAP; + ADD INDEX index11 (k11) USING BITMAP; """ max_try_secs = 60 while (max_try_secs--) { @@ -248,7 +247,7 @@ suite("test_bitmap_index", "inverted_index") { qt_sql "desc ${tbName3};" show_result = sql "show index from ${tbName3}" logger.info("show index from " + tbName3 + " result: " + show_result) - assertEquals(show_result.size(), 12) + assertEquals(show_result.size(), 11) assertEquals(show_result[0][2], "index1") assertEquals(show_result[1][2], "index2") assertEquals(show_result[2][2], "index3") @@ -260,7 +259,6 @@ suite("test_bitmap_index", "inverted_index") { assertEquals(show_result[8][2], "index9") assertEquals(show_result[9][2], "index10") assertEquals(show_result[10][2], "index11") - assertEquals(show_result[11][2], "index12") qt_sql "select * from ${tbName3};" sql "DROP INDEX IF EXISTS index1 ON ${tbName3};" diff --git a/regression-test/suites/inverted_index_p0/test_create_index_1.groovy b/regression-test/suites/inverted_index_p0/test_create_index_1.groovy index 49bd5033dd4..e2ba1609dbc 100644 --- a/regression-test/suites/inverted_index_p0/test_create_index_1.groovy +++ b/regression-test/suites/inverted_index_p0/test_create_index_1.groovy @@ -120,8 +120,7 @@ suite("test_create_index_1", "inverted_index"){ registDate datetime NULL, studentInfo char(100), tearchComment string, - INDEX age_idx_1(age) USING BITMAP COMMENT 'age index', - INDEX age_idx_2(age) USING INVERTED COMMENT 'age index' + INDEX age_idx_1(age) USING INVERTED COMMENT 'age index' ) DUPLICATE KEY(`name`) DISTRIBUTED BY HASH(`name`) BUCKETS 10 @@ -130,25 +129,23 @@ suite("test_create_index_1", "inverted_index"){ def show_result = sql "show index from ${indexTbName1}" logger.info("show index from " + indexTbName1 + " result: " + show_result) - assertEquals(show_result.size(), 2) + assertEquals(show_result.size(), 1) assertEquals(show_result[0][2], "age_idx_1") - assertEquals(show_result[1][2], "age_idx_2") // drop index sql "drop index age_idx_1 on ${indexTbName1}" wait_for_latest_op_on_table_finish(indexTbName1, timeout) - sql "drop index age_idx_2 on ${indexTbName1}" show_result = sql "show index from ${indexTbName1}" assertEquals(show_result.size(), 0) // case 2: alter add index - sql "create index age_idx on ${indexTbName1}(age) using inverted" + sql "create index studentInfo_idx on ${indexTbName1}(studentInfo) using inverted" show_result = sql "show index from ${indexTbName1}" logger.info("show index from " + indexTbName1 + " result: " + show_result) - assertEquals(show_result[0][2], "age_idx") + assertEquals(show_result[0][2], "studentInfo_idx") // case 2.1: create duplicate same index for one colume with same name try { - sql "create index age_idx on ${indexTbName1}(`age`) using inverted" + sql "create index studentInfo_idx on ${indexTbName1}(`studentInfo`) using inverted" create_dup_index_result = "success" } catch(Exception ex) { logger.info("create duplicate same index for one colume with same name, result: " + ex) @@ -156,7 +153,7 @@ suite("test_create_index_1", "inverted_index"){ // case 2.2: create duplicate same index for one colume with different name try { - sql "create index age_idx_2 on ${indexTbName1}(`age`) using inverted" + sql "create index studentInfo_idx_2 on ${indexTbName1}(`studentInfo`) using inverted" create_dup_index_result = "success" } catch(Exception ex) { logger.info("create duplicate same index for one colume with different name, result: " + ex) @@ -164,19 +161,19 @@ suite("test_create_index_1", "inverted_index"){ // case 2.3: create duplicate different index for one colume with same name try { - sql "create index age_idx on ${indexTbName1}(`age`) using bitmap" + sql "create index studentInfo_idx on ${indexTbName1}(`studentInfo`) using NGRAM_BF" create_dup_index_result = "success" } catch(Exception ex) { logger.info("create duplicate different index for one colume with same name, result: " + ex) } // 2.4: create duplicate different index for one colume with different name - sql "create index age_idx_2 on ${indexTbName1}(`age`) using bitmap" + sql "create index studentInfo_idx_2 on ${indexTbName1}(`studentInfo`) using NGRAM_BF" wait_for_latest_op_on_table_finish(indexTbName1, timeout) show_result = sql "show index from ${indexTbName1}" logger.info("show index from " + indexTbName1 + " result: " + show_result) assertEquals(show_result.size(), 2) - assertEquals(show_result[0][2], "age_idx") - assertEquals(show_result[1][2], "age_idx_2") + assertEquals(show_result[0][2], "studentInfo_idx") + assertEquals(show_result[1][2], "studentInfo_idx_2") } diff --git a/regression-test/suites/inverted_index_p0/test_inverted_index.groovy b/regression-test/suites/inverted_index_p0/test_inverted_index.groovy index 58ef93f7d1a..60d85999763 100644 --- a/regression-test/suites/inverted_index_p0/test_inverted_index.groovy +++ b/regression-test/suites/inverted_index_p0/test_inverted_index.groovy @@ -175,7 +175,7 @@ suite("test_inverted_index", "inverted_index") { } test{ sql "ALTER TABLE ${tbName2} ADD INDEX index16 (v1) USING INVERTED;" - exception "errCode = 2, detailMessage = INVERTED index only used in columns of DUP_KEYS table or UNIQUE_KEYS table with merge_on_write enabled or key columns of AGG_KEYS table. invalid column: v1" + exception "errCode = 2, detailMessage = INVERTED index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table. invalid index: index16" } sql "insert into ${tbName2} values(1,1,1,1,'1','1','2022-05-31','2022-05-31 10:00:00',1,1.0,1,'2022-05-31','2022-05-31 10:00:00.111111','2022-05-31 10:00:00.111111','2022-05-31 10:00:00.111111',1);" diff --git a/regression-test/suites/load_p0/broker_load/ddl/mow_tbl_basic.sql b/regression-test/suites/load_p0/broker_load/ddl/mow_tbl_basic.sql index dbb45d263f0..e06f648580c 100644 --- a/regression-test/suites/load_p0/broker_load/ddl/mow_tbl_basic.sql +++ b/regression-test/suites/load_p0/broker_load/ddl/mow_tbl_basic.sql @@ -43,11 +43,6 @@ CREATE TABLE mow_tbl_basic INDEX idx_inverted_k113 (`k13`) USING INVERTED, INDEX idx_inverted_k114 (`k14`) USING INVERTED, INDEX idx_inverted_k117 (`k17`) USING INVERTED PROPERTIES("parser" = "english"), - INDEX idx_bitmap_k104 (`k05`) USING BITMAP, - INDEX idx_bitmap_k110 (`k11`) USING BITMAP, - INDEX idx_bitmap_k113 (`k13`) USING BITMAP, - INDEX idx_bitmap_k114 (`k14`) USING BITMAP, - INDEX idx_bitmap_k117 (`k17`) USING BITMAP, INDEX idx_ngrambf_k115 (`k15`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256"), INDEX idx_ngrambf_k116 (`k16`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256"), INDEX idx_ngrambf_k117 (`k17`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256") diff --git a/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic.sql b/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic.sql index 993e7046ec9..560a9a78a3d 100644 --- a/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic.sql +++ b/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic.sql @@ -36,13 +36,7 @@ CREATE TABLE uniq_tbl_basic kd15 CHAR(255) NOT NULL DEFAULT "我能吞下玻璃而不伤身体", kd16 VARCHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体", kd17 STRING NOT NULL DEFAULT "我能吞下玻璃而不伤身体", - kd18 JSON NULL, - - INDEX idx_bitmap_k104 (`k02`) USING BITMAP, - INDEX idx_bitmap_k110 (`kd01`) USING BITMAP, - INDEX idx_bitmap_k113 (`k13`) USING BITMAP, - INDEX idx_bitmap_k114 (`k14`) USING BITMAP, - INDEX idx_bitmap_k117 (`k17`) USING BITMAP + kd18 JSON NULL ) UNIQUE KEY(k00,k01) PARTITION BY RANGE(k01) diff --git a/regression-test/suites/load_p0/broker_load/test_seq_load.groovy b/regression-test/suites/load_p0/broker_load/test_seq_load.groovy index 0c94467c6a5..eca3074bfdb 100644 --- a/regression-test/suites/load_p0/broker_load/test_seq_load.groovy +++ b/regression-test/suites/load_p0/broker_load/test_seq_load.groovy @@ -60,13 +60,7 @@ suite("test_seq_load", "load_p0") { kd15 CHAR(255) NOT NULL DEFAULT "我能吞下玻璃而不伤身体", kd16 VARCHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体", kd17 STRING NOT NULL DEFAULT "我能吞下玻璃而不伤身体", - kd18 JSON NULL, - - INDEX idx_bitmap_k104 (`k02`) USING BITMAP, - INDEX idx_bitmap_k110 (`kd01`) USING BITMAP, - INDEX idx_bitmap_k113 (`k13`) USING BITMAP, - INDEX idx_bitmap_k114 (`k14`) USING BITMAP, - INDEX idx_bitmap_k117 (`k17`) USING BITMAP + kd18 JSON NULL ) UNIQUE KEY(k00,k01) PARTITION BY RANGE(k01) diff --git a/regression-test/suites/load_p0/routine_load/ddl/mow_tbl_basic_create.sql b/regression-test/suites/load_p0/routine_load/ddl/mow_tbl_basic_create.sql index c2398b9d038..9a15dac0418 100644 --- a/regression-test/suites/load_p0/routine_load/ddl/mow_tbl_basic_create.sql +++ b/regression-test/suites/load_p0/routine_load/ddl/mow_tbl_basic_create.sql @@ -43,11 +43,6 @@ CREATE TABLE routine_load_mow_tbl_basic INDEX idx_inverted_k113 (`k13`) USING INVERTED, INDEX idx_inverted_k114 (`k14`) USING INVERTED, INDEX idx_inverted_k117 (`k17`) USING INVERTED PROPERTIES("parser" = "english"), - INDEX idx_bitmap_k104 (`k05`) USING BITMAP, - INDEX idx_bitmap_k110 (`k11`) USING BITMAP, - INDEX idx_bitmap_k113 (`k13`) USING BITMAP, - INDEX idx_bitmap_k114 (`k14`) USING BITMAP, - INDEX idx_bitmap_k117 (`k17`) USING BITMAP, INDEX idx_ngrambf_k115 (`k15`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256"), INDEX idx_ngrambf_k116 (`k16`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256"), INDEX idx_ngrambf_k117 (`k17`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256") diff --git a/regression-test/suites/load_p0/routine_load/ddl/uniq_tbl_basic_create.sql b/regression-test/suites/load_p0/routine_load/ddl/uniq_tbl_basic_create.sql index 68a78e9b755..38b9d267742 100644 --- a/regression-test/suites/load_p0/routine_load/ddl/uniq_tbl_basic_create.sql +++ b/regression-test/suites/load_p0/routine_load/ddl/uniq_tbl_basic_create.sql @@ -36,13 +36,7 @@ CREATE TABLE routine_load_uniq_tbl_basic kd15 CHAR(255) NOT NULL DEFAULT "我能吞下玻璃而不伤身体", kd16 VARCHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体", kd17 STRING NOT NULL DEFAULT "我能吞下玻璃而不伤身体", - kd18 JSON NULL, - - INDEX idx_bitmap_k104 (`k02`) USING BITMAP, - INDEX idx_bitmap_k110 (`kd01`) USING BITMAP, - INDEX idx_bitmap_k113 (`k13`) USING BITMAP, - INDEX idx_bitmap_k114 (`k14`) USING BITMAP, - INDEX idx_bitmap_k117 (`k17`) USING BITMAP + kd18 JSON NULL ) UNIQUE KEY(k00,k01) PARTITION BY RANGE(k01) diff --git a/regression-test/suites/load_p0/stream_load/ddl/mow_tbl_basic_create.sql b/regression-test/suites/load_p0/stream_load/ddl/mow_tbl_basic_create.sql index b64602b2e7f..d7421817002 100644 --- a/regression-test/suites/load_p0/stream_load/ddl/mow_tbl_basic_create.sql +++ b/regression-test/suites/load_p0/stream_load/ddl/mow_tbl_basic_create.sql @@ -43,11 +43,6 @@ CREATE TABLE stream_load_mow_tbl_basic INDEX idx_inverted_k113 (`k13`) USING INVERTED, INDEX idx_inverted_k114 (`k14`) USING INVERTED, INDEX idx_inverted_k117 (`k17`) USING INVERTED PROPERTIES("parser" = "english"), - INDEX idx_bitmap_k104 (`k05`) USING BITMAP, - INDEX idx_bitmap_k110 (`k11`) USING BITMAP, - INDEX idx_bitmap_k113 (`k13`) USING BITMAP, - INDEX idx_bitmap_k114 (`k14`) USING BITMAP, - INDEX idx_bitmap_k117 (`k17`) USING BITMAP, INDEX idx_ngrambf_k115 (`k15`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256"), INDEX idx_ngrambf_k116 (`k16`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256"), INDEX idx_ngrambf_k117 (`k17`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256") diff --git a/regression-test/suites/load_p0/stream_load/ddl/uniq_tbl_basic_create.sql b/regression-test/suites/load_p0/stream_load/ddl/uniq_tbl_basic_create.sql index 44d6ff98b2e..d6e4f5a8c48 100644 --- a/regression-test/suites/load_p0/stream_load/ddl/uniq_tbl_basic_create.sql +++ b/regression-test/suites/load_p0/stream_load/ddl/uniq_tbl_basic_create.sql @@ -36,13 +36,7 @@ CREATE TABLE stream_load_uniq_tbl_basic kd15 CHAR(255) NOT NULL DEFAULT "我能吞下玻璃而不伤身体", kd16 VARCHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体", kd17 STRING NOT NULL DEFAULT "我能吞下玻璃而不伤身体", - kd18 JSON NULL, - - INDEX idx_bitmap_k104 (`k02`) USING BITMAP, - INDEX idx_bitmap_k110 (`kd01`) USING BITMAP, - INDEX idx_bitmap_k113 (`k13`) USING BITMAP, - INDEX idx_bitmap_k114 (`k14`) USING BITMAP, - INDEX idx_bitmap_k117 (`k17`) USING BITMAP + kd18 JSON NULL ) UNIQUE KEY(k00,k01) PARTITION BY RANGE(k01) diff --git a/regression-test/suites/load_p0/stream_load/ddl/uniq_tbl_basic_create_sequence.sql b/regression-test/suites/load_p0/stream_load/ddl/uniq_tbl_basic_create_sequence.sql index 4695b0fc2c8..557db73feaf 100644 --- a/regression-test/suites/load_p0/stream_load/ddl/uniq_tbl_basic_create_sequence.sql +++ b/regression-test/suites/load_p0/stream_load/ddl/uniq_tbl_basic_create_sequence.sql @@ -36,13 +36,7 @@ CREATE TABLE stream_load_uniq_tbl_basic_sequence kd15 CHAR(255) NOT NULL DEFAULT "我能吞下玻璃而不伤身体", kd16 VARCHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体", kd17 STRING NOT NULL DEFAULT "我能吞下玻璃而不伤身体", - kd18 JSON NULL, - - INDEX idx_bitmap_k104 (`k02`) USING BITMAP, - INDEX idx_bitmap_k110 (`kd01`) USING BITMAP, - INDEX idx_bitmap_k113 (`k13`) USING BITMAP, - INDEX idx_bitmap_k114 (`k14`) USING BITMAP, - INDEX idx_bitmap_k117 (`k17`) USING BITMAP + kd18 JSON NULL ) UNIQUE KEY(k00,k01) DISTRIBUTED BY HASH(k00) BUCKETS 32 diff --git a/regression-test/suites/load_p0/tvf/ddl/mow_tbl_basic_tvf.sql b/regression-test/suites/load_p0/tvf/ddl/mow_tbl_basic_tvf.sql index b34e7749317..b20126a7768 100644 --- a/regression-test/suites/load_p0/tvf/ddl/mow_tbl_basic_tvf.sql +++ b/regression-test/suites/load_p0/tvf/ddl/mow_tbl_basic_tvf.sql @@ -43,11 +43,6 @@ CREATE TABLE mow_tbl_basic_tvf INDEX idx_inverted_k113 (`k13`) USING INVERTED, INDEX idx_inverted_k114 (`k14`) USING INVERTED, INDEX idx_inverted_k117 (`k17`) USING INVERTED PROPERTIES("parser" = "english"), - INDEX idx_bitmap_k104 (`k05`) USING BITMAP, - INDEX idx_bitmap_k110 (`k11`) USING BITMAP, - INDEX idx_bitmap_k113 (`k13`) USING BITMAP, - INDEX idx_bitmap_k114 (`k14`) USING BITMAP, - INDEX idx_bitmap_k117 (`k17`) USING BITMAP, INDEX idx_ngrambf_k115 (`k15`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256"), INDEX idx_ngrambf_k116 (`k16`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256"), INDEX idx_ngrambf_k117 (`k17`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256") diff --git a/regression-test/suites/load_p0/tvf/ddl/uniq_tbl_basic_tvf.sql b/regression-test/suites/load_p0/tvf/ddl/uniq_tbl_basic_tvf.sql index 8af1681fec8..d477f030289 100644 --- a/regression-test/suites/load_p0/tvf/ddl/uniq_tbl_basic_tvf.sql +++ b/regression-test/suites/load_p0/tvf/ddl/uniq_tbl_basic_tvf.sql @@ -36,13 +36,7 @@ CREATE TABLE uniq_tbl_basic_tvf kd15 CHAR(255) NOT NULL DEFAULT "我能吞下玻璃而不伤身体", kd16 VARCHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体", kd17 STRING NOT NULL DEFAULT "我能吞下玻璃而不伤身体", - kd18 JSON NULL, - - INDEX idx_bitmap_k104 (`k02`) USING BITMAP, - INDEX idx_bitmap_k110 (`kd01`) USING BITMAP, - INDEX idx_bitmap_k113 (`k13`) USING BITMAP, - INDEX idx_bitmap_k114 (`k14`) USING BITMAP, - INDEX idx_bitmap_k117 (`k17`) USING BITMAP + kd18 JSON NULL ) UNIQUE KEY(k00,k01) PARTITION BY RANGE(k01) diff --git a/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys_index.groovy b/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys_index.groovy index 0cbb33c39c5..4e716f48951 100644 --- a/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys_index.groovy +++ b/regression-test/suites/segcompaction_p2/test_segcompaction_unique_keys_index.groovy @@ -68,7 +68,7 @@ suite("test_segcompaction_unique_keys_index") { UNIQUE KEY(`col_0`) DISTRIBUTED BY HASH(`col_0`) BUCKETS 1 PROPERTIES ( "replication_num" = "1" ); """ - exception "INVERTED index only used in columns of DUP_KEYS table or UNIQUE_KEYS table with merge_on_write enabled or key columns of AGG_KEYS table. invalid column: col_0" + exception "INVERTED index only used in columns of DUP_KEYS/UNIQUE_KEYS MOW table or key columns of all table. invalid indexName: index_col_1" } } finally { try_sql("DROP TABLE IF EXISTS ${tableName}") --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org