This is an automated email from the ASF dual-hosted git repository.
airborne pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 99d4294a16d [feature](index change)Support light index change for
ngram bf index (#48461)
99d4294a16d is described below
commit 99d4294a16dc4f46174fb3b044d70ae34111e274
Author: qiye <jianliang5...@gmail.com>
AuthorDate: Thu May 8 11:30:01 2025 +0800
[feature](index change)Support light index change for ngram bf index
(#48461)
### What problem does this PR solve?
Currently, NGram bloom filter index only supports directly schema
change, and users need to build indexes incrementally when using it.
The design goal is that ngrambf supports light_index_change, including
local and cloud mode, which can incrementally add indexes or build
indexes on stock data.
Inverted indexes are currently only supported in local mode for
light_schema_change, cloud mode is still a directly SC, this time it
does not involve inverted indexes, and its functionality remains
unchanged.
After the completion of the function, the NGram BF index construction
can be used in the following way, following the existing syntax, does
not involve changes or additions.
```sql
# add new index
alter table t1 add index idx_ngram_k2 (`k2`) using ngram_bf
properties("bf_size" = "1024", "gram_size" = "3");
create index idx_ngram_k2 (`k2`) on t1 using ngram_bf properties("bf_size"
= "1024", "gram_size" = "3");
# build index on stock data
build index idx_ngram_k2 on t1;
# show build ngram index
show alter table column;
# cancel build index
cancel build index on t1;
```
**NOTE:** Currently, building an index by partition is not supported. If
you want to build an index for stock data, you need to build it for all
data, including new data written after the Add index has been added.
Build index by partition will be supported in next stage.
### Release note
Support light index change for NGram bf index
---
fe/fe-core/src/main/cup/sql_parser.cup | 10 +-
.../apache/doris/alter/SchemaChangeHandler.java | 111 ++++----
.../org/apache/doris/alter/SchemaChangeJobV2.java | 15 +-
.../apache/doris/analysis/BuildIndexClause.java | 63 ++++-
.../java/org/apache/doris/analysis/IndexDef.java | 4 +-
.../main/java/org/apache/doris/catalog/Index.java | 12 +-
.../java/org/apache/doris/catalog/OlapTable.java | 14 +
.../doris/nereids/parser/LogicalPlanBuilder.java | 4 +-
.../trees/plans/commands/info/BuildIndexOp.java | 68 ++++-
.../trees/plans/commands/info/IndexDefinition.java | 10 +-
.../org/apache/doris/alter/IndexChangeJobTest.java | 156 ++++++++++-
.../doris/alter/SchemaChangeHandlerTest.java | 9 +-
.../test_ngram_bloomfilter_index_change.out | Bin 0 -> 3569 bytes
.../pipeline/cloud_p0/conf/fe_custom.conf | 1 -
.../pipeline/cloud_p1/conf/fe_custom.conf | 1 -
.../pipeline/vault_p0/conf/fe_custom.conf | 1 -
.../test_ngram_bloomfilter_index_change.groovy | 286 +++++++++++++++++++++
17 files changed, 639 insertions(+), 126 deletions(-)
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup
b/fe/fe-core/src/main/cup/sql_parser.cup
index fa597fcb3d1..8abddccf7d3 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -879,7 +879,6 @@ nonterminal Map<String, String> key_value_map,
opt_key_value_map, opt_key_value_
opt_ext_properties, opt_enable_feature_properties, properties;
nonterminal ColumnDef column_definition;
nonterminal IndexDef index_definition;
-nonterminal IndexDef build_index_definition;
nonterminal ArrayList<ColumnDef> column_definition_list;
nonterminal ArrayList<IndexDef> index_definition_list;
nonterminal AggregateType opt_agg_type;
@@ -2070,7 +2069,7 @@ create_stmt ::=
:}
| KW_BUILD KW_INDEX ident:indexName KW_ON table_name:tableName
opt_partition_names:partitionNames
{:
- RESULT = new AlterTableStmt(tableName, Lists.newArrayList(new
BuildIndexClause(tableName, new IndexDef(indexName, partitionNames, true),
false)));
+ RESULT = new AlterTableStmt(tableName, Lists.newArrayList(new
BuildIndexClause(tableName, indexName, partitionNames, false)));
:}
/* stage */
| KW_CREATE KW_STAGE opt_if_not_exists:ifNotExists ident:stageName
KW_PROPERTIES opt_key_value_map:properties
@@ -4043,13 +4042,6 @@ index_definition ::=
:}
;
-build_index_definition ::=
- KW_INDEX ident:indexName opt_partition_names:partitionNames
- {:
- RESULT = new IndexDef(indexName, partitionNames, true);
- :}
- ;
-
opt_nullable_type ::=
{:
RESULT = ColumnNullableType.DEFAULT;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index 79840ce92ab..5e742d11caa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -1259,7 +1259,12 @@ public class SchemaChangeHandler extends AlterHandler {
}
private void createJob(String rawSql, long dbId, OlapTable olapTable,
Map<Long, LinkedList<Column>> indexSchemaMap,
- Map<String, String> propertyMap, List<Index>
indexes) throws UserException {
+ Map<String, String> propertyMap, List<Index>
indexes,
+ boolean isBuildIndex) throws UserException {
+ if (isBuildIndex) {
+ // remove the index which is not the base index, only base index
can be built index
+ indexSchemaMap.entrySet().removeIf(entry ->
!entry.getKey().equals(olapTable.getBaseIndexId()));
+ }
checkReplicaCount(olapTable);
// process properties first
@@ -1295,7 +1300,7 @@ public class SchemaChangeHandler extends AlterHandler {
boolean hasIndexChange = false;
Set<Index> newSet = new HashSet<>(indexes);
Set<Index> oriSet = new HashSet<>(olapTable.getIndexes());
- if (!newSet.equals(oriSet)) {
+ if (!newSet.equals(oriSet) || isBuildIndex) {
hasIndexChange = true;
}
@@ -1311,7 +1316,7 @@ public class SchemaChangeHandler extends AlterHandler {
throw new DdlException(e.getMessage());
}
- // check bloom filter has change
+ // check bloom filter has been changed
boolean hasBfChange = false;
Set<String> oriBfColumns = olapTable.getCopiedBfColumns();
double oriBfFpp = olapTable.getBfFpp();
@@ -1947,7 +1952,7 @@ public class SchemaChangeHandler extends AlterHandler {
List<Index> newIndexes = olapTable.getCopiedIndexes();
List<Index> alterIndexes = new ArrayList<>();
- Map<Long, Set<String>> invertedIndexOnPartitions = new HashMap<>();
+ Map<Long, Set<String>> indexOnPartitions = new HashMap<>();
boolean isDropIndex = false;
Map<String, String> propertyMap = new HashMap<>();
for (AlterClause alterClause : alterClauses) {
@@ -2088,68 +2093,29 @@ public class SchemaChangeHandler extends AlterHandler {
}
lightSchemaChange = false;
- if (index.isLightIndexChangeSupported() &&
!Config.isCloudMode()) {
+ // ngram_bf index can do light_schema_change in both local
and cloud mode
+ // inverted index can only do light_schema_change in local
mode
+ if (index.isLightIndexChangeSupported()) {
alterIndexes.add(index);
isDropIndex = false;
- // now only support light index change for inverted
index
lightIndexChange = true;
}
} else if (alterClause instanceof BuildIndexClause) {
BuildIndexClause buildIndexClause = (BuildIndexClause)
alterClause;
IndexDef indexDef = buildIndexClause.getIndexDef();
Index index = buildIndexClause.getIndex();
- if (Config.isCloudMode()) {
+ if (Config.isCloudMode() && index.getIndexType() ==
IndexDef.IndexType.INVERTED) {
throw new DdlException("BUILD INDEX operation failed:
No need to do it in cloud mode.");
}
- if (!olapTable.isPartitionedTable()) {
- List<String> specifiedPartitions =
indexDef.getPartitionNames();
- if (!specifiedPartitions.isEmpty()) {
- throw new DdlException("table " +
olapTable.getName()
- + " is not partitioned, cannot build index
with partitions.");
- }
- }
- List<Index> existedIndexes = olapTable.getIndexes();
- boolean found = false;
- for (Index existedIdx : existedIndexes) {
- if
(existedIdx.getIndexName().equalsIgnoreCase(indexDef.getIndexName())) {
- found = true;
- if (!existedIdx.isLightIndexChangeSupported()) {
- throw new DdlException("BUILD INDEX operation
failed: The index "
- + existedIdx.getIndexName() + " of
type " + existedIdx.getIndexType()
- + " does not support lightweight index
changes.");
- }
- for (Column column : olapTable.getBaseSchema()) {
- if (!column.getType().isVariantType()) {
- continue;
- }
- // variant type column can not support for
building index
- for (String indexColumn :
existedIdx.getColumns()) {
- if
(column.getName().equalsIgnoreCase(indexColumn)) {
- throw new DdlException("BUILD INDEX
operation failed: The "
- + indexDef.getIndexName() + "
index can not be built on the "
- + indexColumn + " column,
because it is a variant type column.");
- }
- }
- }
- index = existedIdx.clone();
- if (indexDef.getPartitionNames().isEmpty()) {
-
invertedIndexOnPartitions.put(index.getIndexId(),
olapTable.getPartitionNames());
- } else {
- invertedIndexOnPartitions.put(
- index.getIndexId(), new
HashSet<>(indexDef.getPartitionNames()));
- }
- break;
- }
- }
- if (!found) {
- throw new DdlException("index " +
indexDef.getIndexName()
- + " not exist, cannot build it with
defferred.");
+ if (indexDef.getPartitionNames().isEmpty()) {
+ indexOnPartitions.put(index.getIndexId(),
olapTable.getPartitionNames());
+ } else {
+ indexOnPartitions.put(
+ index.getIndexId(), new
HashSet<>(indexDef.getPartitionNames()));
}
- if (indexDef.isInvertedIndex()) {
- alterIndexes.add(index);
- }
+ alterIndexes.add(index);
buildIndexChange = true;
lightSchemaChange = false;
} else if (alterClause instanceof DropIndexClause) {
@@ -2167,7 +2133,9 @@ public class SchemaChangeHandler extends AlterHandler {
break;
}
}
- if (found.isLightIndexChangeSupported() &&
!Config.isCloudMode()) {
+ // only inverted index with local mode can do light drop
index change
+ if (found != null && found.getIndexType() ==
IndexDef.IndexType.INVERTED
+ && Config.isNotCloudMode()) {
alterIndexes.add(found);
isDropIndex = true;
lightIndexChange = true;
@@ -2188,19 +2156,27 @@ public class SchemaChangeHandler extends AlterHandler {
long jobId = Env.getCurrentEnv().getNextId();
//for schema change add/drop value column optimize, direct
modify table meta.
modifyTableLightSchemaChange(rawSql, db, olapTable,
indexSchemaMap, newIndexes,
- null, isDropIndex, jobId, false);
+ null, isDropIndex, jobId, false,
propertyMap);
} else if (Config.enable_light_index_change && lightIndexChange) {
long jobId = Env.getCurrentEnv().getNextId();
- //for schema change add/drop inverted index optimize, direct
modify table meta firstly.
+ //for schema change add/drop inverted index and ngram_bf
optimize, direct modify table meta firstly.
modifyTableLightSchemaChange(rawSql, db, olapTable,
indexSchemaMap, newIndexes,
- alterIndexes, isDropIndex, jobId,
false);
+ alterIndexes, isDropIndex, jobId,
false, propertyMap);
} else if (buildIndexChange) {
+ if (alterIndexes.isEmpty()) {
+ throw new DdlException("Altered index is empty. please
check your alter stmt.");
+ }
+ IndexDef.IndexType indexType =
alterIndexes.get(0).getIndexType();
if (Config.enable_light_index_change) {
- buildOrDeleteTableInvertedIndices(db, olapTable,
indexSchemaMap,
- alterIndexes,
invertedIndexOnPartitions, false);
+ if (indexType == IndexDef.IndexType.INVERTED) {
+ buildOrDeleteTableInvertedIndices(db, olapTable,
indexSchemaMap,
+ alterIndexes, indexOnPartitions, false);
+ } else {
+ createJob(rawSql, db.getId(), olapTable,
indexSchemaMap, propertyMap, newIndexes, true);
+ }
}
} else {
- createJob(rawSql, db.getId(), olapTable, indexSchemaMap,
propertyMap, newIndexes);
+ createJob(rawSql, db.getId(), olapTable, indexSchemaMap,
propertyMap, newIndexes, false);
}
} finally {
olapTable.writeUnlock();
@@ -2699,6 +2675,8 @@ public class SchemaChangeHandler extends AlterHandler {
olapTable.writeUnlock();
}
+ // if this table has ngram_bf index, we must run cancel for schema
change job
+ boolean hasNGramBFIndex = ((OlapTable)
olapTable).hasIndexOfType(IndexDef.IndexType.NGRAM_BF);
// alter job v2's cancel must be called outside the table lock
if (jobList.size() > 0) {
for (IndexChangeJob job : jobList) {
@@ -2713,6 +2691,8 @@ public class SchemaChangeHandler extends AlterHandler {
LOG.info("cancel build index job {} on table {} success",
jobId, tableName);
}
}
+ } else if (hasNGramBFIndex) {
+ cancelColumnJob(cancelAlterTableStmt);
} else {
throw new DdlException("No job to cancel for Table[" + tableName +
"]");
}
@@ -2752,7 +2732,7 @@ public class SchemaChangeHandler extends AlterHandler {
Column column = olapTable.getColumn(col);
if (column != null) {
indexDef.checkColumn(column, olapTable.getKeysType(),
-
olapTable.getTableProperty().getEnableUniqueKeyMergeOnWrite(),
+ olapTable.getEnableUniqueKeyMergeOnWrite(),
olapTable.getInvertedIndexFileStorageFormat());
} else {
throw new DdlException("index column does not exist in table.
invalid column: " + col);
@@ -2887,7 +2867,7 @@ public class SchemaChangeHandler extends AlterHandler {
public void modifyTableLightSchemaChange(String rawSql, Database db,
OlapTable olapTable,
Map<Long, LinkedList<Column>>
indexSchemaMap, List<Index> indexes,
List<Index> alterIndexes, boolean
isDropIndex,
- long jobId, boolean isReplay)
+ long jobId, boolean isReplay,
Map<String, String> propertyMap)
throws DdlException, AnalysisException {
if (LOG.isDebugEnabled()) {
@@ -2966,7 +2946,7 @@ public class SchemaChangeHandler extends AlterHandler {
}
try {
buildOrDeleteTableInvertedIndices(db, olapTable,
indexSchemaMap,
- alterIndexes,
invertedIndexOnPartitions, true);
+ alterIndexes, invertedIndexOnPartitions, true);
} catch (Exception e) {
throw new DdlException(e.getMessage());
}
@@ -3027,7 +3007,8 @@ public class SchemaChangeHandler extends AlterHandler {
OlapTable olapTable = (OlapTable) db.getTableOrMetaException(tableId,
TableType.OLAP);
olapTable.writeLock();
try {
- modifyTableLightSchemaChange("", db, olapTable, indexSchemaMap,
indexes, null, false, jobId, true);
+ modifyTableLightSchemaChange("", db, olapTable, indexSchemaMap,
indexes, null, false, jobId,
+ true, new HashMap<>());
} catch (DdlException e) {
// should not happen
LOG.warn("failed to replay modify table add or drop or modify
columns", e);
@@ -3167,7 +3148,7 @@ public class SchemaChangeHandler extends AlterHandler {
olapTable.writeLock();
try {
modifyTableLightSchemaChange("", db, olapTable, indexSchemaMap,
newIndexes,
- alterIndexes, isDropIndex, jobId,
true);
+ alterIndexes, isDropIndex, jobId,
true, new HashMap<>());
} catch (UserException e) {
// should not happen
LOG.warn("failed to replay modify table add or drop indexes", e);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
index 693629bfef4..531a5e13cf1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
@@ -145,7 +145,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
protected boolean hasRowStoreChange = false;
// save all schema change tasks
- private AgentBatchTask schemaChangeBatchTask = new AgentBatchTask();
+ AgentBatchTask schemaChangeBatchTask = new AgentBatchTask();
protected SchemaChangeJobV2() {
super(JobType.SCHEMA_CHANGE);
@@ -651,14 +651,13 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
healthyReplicaNum++;
}
}
- if (!FeConstants.runningUnitTest) {
- if (healthyReplicaNum < expectReplicationNum / 2 +
1) {
- LOG.warn("shadow tablet {} has few healthy
replicas: {}, schema change job: {}"
- + " healthyReplicaNum {}
expectReplicationNum {}",
- shadowTablet.getId(), replicas, jobId,
healthyReplicaNum, expectReplicationNum);
- throw new AlterCancelException(
+
+ if ((healthyReplicaNum < expectReplicationNum / 2 + 1)
&& !FeConstants.runningUnitTest) {
+ LOG.warn("shadow tablet {} has few healthy
replicas: {}, schema change job: {}"
+ + " healthyReplicaNum {}
expectReplicationNum {}",
+ shadowTablet.getId(), replicas, jobId,
healthyReplicaNum, expectReplicationNum);
+ throw new AlterCancelException(
"shadow tablet " + shadowTablet.getId() +
" has few healthy replicas");
- }
}
} // end for tablets
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java
index 6da8b86cdad..ba72155f7ec 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java
@@ -18,12 +18,17 @@
package org.apache.doris.analysis;
import org.apache.doris.alter.AlterOpType;
+import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Index;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.Table;
+import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
import com.google.common.collect.Maps;
+import java.util.List;
import java.util.Map;
public class BuildIndexClause extends AlterTableClause {
@@ -36,11 +41,14 @@ public class BuildIndexClause extends AlterTableClause {
private boolean alter;
// index internal class
private Index index;
+ private String indexName;
+ private PartitionNames partitionNames;
- public BuildIndexClause(TableName tableName, IndexDef indexDef, boolean
alter) {
+ public BuildIndexClause(TableName tableName, String indexName,
PartitionNames partitionNames, boolean alter) {
super(AlterOpType.SCHEMA_CHANGE);
this.tableName = tableName;
- this.indexDef = indexDef;
+ this.indexName = indexName;
+ this.partitionNames = partitionNames;
this.alter = alter;
}
@@ -76,17 +84,52 @@ public class BuildIndexClause extends AlterTableClause {
@Override
public void analyze(Analyzer analyzer) throws AnalysisException {
- if (indexDef == null) {
- throw new AnalysisException("index definition expected.");
+ tableName.analyze(analyzer);
+ DatabaseIf<Table> db =
Env.getCurrentEnv().getCatalogMgr().getInternalCatalog()
+ .getDb(tableName.getDb()).orElse(null);
+ if (db == null) {
+ throw new AnalysisException("Database[" + tableName.getDb() + "]
is not exist");
+ }
+
+ TableIf table = db.getTable(tableName.getTbl()).orElse(null);
+ if (table == null) {
+ throw new AnalysisException("Table[" + tableName.getTbl() + "] is
not exist");
+ }
+ if (!(table instanceof OlapTable)) {
+ throw new AnalysisException("Only olap table support build index");
+ }
+
+ Index existedIdx = null;
+ for (Index index : table.getTableIndexes().getIndexes()) {
+ if (index.getIndexName().equalsIgnoreCase(indexName)) {
+ existedIdx = index;
+ if (!existedIdx.isLightIndexChangeSupported()) {
+ throw new AnalysisException("BUILD INDEX operation failed:
The index "
+ + existedIdx.getIndexName() + " of type " +
existedIdx.getIndexType()
+ + " does not support lightweight index changes.");
+ }
+ break;
+ }
+ }
+ if (existedIdx == null) {
+ throw new AnalysisException("Index[" + indexName + "] is not exist
in table[" + tableName.getTbl() + "]");
}
- if (indexDef.getIndexType() == IndexDef.IndexType.NGRAM_BF
- || indexDef.getIndexType() == IndexDef.IndexType.BLOOMFILTER) {
- throw new AnalysisException("ngram bloomfilter or bloomfilter
index is not needed to build.");
+
+ IndexDef.IndexType indexType = existedIdx.getIndexType();
+ if (!existedIdx.isLightIndexChangeSupported()) {
+ throw new AnalysisException(indexType.toString() + " index is not
needed to build.");
+ }
+
+ indexDef = new IndexDef(indexName, partitionNames, indexType, true);
+ if (!table.isPartitionedTable()) {
+ List<String> specifiedPartitions = indexDef.getPartitionNames();
+ if (!specifiedPartitions.isEmpty()) {
+ throw new AnalysisException("table " + table.getName()
+ + " is not partitioned, cannot build index with
partitions.");
+ }
}
indexDef.analyze();
- this.index = new Index(Env.getCurrentEnv().getNextId(),
indexDef.getIndexName(),
- indexDef.getColumns(), indexDef.getIndexType(),
- indexDef.getProperties(), indexDef.getComment());
+ this.index = existedIdx.clone();
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
index 143c9f09d2a..06334873bd9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
@@ -83,9 +83,9 @@ public class IndexDef {
}
}
- public IndexDef(String indexName, PartitionNames partitionNames, boolean
isBuildDeferred) {
+ public IndexDef(String indexName, PartitionNames partitionNames, IndexType
indexType, boolean isBuildDeferred) {
this.indexName = indexName;
- this.indexType = IndexType.INVERTED;
+ this.indexType = indexType;
this.partitionNames = partitionNames;
this.isBuildDeferred = isBuildDeferred;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java
index 1718c26bdb4..af4ff1501d6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java
@@ -20,6 +20,7 @@ package org.apache.doris.catalog;
import org.apache.doris.analysis.IndexDef;
import org.apache.doris.analysis.InvertedIndexUtil;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.PrintableMap;
@@ -167,8 +168,17 @@ public class Index implements Writable {
return InvertedIndexUtil.getInvertedIndexParserStopwords(properties);
}
+ // Whether the index can be changed in light mode
+ // cloud mode only supports light change for ngram_bf index
+ // local mode supports light change for both inverted index and ngram_bf
index
+ // the rest of the index types do not support light change
public boolean isLightIndexChangeSupported() {
- return indexType == IndexDef.IndexType.INVERTED;
+ if (Config.isCloudMode()) {
+ return indexType == IndexDef.IndexType.NGRAM_BF;
+ } else {
+ return indexType == IndexDef.IndexType.INVERTED
+ || indexType == IndexDef.IndexType.NGRAM_BF;
+ }
}
public String getComment() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index ca97c09f43d..a7c5a066971 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -24,6 +24,7 @@ import org.apache.doris.analysis.ColumnDef;
import org.apache.doris.analysis.CreateMaterializedViewStmt;
import org.apache.doris.analysis.DataSortInfo;
import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.IndexDef;
import org.apache.doris.analysis.SlotDescriptor;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.backup.Status;
@@ -366,6 +367,19 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
return indexes.getIndexIds();
}
+ /**
+ * Checks if the table contains at least one index of the specified type.
+ * @param indexType The index type to check for
+ * @return true if the table has at least one index of the specified type,
false otherwise
+ */
+ public boolean hasIndexOfType(IndexDef.IndexType indexType) {
+ if (indexes == null) {
+ return false;
+ }
+ return indexes.getIndexes().stream()
+ .anyMatch(index -> index.getIndexType() == indexType);
+ }
+
@Override
public TableIndexes getTableIndexes() {
return indexes;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 0b0e9659f69..4f1e7d37e24 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -5021,8 +5021,8 @@ public class LogicalPlanBuilder extends
DorisParserBaseVisitor<Object> {
Pair<Boolean, List<String>> partitionSpec =
visitPartitionSpec(ctx.partitionSpec());
partitionNamesInfo = new PartitionNamesInfo(partitionSpec.first,
partitionSpec.second);
}
- IndexDefinition indexDefinition = new IndexDefinition(name,
partitionNamesInfo);
- List<AlterTableOp> alterTableOps = Lists.newArrayList(new
BuildIndexOp(tableName, indexDefinition, false));
+ List<AlterTableOp> alterTableOps = Lists.newArrayList(new
BuildIndexOp(tableName, name, partitionNamesInfo,
+ false));
return new AlterTableCommand(tableName, alterTableOps);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BuildIndexOp.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BuildIndexOp.java
index 11b9df12ece..36439874760 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BuildIndexOp.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BuildIndexOp.java
@@ -20,13 +20,20 @@ package org.apache.doris.nereids.trees.plans.commands.info;
import org.apache.doris.alter.AlterOpType;
import org.apache.doris.analysis.AlterTableClause;
import org.apache.doris.analysis.BuildIndexClause;
+import org.apache.doris.analysis.IndexDef;
+import org.apache.doris.catalog.DatabaseIf;
+import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Index;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.Table;
+import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.UserException;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.Maps;
+import java.util.List;
import java.util.Map;
/**
@@ -36,17 +43,23 @@ public class BuildIndexOp extends AlterTableOp {
// in which table the index on, only used when alter = false
private final TableNameInfo tableName;
// index definition class
- private final IndexDefinition indexDef;
+ private IndexDefinition indexDef;
// when alter = true, clause like: alter table add index xxxx
// when alter = false, clause like: create index xx on table xxxx
private final boolean alter;
// index internal class
private Index index;
+ // index name
+ private final String indexName;
+ // partition names info
+ private final PartitionNamesInfo partitionNamesInfo;
- public BuildIndexOp(TableNameInfo tableName, IndexDefinition indexDef,
boolean alter) {
+ public BuildIndexOp(TableNameInfo tableName, String indexName,
PartitionNamesInfo partitionNamesInfo,
+ boolean alter) {
super(AlterOpType.SCHEMA_CHANGE);
this.tableName = tableName;
- this.indexDef = indexDef;
+ this.indexName = indexName;
+ this.partitionNamesInfo = partitionNamesInfo;
this.alter = alter;
}
@@ -69,16 +82,57 @@ public class BuildIndexOp extends AlterTableOp {
@Override
public void validate(ConnectContext ctx) throws UserException {
- if (indexDef == null) {
- throw new AnalysisException("index definition expected.");
+ tableName.analyze(ctx);
+ DatabaseIf<Table> db =
Env.getCurrentEnv().getCatalogMgr().getInternalCatalog()
+ .getDb(tableName.getDb()).orElse(null);
+ if (db == null) {
+ throw new AnalysisException("Database[" + tableName.getDb() + "]
is not exist");
+ }
+
+ TableIf table = db.getTable(tableName.getTbl()).orElse(null);
+ if (table == null) {
+ throw new AnalysisException("Table[" + tableName.getTbl() + "] is
not exist");
+ }
+ if (!(table instanceof OlapTable)) {
+ throw new AnalysisException("Only olap table support build index");
+ }
+
+ Index existedIdx = null;
+ for (Index index : table.getTableIndexes().getIndexes()) {
+ if (index.getIndexName().equalsIgnoreCase(indexName)) {
+ existedIdx = index;
+ if (!existedIdx.isLightIndexChangeSupported()) {
+ throw new AnalysisException("BUILD INDEX operation failed:
The index "
+ + existedIdx.getIndexName() + " of type " +
existedIdx.getIndexType()
+ + " does not support lightweight index changes.");
+ }
+ break;
+ }
+ }
+ if (existedIdx == null) {
+ throw new AnalysisException("Index[" + indexName + "] is not exist
in table[" + tableName.getTbl() + "]");
+ }
+
+ IndexDef.IndexType indexType = existedIdx.getIndexType();
+ if (!existedIdx.isLightIndexChangeSupported()) {
+ throw new AnalysisException(indexType.toString() + " index is not
needed to build.");
+ }
+
+ indexDef = new IndexDefinition(indexName, partitionNamesInfo,
indexType);
+ if (!table.isPartitionedTable()) {
+ List<String> specifiedPartitions = indexDef.getPartitionNames();
+ if (!specifiedPartitions.isEmpty()) {
+ throw new AnalysisException("table " + table.getName()
+ + " is not partitioned, cannot build index with
partitions.");
+ }
}
indexDef.validate();
- tableName.analyze(ctx);
- index = indexDef.translateToCatalogStyle();
+ this.index = existedIdx.clone();
}
@Override
public AlterTableClause translateToLegacyAlterClause() {
+ indexDef.getIndexType();
return new BuildIndexClause(tableName.transferToTableName(),
indexDef.translateToLegacyIndexDef(), index,
alter);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java
index d051cc45519..0ec4c4cd58e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/IndexDefinition.java
@@ -100,9 +100,9 @@ public class IndexDefinition {
/**
* constructor for build index
*/
- public IndexDefinition(String name, PartitionNamesInfo partitionNames) {
+ public IndexDefinition(String name, PartitionNamesInfo partitionNames,
IndexType indexType) {
this.name = name;
- this.indexType = IndexType.INVERTED;
+ this.indexType = indexType;
this.partitionNames = partitionNames;
this.isBuildDeferred = true;
this.cols = null;
@@ -260,13 +260,17 @@ public class IndexDefinition {
comment);
}
+ public List<String> getPartitionNames() {
+ return partitionNames == null ? Lists.newArrayList() :
partitionNames.getPartitionNames();
+ }
+
/**
* translateToLegacyIndexDef
*/
public IndexDef translateToLegacyIndexDef() {
if (isBuildDeferred) {
return new IndexDef(name, partitionNames != null ?
partitionNames.translateToLegacyPartitionNames() : null,
- true);
+ indexType, true);
} else {
return new IndexDef(name, ifNotExists, cols, indexType,
properties, comment);
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java
b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java
index 6b0943c306f..20bd26fa92d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java
@@ -55,6 +55,8 @@ import org.apache.doris.transaction.GlobalTransactionMgr;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import mockit.Mock;
+import mockit.MockUp;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
@@ -85,6 +87,8 @@ public class IndexChangeJobTest {
private static BuildIndexClause buildIndexClause;
private static DropIndexClause dropIndexClause;
private static CancelAlterTableStmt cancelAlterTableStmt;
+ private static TableName tableName;
+ private static String indexName;
@Rule
public ExpectedException expectedEx = ExpectedException.none();
@@ -108,25 +112,31 @@ public class IndexChangeJobTest {
db =
masterEnv.getInternalCatalog().getDbOrDdlException(CatalogTestUtil.testDbId1);
olapTable = (OlapTable)
db.getTableOrDdlException(CatalogTestUtil.testTableId1);
+ new MockUp<Env>() {
+ @Mock
+ public Env getCurrentEnv() {
+ return masterEnv;
+ }
+ };
+
// set mow table property
Map<String, String> properties = Maps.newHashMap();
properties.put(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE,
"false");
TableProperty tableProperty = new TableProperty(properties);
olapTable.setTableProperty(tableProperty);
- TableName tableName = new
TableName(masterEnv.getInternalCatalog().getName(), db.getName(),
+ tableName = new TableName(masterEnv.getInternalCatalog().getName(),
db.getName(),
olapTable.getName());
- IndexDef indexDef = new IndexDef("index1", false,
+ indexName = "index1";
+ IndexDef indexDef = new IndexDef(indexName, false,
Lists.newArrayList(olapTable.getBaseSchema().get(1).getName()),
IndexDef.IndexType.INVERTED,
Maps.newHashMap(), "balabala");
+ FakeEnv.setEnv(masterEnv);
createIndexClause = new CreateIndexClause(tableName, indexDef, false);
createIndexClause.analyze(analyzer);
- buildIndexClause = new BuildIndexClause(tableName, indexDef, false);
- buildIndexClause.analyze(analyzer);
-
- dropIndexClause = new DropIndexClause("index1", false, tableName,
false);
+ dropIndexClause = new DropIndexClause(indexName, false, tableName,
false);
dropIndexClause.analyze(analyzer);
cancelAlterTableStmt = new
CancelAlterTableStmt(ShowAlterStmt.AlterType.INDEX, tableName);
@@ -163,6 +173,8 @@ public class IndexChangeJobTest {
Assert.assertEquals(olapTable.getIndexes().size(), 1);
Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(),
"index1");
alterClauses.clear();
+ buildIndexClause = new BuildIndexClause(tableName, indexName, null,
false);
+ buildIndexClause.analyze(analyzer);
alterClauses.add(buildIndexClause);
schemaChangeHandler.process(alterClauses, db, olapTable);
Map<Long, IndexChangeJob> indexChangeJobMap =
schemaChangeHandler.getIndexChangeJobs();
@@ -203,6 +215,8 @@ public class IndexChangeJobTest {
Assert.assertEquals(olapTable.getIndexes().size(), 1);
Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(),
"index1");
alterClauses.clear();
+ buildIndexClause = new BuildIndexClause(tableName, indexName, null,
false);
+ buildIndexClause.analyze(analyzer);
alterClauses.add(buildIndexClause);
schemaChangeHandler.process(alterClauses, db, olapTable);
Map<Long, IndexChangeJob> indexChangeJobMap =
schemaChangeHandler.getIndexChangeJobs();
@@ -292,6 +306,8 @@ public class IndexChangeJobTest {
Assert.assertEquals(olapTable.getIndexes().size(), 1);
Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(),
"index1");
alterClauses.clear();
+ buildIndexClause = new BuildIndexClause(tableName, indexName, null,
false);
+ buildIndexClause.analyze(analyzer);
alterClauses.add(buildIndexClause);
schemaChangeHandler.process(alterClauses, db, olapTable);
Map<Long, IndexChangeJob> indexChangeJobMap =
schemaChangeHandler.getIndexChangeJobs();
@@ -339,6 +355,8 @@ public class IndexChangeJobTest {
Assert.assertEquals(olapTable.getIndexes().size(), 1);
Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(),
"index1");
alterClauses.clear();
+ buildIndexClause = new BuildIndexClause(tableName, indexName, null,
false);
+ buildIndexClause.analyze(analyzer);
alterClauses.add(buildIndexClause);
schemaChangeHandler.process(alterClauses, db, olapTable);
Map<Long, IndexChangeJob> indexChangeJobMap =
schemaChangeHandler.getIndexChangeJobs();
@@ -467,6 +485,8 @@ public class IndexChangeJobTest {
Assert.assertEquals(olapTable.getIndexes().size(), 1);
Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(),
"index1");
alterClauses.clear();
+ buildIndexClause = new BuildIndexClause(tableName, indexName, null,
false);
+ buildIndexClause.analyze(analyzer);
alterClauses.add(buildIndexClause);
schemaChangeHandler.process(alterClauses, db, olapTable);
Map<Long, IndexChangeJob> indexChangeJobMap =
schemaChangeHandler.getIndexChangeJobs();
@@ -519,6 +539,8 @@ public class IndexChangeJobTest {
Assert.assertEquals(olapTable.getIndexes().size(), 1);
Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(),
"index1");
alterClauses.clear();
+ buildIndexClause = new BuildIndexClause(tableName, indexName, null,
false);
+ buildIndexClause.analyze(analyzer);
alterClauses.add(buildIndexClause);
schemaChangeHandler.process(alterClauses, db, olapTable);
Map<Long, IndexChangeJob> indexChangeJobMap =
schemaChangeHandler.getIndexChangeJobs();
@@ -526,13 +548,13 @@ public class IndexChangeJobTest {
Assert.assertEquals(OlapTableState.NORMAL, olapTable.getState());
IndexChangeJob indexChangejob =
indexChangeJobMap.values().stream().findAny().get();
-
Assert.assertEquals(indexChangejob.invertedIndexBatchTask.getTaskNum(), 0);
+ Assert.assertEquals(0,
indexChangejob.invertedIndexBatchTask.getTaskNum());
Assert.assertEquals(IndexChangeJob.JobState.WAITING_TXN,
indexChangejob.getJobState());
// run waiting txn job
schemaChangeHandler.runAfterCatalogReady();
Assert.assertEquals(IndexChangeJob.JobState.RUNNING,
indexChangejob.getJobState());
-
Assert.assertEquals(indexChangejob.invertedIndexBatchTask.getTaskNum(), 3);
+ Assert.assertEquals(3,
indexChangejob.invertedIndexBatchTask.getTaskNum());
// run running job
schemaChangeHandler.runAfterCatalogReady();
Assert.assertEquals(IndexChangeJob.JobState.RUNNING,
indexChangejob.getJobState());
@@ -565,17 +587,123 @@ public class IndexChangeJobTest {
fakeEditLog = new FakeEditLog();
FakeEnv.setEnv(masterEnv);
- IndexDef indexDef = new IndexDef("ngram_bf_index", false,
- Lists.newArrayList(olapTable.getBaseSchema().get(1).getName()),
+ OlapTable table = (OlapTable)
db.getTableOrDdlException(CatalogTestUtil.testTableId2);
+ String indexName = "ngram_bf_index";
+ IndexDef indexDef = new IndexDef(indexName, false,
+ Lists.newArrayList(table.getBaseSchema().get(3).getName()),
org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF,
Maps.newHashMap(), "ngram bf index");
TableName tableName = new
TableName(masterEnv.getInternalCatalog().getName(), db.getName(),
- olapTable.getName());
+ table.getName());
createIndexClause = new CreateIndexClause(tableName, indexDef, false);
createIndexClause.analyze(analyzer);
+ SchemaChangeHandler schemaChangeHandler =
Env.getCurrentEnv().getSchemaChangeHandler();
+ ArrayList<AlterClause> alterClauses = new ArrayList<>();
+ alterClauses.add(createIndexClause);
+ schemaChangeHandler.process(alterClauses, db, table);
+ Map<Long, AlterJobV2> indexChangeJobMap =
schemaChangeHandler.getAlterJobsV2();
+ Assert.assertEquals(1, indexChangeJobMap.size());
+ Assert.assertEquals(1, table.getIndexes().size());
+ Assert.assertEquals("ngram_bf_index",
table.getIndexes().get(0).getIndexName());
+
+ long jobId = indexChangeJobMap.values().stream().findAny().get().jobId;
+
+ buildIndexClause = new BuildIndexClause(tableName, indexName, null,
false);
+ buildIndexClause.analyze(analyzer);
+ alterClauses.clear();
+ alterClauses.add(buildIndexClause);
+
+ schemaChangeHandler.process(alterClauses, db, table);
+ Assert.assertEquals(2, indexChangeJobMap.size());
+ Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState());
+
+ SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2)
indexChangeJobMap.values().stream()
+ .filter(job -> job.jobId != jobId)
+ .findFirst()
+ .orElse(null);
+ Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum());
+
+ schemaChangeHandler.runAfterCatalogReady();
+ Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN,
jobV2.getJobState());
+ Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum());
+
+ schemaChangeHandler.runAfterCatalogReady();
+ Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState());
+ Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum());
+
+ schemaChangeHandler.runAfterCatalogReady();
+ Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState());
+ Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum());
+
+ List<AgentTask> tasks = AgentTaskQueue.getTask(TTaskType.ALTER);
+ Assert.assertEquals(1, tasks.size());
+ for (AgentTask agentTask : tasks) {
+ agentTask.setFinished(true);
+ }
- buildIndexClause = new BuildIndexClause(tableName, indexDef, false);
-
org.junit.jupiter.api.Assertions.assertThrows(org.apache.doris.common.AnalysisException.class,
- () -> buildIndexClause.analyze(analyzer));
+ schemaChangeHandler.runAfterCatalogReady();
+ Assert.assertEquals(AlterJobV2.JobState.FINISHED, jobV2.getJobState());
+ }
+
+ @Test
+ public void testCancelNgramBfBuildIndex() throws UserException {
+ fakeEnv = new FakeEnv();
+ fakeEditLog = new FakeEditLog();
+ FakeEnv.setEnv(masterEnv);
+
+ OlapTable table = (OlapTable)
db.getTableOrDdlException(CatalogTestUtil.testTableId2);
+ String indexName = "ngram_bf_index";
+ IndexDef indexDef = new IndexDef(indexName, false,
+ Lists.newArrayList(table.getBaseSchema().get(3).getName()),
+ org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF,
+ Maps.newHashMap(), "ngram bf index");
+ TableName tableName = new
TableName(masterEnv.getInternalCatalog().getName(), db.getName(),
+ table.getName());
+ createIndexClause = new CreateIndexClause(tableName, indexDef, false);
+ createIndexClause.analyze(analyzer);
+ SchemaChangeHandler schemaChangeHandler =
Env.getCurrentEnv().getSchemaChangeHandler();
+ ArrayList<AlterClause> alterClauses = new ArrayList<>();
+ alterClauses.add(createIndexClause);
+ schemaChangeHandler.process(alterClauses, db, table);
+ Map<Long, AlterJobV2> indexChangeJobMap =
schemaChangeHandler.getAlterJobsV2();
+ Assert.assertEquals(1, indexChangeJobMap.size());
+ Assert.assertEquals(1, table.getIndexes().size());
+ Assert.assertEquals("ngram_bf_index",
table.getIndexes().get(0).getIndexName());
+
+ long jobId = indexChangeJobMap.values().stream().findAny().get().jobId;
+
+ buildIndexClause = new BuildIndexClause(tableName, indexName, null,
false);
+ buildIndexClause.analyze(analyzer);
+ alterClauses.clear();
+ alterClauses.add(buildIndexClause);
+
+ schemaChangeHandler.process(alterClauses, db, table);
+ Assert.assertEquals(2, indexChangeJobMap.size());
+ Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState());
+
+ SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2)
indexChangeJobMap.values().stream()
+ .filter(job -> job.jobId != jobId)
+ .findFirst()
+ .orElse(null);
+ Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum());
+
+ schemaChangeHandler.runAfterCatalogReady();
+ Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN,
jobV2.getJobState());
+ Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum());
+
+ schemaChangeHandler.runAfterCatalogReady();
+ Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState());
+ Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum());
+
+ schemaChangeHandler.runAfterCatalogReady();
+ Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState());
+ Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum());
+
+ cancelAlterTableStmt = new
CancelAlterTableStmt(ShowAlterStmt.AlterType.INDEX, tableName);
+ cancelAlterTableStmt.analyze(analyzer);
+ schemaChangeHandler.cancel(cancelAlterTableStmt);
+
+ schemaChangeHandler.runAfterCatalogReady();
+ Assert.assertEquals(AlterJobV2.JobState.CANCELLED,
jobV2.getJobState());
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java
index 73af753a69d..4b8f218acea 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java
@@ -850,8 +850,13 @@ public class SchemaChangeHandlerTest extends
TestWithFeService {
String buildNgramBfIndexStmtStr = "BUILD INDEX idx_error_msg on
test.sc_dup ";
AlterTableStmt buildNgramBfIndexStmt = (AlterTableStmt)
parseAndAnalyzeStmt(buildNgramBfIndexStmtStr);
-
org.junit.jupiter.api.Assertions.assertThrows(org.apache.doris.common.DdlException.class,
- () ->
Env.getCurrentEnv().getAlterInstance().processAlterTable(buildNgramBfIndexStmt));
+
Env.getCurrentEnv().getAlterInstance().processAlterTable(buildNgramBfIndexStmt);
+
+ jobSize++;
+ alterJobs =
Env.getCurrentEnv().getSchemaChangeHandler().getAlterJobsV2();
+ LOG.info("alterJobs:{}", alterJobs);
+ Assertions.assertEquals(jobSize, alterJobs.size());
+ waitAlterJobDone(alterJobs);
tbl.readLock();
try {
diff --git
a/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out
b/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out
new file mode 100644
index 00000000000..6f916a99c91
Binary files /dev/null and
b/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out differ
diff --git a/regression-test/pipeline/cloud_p0/conf/fe_custom.conf
b/regression-test/pipeline/cloud_p0/conf/fe_custom.conf
index 0b53fa2244d..6f067f66056 100644
--- a/regression-test/pipeline/cloud_p0/conf/fe_custom.conf
+++ b/regression-test/pipeline/cloud_p0/conf/fe_custom.conf
@@ -40,7 +40,6 @@ meta_service_endpoint=127.0.0.1:5000
cloud_unique_id=cloud_unique_id_sql_server00
# for case test_build_mtmv.groovy
enable_job_schedule_second_for_test=true
-enable_light_index_change=false
workload_sched_policy_interval_ms = 1000
workload_group_max_num = 25
diff --git a/regression-test/pipeline/cloud_p1/conf/fe_custom.conf
b/regression-test/pipeline/cloud_p1/conf/fe_custom.conf
index 31ef26e19d3..b91a4ed6d38 100644
--- a/regression-test/pipeline/cloud_p1/conf/fe_custom.conf
+++ b/regression-test/pipeline/cloud_p1/conf/fe_custom.conf
@@ -33,7 +33,6 @@ priority_networks=127.0.0.1/24
cloud_http_port=18030
meta_service_endpoint=127.0.0.1:5000
cloud_unique_id=cloud_unique_id_sql_server00
-enable_light_index_change=false
enable_advance_next_id = true
arrow_flight_sql_port = 8081
diff --git a/regression-test/pipeline/vault_p0/conf/fe_custom.conf
b/regression-test/pipeline/vault_p0/conf/fe_custom.conf
index d35632eb748..f62ffa19fc7 100644
--- a/regression-test/pipeline/vault_p0/conf/fe_custom.conf
+++ b/regression-test/pipeline/vault_p0/conf/fe_custom.conf
@@ -37,7 +37,6 @@ meta_service_endpoint=127.0.0.1:5000
cloud_unique_id=cloud_unique_id_sql_server00
# for case test_build_mtmv.groovy
enable_job_schedule_second_for_test=true
-enable_light_index_change=false
workload_sched_policy_interval_ms = 1000
enable_advance_next_id = true
diff --git
a/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy
b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy
new file mode 100644
index 00000000000..6fea7a68f92
--- /dev/null
+++ b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy
@@ -0,0 +1,286 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+import groovy.json.JsonSlurper
+
+suite("test_ngram_bloomfilter_index_change") {
+ def tableName = 'test_ngram_bloomfilter_index_change'
+ def timeout = 60000
+ def delta_time = 1000
+ def alter_res = "null"
+ def useTime = 0
+
+ def wait_for_latest_op_on_table_finish = { table_name, OpTimeout ->
+ for(int t = delta_time; t <= OpTimeout; t += delta_time){
+ alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName =
"${table_name}" ORDER BY CreateTime DESC LIMIT 1;"""
+ alter_res = alter_res.toString()
+ if(alter_res.contains("FINISHED")) {
+ sleep(3000) // wait change table state to normal
+ logger.info(table_name + " latest alter job finished, detail:
" + alter_res)
+ break
+ }
+ useTime = t
+ sleep(delta_time)
+ }
+ assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish
timeout")
+ }
+
+ // Function to insert test data batch
+ def insertTestData = { ->
+ // insert 10 records
+ sql "INSERT INTO ${tableName} VALUES (1001, '2023-10-06 15:00:00',
'Laptop', 'John Smith', 199.99, 'North');"
+ sql "INSERT INTO ${tableName} VALUES (1002, '2023-10-09 17:05:00',
'Smartphone', 'Emily Johnson', 299.99, 'South');"
+ sql "INSERT INTO ${tableName} VALUES (1003, '2023-10-12 19:10:00',
'Headphones', 'Michael Brown', 399.99, 'East');"
+ sql "INSERT INTO ${tableName} VALUES (1004, '2023-10-15 21:15:00',
'Monitor', 'Jessica Davis', 499.99, 'West');"
+ sql "INSERT INTO ${tableName} VALUES (1005, '2023-10-18 23:20:00',
'Keyboard', 'David Wilson', 89.99, 'North');"
+ sql "INSERT INTO ${tableName} VALUES (1006, '2023-10-21 07:25:00',
'Mouse', 'Sarah Taylor', 699.99, 'South');"
+ sql "INSERT INTO ${tableName} VALUES (1007, '2023-10-24 09:30:00',
'Printer', 'Thomas Anderson', 799.99, 'East');"
+ sql "INSERT INTO ${tableName} VALUES (1008, '2023-10-27 11:35:00',
'Speaker', 'Jennifer Martin', 899.99, 'West');"
+ sql "INSERT INTO ${tableName} VALUES (1009, '2023-10-02 13:40:00',
'External SSD', 'Robert Clark', 999.99, 'North');"
+ sql "INSERT INTO ${tableName} VALUES (1010, '2023-10-05 15:45:00',
'Webcam', 'Amanda Lewis', 89.99, 'South');"
+ sql "sync"
+ }
+
+ // Test setup
+ // 1. Create table
+ // 2. Insert test data
+ // 3. Add NGRAM Bloom Filter index
+ // 4. Build index
+ // 5. Insert more data
+ // 6. Drop index
+ sql "DROP TABLE IF EXISTS ${tableName}"
+ sql """
+ CREATE TABLE ${tableName} (
+ `sale_id` int NULL,
+ `sale_date` datetime NULL,
+ `product_name` varchar(100) NULL,
+ `customer_name` varchar(100) NULL,
+ `amount` decimal(10,2) NULL,
+ `region` char(50) NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`sale_id`)
+ PARTITION BY RANGE(`sale_date`) (
+ PARTITION p202310 VALUES [('2023-10-01 00:00:00'), ('2023-11-01
00:00:00')),
+ PARTITION p202311 VALUES [('2023-11-01 00:00:00'), ('2023-12-01
00:00:00')),
+ PARTITION p202312 VALUES [('2023-12-01 00:00:00'), ('2024-01-01 00:00:00'))
+ )
+ DISTRIBUTED BY HASH(`sale_id`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "storage_format" = "V2",
+ "light_schema_change" = "true",
+ "disable_auto_compaction" = "false"
+ );
+ """
+
+ // Insert first batch of data
+ insertTestData()
+
+ // Test settings
+ sql "set enable_function_pushdown=true"
+ sql "set enable_profile=true"
+ sql "set profile_level=2"
+
+ // Verify data loaded correctly
+ qt_select "SELECT * FROM ${tableName} ORDER BY sale_id"
+
+ // Define test query
+ def query = "SELECT /*+SET_VAR(enable_function_pushdown = true,
enable_profile = true, profile_level = 2)*/ * FROM ${tableName} WHERE
customer_name LIKE '%xxxx%' ORDER BY sale_id"
+
+ // Test 1: without NGRAM Bloom Filter index
+ profile("sql_select_like_without_ngram_index") {
+ run {
+ sql "/* sql_select_like_without_ngram_index */ ${query}"
+ sleep(1000) // sleep 1s wait for the profile collection to be
completed
+ }
+
+ check { profileString, exception ->
+ log.info(profileString)
+ assertTrue(profileString.contains("RowsBloomFilterFiltered: 0"))
+ }
+ }
+
+ // Test 2: After adding NGRAM Bloom Filter index
+ sql "ALTER TABLE ${tableName} ADD INDEX
idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' =
'1024', 'gram_size' = '3');"
+ wait_for_latest_op_on_table_finish(tableName, timeout)
+ profile("sql_select_like_with_ngram_index_added") {
+ run {
+ sql "/* sql_select_like_with_ngram_index_added */ ${query}"
+ sleep(1000)
+ }
+
+ check { profileString, exception ->
+ log.info(profileString)
+ assertTrue(profileString.contains("RowsBloomFilterFiltered: 0"))
+ }
+ }
+
+ // Test 3: After building the index
+ sql "BUILD INDEX idx_ngram_customer_name ON ${tableName};"
+ wait_for_latest_op_on_table_finish(tableName, timeout)
+ profile("sql_select_like_with_ngram_index_built") {
+ run {
+ sql "/* sql_select_like_with_ngram_index_built */ ${query}"
+ sleep(1000)
+ }
+
+ check { profileString, exception ->
+ log.info(profileString)
+ assertTrue(profileString.contains("RowsBloomFilterFiltered: 10"))
+ }
+ }
+
+ // Insert second batch of data
+ insertTestData()
+ // Verify data loaded correctly
+ qt_select "SELECT * FROM ${tableName} ORDER BY sale_id"
+
+ // Test 4: Verify filtering with more data
+ profile("sql_select_like_with_ngram_index_more_data") {
+ run {
+ sql "/* sql_select_like_with_ngram_index_more_data */ ${query}"
+ sleep(1000)
+ }
+
+ check { profileString, exception ->
+ log.info(profileString)
+ assertTrue(profileString.contains("RowsBloomFilterFiltered: 20"))
+ }
+ }
+
+ // Test 5: After dropping the index
+ sql "DROP INDEX idx_ngram_customer_name ON ${tableName};"
+ wait_for_latest_op_on_table_finish(tableName, timeout)
+ profile("sql_select_like_with_ngram_index_dropped") {
+ run {
+ sql "/* sql_select_like_with_ngram_index_dropped */ ${query}"
+ sleep(1000)
+ }
+
+ check { profileString, exception ->
+ log.info(profileString)
+ assertTrue(profileString.contains("RowsBloomFilterFiltered: 0"))
+ }
+ }
+
+ // recreate table
+ // 1. Create table
+ // 2. Add NGRAM Bloom Filter index
+ // 3. Insert data
+ // 4. Insert more data
+ // 5. Build index
+ // 6. Drop index
+ sql "DROP TABLE IF EXISTS ${tableName}"
+ sql """
+ CREATE TABLE ${tableName} (
+ `sale_id` int NULL,
+ `sale_date` datetime NULL,
+ `product_name` varchar(100) NULL,
+ `customer_name` varchar(100) NULL,
+ `amount` decimal(10,2) NULL,
+ `region` char(50) NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`sale_id`)
+ PARTITION BY RANGE(`sale_date`) (
+ PARTITION p202310 VALUES [('2023-10-01 00:00:00'), ('2023-11-01
00:00:00')),
+ PARTITION p202311 VALUES [('2023-11-01 00:00:00'), ('2023-12-01
00:00:00')),
+ PARTITION p202312 VALUES [('2023-12-01 00:00:00'), ('2024-01-01 00:00:00'))
+ )
+ DISTRIBUTED BY HASH(`sale_id`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "storage_format" = "V2",
+ "light_schema_change" = "true",
+ "disable_auto_compaction" = "false"
+ );
+ """
+
+ // add ngram bf index
+ sql "ALTER TABLE ${tableName} ADD INDEX
idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' =
'1024', 'gram_size' = '3');"
+ wait_for_latest_op_on_table_finish(tableName, timeout)
+
+ // insert data
+ insertTestData()
+
+ // Verify data loaded correctly
+ qt_select "SELECT * FROM ${tableName} ORDER BY sale_id"
+
+ // Test 6: Verify filtering with index added
+ profile("sql_select_like_with_ngram_index_recreated") {
+ run {
+ sql "/* sql_select_like_with_ngram_index_recreated */ ${query}"
+ sleep(1000)
+ }
+
+ check { profileString, exception ->
+ log.info(profileString)
+ assertTrue(profileString.contains("RowsBloomFilterFiltered: 10"))
+ }
+ }
+
+ // insert more data
+ insertTestData()
+
+ // Verify data loaded correctly
+ qt_select "SELECT * FROM ${tableName} ORDER BY sale_id"
+
+ // Test 7: Verify filtering with more data
+ profile("sql_select_like_with_ngram_index_recreated_more_data") {
+ run {
+ sql "/* sql_select_like_with_ngram_index_recreated_more_data */
${query}"
+ sleep(1000)
+ }
+
+ check { profileString, exception ->
+ log.info(profileString)
+ assertTrue(profileString.contains("RowsBloomFilterFiltered: 20"))
+ }
+ }
+
+ // build index
+ sql "BUILD INDEX idx_ngram_customer_name ON ${tableName};"
+ wait_for_latest_op_on_table_finish(tableName, timeout)
+
+ // Test 8: Verify filtering with index built
+ profile("sql_select_like_with_ngram_index_recreated_built") {
+ run {
+ sql "/* sql_select_like_with_ngram_index_recreated_built */
${query}"
+ sleep(1000)
+ }
+
+ check { profileString, exception ->
+ log.info(profileString)
+ assertTrue(profileString.contains("RowsBloomFilterFiltered: 20"))
+ }
+ }
+
+ // drop index
+ sql "DROP INDEX idx_ngram_customer_name ON ${tableName};"
+ wait_for_latest_op_on_table_finish(tableName, timeout)
+
+ // Test 9: Verify filtering with index dropped
+ profile("sql_select_like_with_ngram_index_recreated_dropped") {
+ run {
+ sql "/* sql_select_like_with_ngram_index_recreated_dropped */
${query}"
+ sleep(1000)
+ }
+
+ check { profileString, exception ->
+ log.info(profileString)
+ assertTrue(profileString.contains("RowsBloomFilterFiltered: 0"))
+ }
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
