This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new b779c4a3a35 branch-3.1: [feature](index)Support light index add for
inverted index without parser and ngram bf index #48461 #52251 (#52894)
b779c4a3a35 is described below
commit b779c4a3a35a86e67c7d73b865a2038e61e9da7d
Author: airborne12 <[email protected]>
AuthorDate: Wed Jul 9 13:42:39 2025 +0800
branch-3.1: [feature](index)Support light index add for inverted index
without parser and ngram bf index #48461 #52251 (#52894)
cherry pick from #48461 and #52251
---------
Co-authored-by: qiye <[email protected]>
---
fe/fe-core/src/main/cup/sql_parser.cup | 10 +-
.../apache/doris/alter/SchemaChangeHandler.java | 92 ++-
.../org/apache/doris/alter/SchemaChangeJobV2.java | 15 +-
.../apache/doris/analysis/BuildIndexClause.java | 61 +-
.../java/org/apache/doris/analysis/IndexDef.java | 4 +-
.../main/java/org/apache/doris/catalog/Index.java | 23 +
.../java/org/apache/doris/catalog/OlapTable.java | 14 +
.../cloud/datasource/CloudInternalCatalog.java | 8 +
.../java/org/apache/doris/qe/SessionVariable.java | 17 +
.../org/apache/doris/alter/CloudIndexTest.java | 658 +++++++++++++++++++++
.../org/apache/doris/alter/IndexChangeJobTest.java | 179 +++++-
.../doris/alter/SchemaChangeHandlerTest.java | 5 +-
.../org/apache/doris/catalog/CatalogTestUtil.java | 42 +-
.../test_ngram_bloomfilter_index_change.out | Bin 0 -> 5433 bytes
.../pipeline/cloud_p1/conf/fe_custom.conf | 1 -
.../test_ngram_bloomfilter_index_change.groovy | 325 ++++++++++
16 files changed, 1341 insertions(+), 113 deletions(-)
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup
b/fe/fe-core/src/main/cup/sql_parser.cup
index e2f77ba262f..d56090de37c 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -902,7 +902,6 @@ nonterminal Map<String, String> key_value_map,
opt_key_value_map, opt_key_value_
opt_ext_properties, opt_enable_feature_properties, properties;
nonterminal ColumnDef column_definition;
nonterminal IndexDef index_definition;
-nonterminal IndexDef build_index_definition;
nonterminal ArrayList<ColumnDef> column_definition_list;
nonterminal ArrayList<IndexDef> index_definition_list;
nonterminal AggregateType opt_agg_type;
@@ -2206,7 +2205,7 @@ create_stmt ::=
:}
| KW_BUILD KW_INDEX ident:indexName KW_ON table_name:tableName
opt_partition_names:partitionNames
{:
- RESULT = new AlterTableStmt(tableName, Lists.newArrayList(new
BuildIndexClause(tableName, new IndexDef(indexName, partitionNames, true),
false)));
+ RESULT = new AlterTableStmt(tableName, Lists.newArrayList(new
BuildIndexClause(tableName, indexName, partitionNames, false)));
:}
/* stage */
| KW_CREATE KW_STAGE opt_if_not_exists:ifNotExists ident:stageName
KW_PROPERTIES opt_key_value_map:properties
@@ -4133,13 +4132,6 @@ index_definition ::=
:}
;
-build_index_definition ::=
- KW_INDEX ident:indexName opt_partition_names:partitionNames
- {:
- RESULT = new IndexDef(indexName, partitionNames, true);
- :}
- ;
-
opt_nullable_type ::=
{:
RESULT = ColumnNullableType.DEFAULT;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index 7751cbac9f1..37e415afe02 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -1330,7 +1330,7 @@ public class SchemaChangeHandler extends AlterHandler {
throw new DdlException(e.getMessage());
}
- // check bloom filter has change
+ // check bloom filter has been changed
boolean hasBfChange = false;
Set<String> oriBfColumns = olapTable.getCopiedBfColumns();
double oriBfFpp = olapTable.getBfFpp();
@@ -1991,7 +1991,7 @@ public class SchemaChangeHandler extends AlterHandler {
List<Index> newIndexes = olapTable.getCopiedIndexes();
List<Index> alterIndexes = new ArrayList<>();
- Map<Long, Set<String>> invertedIndexOnPartitions = new HashMap<>();
+ Map<Long, Set<String>> indexOnPartitions = new HashMap<>();
boolean isDropIndex = false;
Map<String, String> propertyMap = new HashMap<>();
for (AlterClause alterClause : alterClauses) {
@@ -2145,10 +2145,23 @@ public class SchemaChangeHandler extends AlterHandler {
}
lightSchemaChange = false;
- if (index.isLightIndexChangeSupported() &&
!Config.isCloudMode()) {
+ // Check if the index supports light index change and
session variable is enabled
+ boolean enableAddIndexForNewData = true;
+ try {
+ ConnectContext context = ConnectContext.get();
+ if (context != null && context.getSessionVariable() !=
null) {
+ enableAddIndexForNewData =
context.getSessionVariable().isEnableAddIndexForNewData();
+ }
+ } catch (Exception e) {
+ LOG.warn("Failed to get session variable
enable_add_index_for_new_data, "
+ + "using default value: false", e);
+ }
+
+ // ngram_bf index can do light_schema_change in both local
and cloud mode
+ // inverted index can only do light_schema_change in local
mode
+ if
(index.isLightAddIndexSupported(enableAddIndexForNewData)) {
alterIndexes.add(index);
isDropIndex = false;
- // now only support light index change for inverted
index
lightIndexChange = true;
}
} else if (alterClause instanceof BuildIndexClause) {
@@ -2159,54 +2172,14 @@ public class SchemaChangeHandler extends AlterHandler {
throw new DdlException("BUILD INDEX operation failed:
No need to do it in cloud mode.");
}
- if (!olapTable.isPartitionedTable()) {
- List<String> specifiedPartitions =
indexDef.getPartitionNames();
- if (!specifiedPartitions.isEmpty()) {
- throw new DdlException("table " +
olapTable.getName()
- + " is not partitioned, cannot build index
with partitions.");
- }
- }
- List<Index> existedIndexes = olapTable.getIndexes();
- boolean found = false;
- for (Index existedIdx : existedIndexes) {
- if
(existedIdx.getIndexName().equalsIgnoreCase(indexDef.getIndexName())) {
- found = true;
- if (!existedIdx.isLightIndexChangeSupported()) {
- throw new DdlException("BUILD INDEX operation
failed: The index "
- + existedIdx.getIndexName() + " of
type " + existedIdx.getIndexType()
- + " does not support lightweight index
changes.");
- }
- for (Column column : olapTable.getBaseSchema()) {
- if (!column.getType().isVariantType()) {
- continue;
- }
- // variant type column can not support for
building index
- for (String indexColumn :
existedIdx.getColumns()) {
- if
(column.getName().equalsIgnoreCase(indexColumn)) {
- throw new DdlException("BUILD INDEX
operation failed: The "
- + indexDef.getIndexName() + "
index can not be built on the "
- + indexColumn + " column,
because it is a variant type column.");
- }
- }
- }
- index = existedIdx.clone();
- if (indexDef.getPartitionNames().isEmpty()) {
-
invertedIndexOnPartitions.put(index.getIndexId(),
olapTable.getPartitionNames());
- } else {
- invertedIndexOnPartitions.put(
- index.getIndexId(), new
HashSet<>(indexDef.getPartitionNames()));
- }
- break;
- }
- }
- if (!found) {
- throw new DdlException("index " +
indexDef.getIndexName()
- + " not exist, cannot build it with
defferred.");
+ if (indexDef.getPartitionNames().isEmpty()) {
+ indexOnPartitions.put(index.getIndexId(),
olapTable.getPartitionNames());
+ } else {
+ indexOnPartitions.put(
+ index.getIndexId(), new
HashSet<>(indexDef.getPartitionNames()));
}
- if (indexDef.isInvertedIndex()) {
- alterIndexes.add(index);
- }
+ alterIndexes.add(index);
buildIndexChange = true;
lightSchemaChange = false;
} else if (alterClause instanceof DropIndexClause) {
@@ -2224,7 +2197,9 @@ public class SchemaChangeHandler extends AlterHandler {
break;
}
}
- if (found.isLightIndexChangeSupported() &&
!Config.isCloudMode()) {
+ // only inverted index with local mode can do light drop
index change
+ if (found != null && found.getIndexType() ==
IndexDef.IndexType.INVERTED
+ && Config.isNotCloudMode()) {
alterIndexes.add(found);
isDropIndex = true;
lightIndexChange = true;
@@ -2248,13 +2223,16 @@ public class SchemaChangeHandler extends AlterHandler {
null, isDropIndex, jobId, false);
} else if (Config.enable_light_index_change && lightIndexChange) {
long jobId = Env.getCurrentEnv().getNextId();
- //for schema change add/drop inverted index optimize, direct
modify table meta firstly.
+ //for schema change add/drop inverted index and ngram_bf
optimize, direct modify table meta firstly.
modifyTableLightSchemaChange(rawSql, db, olapTable,
indexSchemaMap, newIndexes,
alterIndexes, isDropIndex, jobId,
false);
} else if (buildIndexChange) {
+ if (alterIndexes.isEmpty()) {
+ throw new DdlException("Altered index is empty. please
check your alter stmt.");
+ }
if (Config.enable_light_index_change) {
buildOrDeleteTableInvertedIndices(db, olapTable,
indexSchemaMap,
- alterIndexes,
invertedIndexOnPartitions, false);
+ alterIndexes, indexOnPartitions, false);
}
} else {
createJob(rawSql, db.getId(), olapTable, indexSchemaMap,
propertyMap, newIndexes);
@@ -2747,6 +2725,8 @@ public class SchemaChangeHandler extends AlterHandler {
olapTable.writeUnlock();
}
+ // if this table has ngram_bf index, we must run cancel for schema
change job
+ boolean hasNGramBFIndex = ((OlapTable)
olapTable).hasIndexOfType(IndexDef.IndexType.NGRAM_BF);
// alter job v2's cancel must be called outside the table lock
if (jobList.size() > 0) {
for (IndexChangeJob job : jobList) {
@@ -2761,6 +2741,8 @@ public class SchemaChangeHandler extends AlterHandler {
LOG.info("cancel build index job {} on table {} success",
jobId, tableName);
}
}
+ } else if (hasNGramBFIndex) {
+ cancelColumnJob(cancelAlterTableStmt);
} else {
throw new DdlException("No job to cancel for Table[" + tableName +
"]");
}
@@ -2800,7 +2782,7 @@ public class SchemaChangeHandler extends AlterHandler {
Column column = olapTable.getColumn(col);
if (column != null) {
indexDef.checkColumn(column, olapTable.getKeysType(),
-
olapTable.getTableProperty().getEnableUniqueKeyMergeOnWrite(),
+ olapTable.getEnableUniqueKeyMergeOnWrite(),
olapTable.getInvertedIndexFileStorageFormat());
if
(!InvertedIndexUtil.getInvertedIndexFieldPattern(indexDef.getProperties()).isEmpty())
{
throw new DdlException("Can not create index with field
pattern");
@@ -3040,7 +3022,7 @@ public class SchemaChangeHandler extends AlterHandler {
}
try {
buildOrDeleteTableInvertedIndices(db, olapTable,
indexSchemaMap,
- alterIndexes,
invertedIndexOnPartitions, true);
+ alterIndexes, invertedIndexOnPartitions, true);
} catch (Exception e) {
throw new DdlException(e.getMessage());
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
index 2e4ca2c6a70..ad4a6ef60d1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
@@ -151,7 +151,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
private boolean hasEnableUniqueKeySkipBitmapChanged = false;
// save all schema change tasks
- private AgentBatchTask schemaChangeBatchTask = new AgentBatchTask();
+ AgentBatchTask schemaChangeBatchTask = new AgentBatchTask();
protected SchemaChangeJobV2() {
super(JobType.SCHEMA_CHANGE);
@@ -678,14 +678,13 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
healthyReplicaNum++;
}
}
- if (!FeConstants.runningUnitTest) {
- if (healthyReplicaNum < expectReplicationNum / 2 +
1) {
- LOG.warn("shadow tablet {} has few healthy
replicas: {}, schema change job: {}"
- + " healthyReplicaNum {}
expectReplicationNum {}",
- shadowTablet.getId(), replicas, jobId,
healthyReplicaNum, expectReplicationNum);
- throw new AlterCancelException(
+
+ if ((healthyReplicaNum < expectReplicationNum / 2 + 1)
&& !FeConstants.runningUnitTest) {
+ LOG.warn("shadow tablet {} has few healthy
replicas: {}, schema change job: {}"
+ + " healthyReplicaNum {}
expectReplicationNum {}",
+ shadowTablet.getId(), replicas, jobId,
healthyReplicaNum, expectReplicationNum);
+ throw new AlterCancelException(
"shadow tablet " + shadowTablet.getId() +
" has few healthy replicas");
- }
}
} // end for tablets
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java
index d04d24e86e4..7cd1d915150 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BuildIndexClause.java
@@ -18,12 +18,17 @@
package org.apache.doris.analysis;
import org.apache.doris.alter.AlterOpType;
+import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Index;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.Table;
+import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.AnalysisException;
import com.google.common.collect.Maps;
+import java.util.List;
import java.util.Map;
public class BuildIndexClause extends AlterTableClause {
@@ -36,11 +41,14 @@ public class BuildIndexClause extends AlterTableClause {
private boolean alter;
// index internal class
private Index index;
+ private String indexName;
+ private PartitionNames partitionNames;
- public BuildIndexClause(TableName tableName, IndexDef indexDef, boolean
alter) {
+ public BuildIndexClause(TableName tableName, String indexName,
PartitionNames partitionNames, boolean alter) {
super(AlterOpType.SCHEMA_CHANGE);
this.tableName = tableName;
- this.indexDef = indexDef;
+ this.indexName = indexName;
+ this.partitionNames = partitionNames;
this.alter = alter;
}
@@ -67,17 +75,52 @@ public class BuildIndexClause extends AlterTableClause {
@Override
public void analyze(Analyzer analyzer) throws AnalysisException {
- if (indexDef == null) {
- throw new AnalysisException("index definition expected.");
+ tableName.analyze(analyzer);
+ DatabaseIf<Table> db =
Env.getCurrentEnv().getCatalogMgr().getInternalCatalog()
+ .getDb(tableName.getDb()).orElse(null);
+ if (db == null) {
+ throw new AnalysisException("Database[" + tableName.getDb() + "]
is not exist");
}
- if (indexDef.getIndexType() == IndexDef.IndexType.NGRAM_BF
- || indexDef.getIndexType() == IndexDef.IndexType.BLOOMFILTER) {
+
+ TableIf table = db.getTable(tableName.getTbl()).orElse(null);
+ if (table == null) {
+ throw new AnalysisException("Table[" + tableName.getTbl() + "] is
not exist");
+ }
+ if (!(table instanceof OlapTable)) {
+ throw new AnalysisException("Only olap table support build index");
+ }
+
+ Index existedIdx = null;
+ for (Index index : table.getTableIndexes().getIndexes()) {
+ if (index.getIndexName().equalsIgnoreCase(indexName)) {
+ existedIdx = index;
+ if (!existedIdx.isLightIndexChangeSupported()) {
+ throw new AnalysisException("BUILD INDEX operation failed:
The index "
+ + existedIdx.getIndexName() + " of type " +
existedIdx.getIndexType()
+ + " does not support lightweight index changes.");
+ }
+ break;
+ }
+ }
+ if (existedIdx == null) {
+ throw new AnalysisException("Index[" + indexName + "] is not exist
in table[" + tableName.getTbl() + "]");
+ }
+
+ IndexDef.IndexType indexType = existedIdx.getIndexType();
+ if (indexType == IndexDef.IndexType.NGRAM_BF
+ || indexType == IndexDef.IndexType.BLOOMFILTER) {
throw new AnalysisException("ngram bloomfilter or bloomfilter
index is not needed to build.");
}
+ indexDef = new IndexDef(indexName, partitionNames, indexType, true);
+ if (!table.isPartitionedTable()) {
+ List<String> specifiedPartitions = indexDef.getPartitionNames();
+ if (!specifiedPartitions.isEmpty()) {
+ throw new AnalysisException("table " + table.getName()
+ + " is not partitioned, cannot build index with
partitions.");
+ }
+ }
indexDef.analyze();
- this.index = new Index(Env.getCurrentEnv().getNextId(),
indexDef.getIndexName(),
- indexDef.getColumns(), indexDef.getIndexType(),
- indexDef.getProperties(), indexDef.getComment());
+ this.index = existedIdx.clone();
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
index 8d1f6a5bfd6..a1df981898e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java
@@ -83,9 +83,9 @@ public class IndexDef {
}
}
- public IndexDef(String indexName, PartitionNames partitionNames, boolean
isBuildDeferred) {
+ public IndexDef(String indexName, PartitionNames partitionNames, IndexType
indexType, boolean isBuildDeferred) {
this.indexName = indexName;
- this.indexType = IndexType.INVERTED;
+ this.indexType = indexType;
this.partitionNames = partitionNames;
this.isBuildDeferred = isBuildDeferred;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java
index 312e3194ed3..54407443a24 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Index.java
@@ -20,6 +20,7 @@ package org.apache.doris.catalog;
import org.apache.doris.analysis.IndexDef;
import org.apache.doris.analysis.InvertedIndexUtil;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.PrintableMap;
@@ -152,6 +153,10 @@ public class Index implements Writable {
return InvertedIndexUtil.getInvertedIndexParser(properties);
}
+ public boolean isInvertedIndexParserNone() {
+ return
InvertedIndexUtil.INVERTED_INDEX_PARSER_NONE.equals(getInvertedIndexParser());
+ }
+
public String getInvertedIndexParserMode() {
return InvertedIndexUtil.getInvertedIndexParserMode(properties);
}
@@ -168,10 +173,28 @@ public class Index implements Writable {
return InvertedIndexUtil.getInvertedIndexParserStopwords(properties);
}
+ // Whether the index can be changed in light mode
public boolean isLightIndexChangeSupported() {
return indexType == IndexDef.IndexType.INVERTED;
}
+ // Whether the index can be added in light mode
+ // cloud mode supports light add for ngram_bf index and non-tokenized
inverted index (parser="none")
+ // local mode supports light add for both inverted index and ngram_bf index
+ // the rest of the index types do not support light add
+ public boolean isLightAddIndexSupported(boolean enableAddIndexForNewData) {
+ if (Config.isCloudMode()) {
+ if (indexType == IndexDef.IndexType.INVERTED) {
+ return isInvertedIndexParserNone() && enableAddIndexForNewData;
+ } else if (indexType == IndexDef.IndexType.NGRAM_BF) {
+ return enableAddIndexForNewData;
+ }
+ return false;
+ }
+ return (indexType == IndexDef.IndexType.NGRAM_BF &&
enableAddIndexForNewData)
+ || (indexType == IndexDef.IndexType.INVERTED);
+ }
+
public String getInvertedIndexCustomAnalyzer() {
return InvertedIndexUtil.getInvertedIndexCustomAnalyzer(properties);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index d4dca760bfe..e39cc1ec0d0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -24,6 +24,7 @@ import org.apache.doris.analysis.ColumnDef;
import org.apache.doris.analysis.CreateMaterializedViewStmt;
import org.apache.doris.analysis.DataSortInfo;
import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.IndexDef;
import org.apache.doris.analysis.SlotDescriptor;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.backup.Status;
@@ -356,6 +357,19 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
return indexes.getIndexIds();
}
+ /**
+ * Checks if the table contains at least one index of the specified type.
+ * @param indexType The index type to check for
+ * @return true if the table has at least one index of the specified type,
false otherwise
+ */
+ public boolean hasIndexOfType(IndexDef.IndexType indexType) {
+ if (indexes == null) {
+ return false;
+ }
+ return indexes.getIndexes().stream()
+ .anyMatch(index -> index.getIndexType() == indexType);
+ }
+
@Override
public TableIndexes getTableIndexes() {
return indexes;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
index bcb17d3c039..f4f3893c87d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -339,6 +339,14 @@ public class CloudInternalCatalog extends InternalCatalog {
schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V2);
} else if (invertedIndexFileStorageFormat ==
TInvertedIndexFileStorageFormat.V3) {
schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V3);
+ } else if (invertedIndexFileStorageFormat ==
TInvertedIndexFileStorageFormat.DEFAULT) {
+ if
(Config.inverted_index_storage_format.equalsIgnoreCase("V1")) {
+
schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V1);
+ } else if
(Config.inverted_index_storage_format.equalsIgnoreCase("V3")) {
+
schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V3);
+ } else {
+
schemaBuilder.setInvertedIndexStorageFormat(OlapFile.InvertedIndexStorageFormatPB.V2);
+ }
} else {
throw new DdlException("invalid inverted index storage
format");
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index c268fddee53..7891b735d03 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -732,6 +732,7 @@ public class SessionVariable implements Serializable,
Writable {
public static final String SQL_CONVERTOR_CONFIG = "sql_convertor_config";
public static final String PREFER_UDF_OVER_BUILTIN =
"prefer_udf_over_builtin";
+ public static final String ENABLE_ADD_INDEX_FOR_NEW_DATA =
"enable_add_index_for_new_data";
/**
* If set false, user couldn't submit analyze SQL and FE won't allocate
any related resources.
@@ -2558,6 +2559,14 @@ public class SessionVariable implements Serializable,
Writable {
return enableSortSpill;
}
+ @VariableMgr.VarAttr(name = ENABLE_ADD_INDEX_FOR_NEW_DATA, fuzzy = true,
description = {
+ "是否启用仅对新数据生效的索引添加模式，开启时新建索引只对后续写入的数据生效，关闭时对全部数据重建索引",
+ "Whether to enable add index mode that only affects new data, "
+ + "when enabled new indexes only affect subsequently
written data, "
+ + "when disabled rebuild indexes for all data"
+ })
+ public boolean enableAddIndexForNewData = false;
+
// If this fe is in fuzzy mode, then will use initFuzzyModeVariables to
generate some variables,
// not the default value set in the code.
@SuppressWarnings("checkstyle:Indentation")
@@ -4873,5 +4882,13 @@ public class SessionVariable implements Serializable,
Writable {
public int getGlobalVariantMaxSubcolumnsCount() {
return globalVariantMaxSubcolumnsCount;
}
+
+ public boolean isEnableAddIndexForNewData() {
+ return enableAddIndexForNewData;
+ }
+
+ public void setEnableAddIndexForNewData(boolean enableAddIndexForNewData) {
+ this.enableAddIndexForNewData = enableAddIndexForNewData;
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java
b/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java
new file mode 100644
index 00000000000..10adecb1676
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/alter/CloudIndexTest.java
@@ -0,0 +1,658 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.alter;
+
+import org.apache.doris.analysis.AlterClause;
+import org.apache.doris.analysis.Analyzer;
+import org.apache.doris.analysis.CreateIndexClause;
+import org.apache.doris.analysis.DataSortInfo;
+import org.apache.doris.analysis.IndexDef;
+import org.apache.doris.analysis.IndexDef.IndexType;
+import org.apache.doris.analysis.ResourceTypeEnum;
+import org.apache.doris.analysis.TableName;
+import org.apache.doris.analysis.UserIdentity;
+import org.apache.doris.catalog.CatalogTestUtil;
+import org.apache.doris.catalog.Database;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.EnvFactory;
+import org.apache.doris.catalog.FakeEditLog;
+import org.apache.doris.catalog.FakeEnv;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.OlapTable.OlapTableState;
+import org.apache.doris.cloud.catalog.CloudEnv;
+import org.apache.doris.cloud.catalog.CloudEnvFactory;
+import org.apache.doris.cloud.catalog.ComputeGroup;
+import org.apache.doris.cloud.datasource.CloudInternalCatalog;
+import org.apache.doris.cloud.proto.Cloud;
+import org.apache.doris.cloud.proto.Cloud.MetaServiceCode;
+import org.apache.doris.cloud.rpc.MetaServiceProxy;
+import org.apache.doris.cloud.system.CloudSystemInfoService;
+import org.apache.doris.common.Config;
+import org.apache.doris.common.FeConstants;
+import org.apache.doris.common.UserException;
+import org.apache.doris.mysql.privilege.AccessControllerManager;
+import org.apache.doris.mysql.privilege.Auth;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.persist.EditLog;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.system.Backend;
+import org.apache.doris.system.SystemInfoService;
+import org.apache.doris.task.AgentTask;
+import org.apache.doris.task.AgentTaskQueue;
+import org.apache.doris.thrift.TInvertedIndexFileStorageFormat;
+import org.apache.doris.thrift.TSortType;
+import org.apache.doris.thrift.TTaskType;
+import org.apache.doris.utframe.MockedMetaServerFactory;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import mockit.Mock;
+import mockit.MockUp;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class CloudIndexTest {
+ private static final Logger LOG =
LogManager.getLogger(CloudIndexTest.class);
+
+ // Simple ComputeGroupMgr wrapper for CloudSystemInfoService
+ public static class ComputeGroupMgr {
+ private final SystemInfoService systemInfoService;
+
+ public ComputeGroupMgr(SystemInfoService systemInfoService) {
+ this.systemInfoService = systemInfoService;
+ }
+
+ public ComputeGroup getComputeGroupByName(String name) {
+ if (systemInfoService instanceof CloudSystemInfoService) {
+ return ((CloudSystemInfoService)
systemInfoService).getComputeGroupByName(name);
+ }
+ return null;
+ }
+
+ public ComputeGroup getAllBackendComputeGroup() {
+ // Return a default compute group for all backends
+ return new ComputeGroup("default_compute_group",
"default_compute_group",
+ ComputeGroup.ComputeTypeEnum.SQL);
+ }
+ }
+
+ private static String fileName = "./CloudIndexTest";
+
+ private static FakeEditLog fakeEditLog;
+ private static FakeEnv fakeEnv;
+ private static Env masterEnv;
+ private static EditLog testEditLog;
+ private ConnectContext ctx;
+ private static OlapTable olapTable;
+
+ private static Analyzer analyzer;
+ private static Database db;
+ private static CreateIndexClause createIndexClause;
+ private static SchemaChangeHandler schemaChangeHandler;
+
+ @Before
+ public void setUp() throws InstantiationException, IllegalAccessException,
IllegalArgumentException,
+ InvocationTargetException, NoSuchMethodException,
SecurityException, UserException {
+ FeConstants.runningUnitTest = true;
+ // Setup for MetaServiceProxy mock
+ new MockUp<MetaServiceProxy>(MetaServiceProxy.class) {
+
+ @Mock
+ public Cloud.BeginTxnResponse beginTxn(Cloud.BeginTxnRequest
request) {
+ Cloud.BeginTxnResponse.Builder beginTxnResponseBuilder =
Cloud.BeginTxnResponse.newBuilder();
+ beginTxnResponseBuilder.setTxnId(1000)
+ .setStatus(
+
Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK).setMsg("OK"));
+ return beginTxnResponseBuilder.build();
+ }
+
+ @Mock
+ public Cloud.CommitTxnResponse commitTxn(Cloud.CommitTxnRequest
request) {
+ Cloud.TxnInfoPB.Builder txnInfoBuilder =
Cloud.TxnInfoPB.newBuilder();
+ txnInfoBuilder.setDbId(CatalogTestUtil.testDbId1);
+
txnInfoBuilder.addAllTableIds(Lists.newArrayList(olapTable.getId()));
+ txnInfoBuilder.setLabel("test_label");
+ txnInfoBuilder.setListenerId(-1);
+ Cloud.CommitTxnResponse.Builder commitTxnResponseBuilder =
Cloud.CommitTxnResponse.newBuilder();
+
commitTxnResponseBuilder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder()
+ .setCode(MetaServiceCode.OK).setMsg("OK"))
+ .setTxnInfo(txnInfoBuilder.build());
+ return commitTxnResponseBuilder.build();
+ }
+
+ @Mock
+ public Cloud.CheckTxnConflictResponse
checkTxnConflict(Cloud.CheckTxnConflictRequest request) {
+ Cloud.CheckTxnConflictResponse.Builder
checkTxnConflictResponseBuilder =
+ Cloud.CheckTxnConflictResponse.newBuilder();
+
checkTxnConflictResponseBuilder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder()
+ .setCode(MetaServiceCode.OK).setMsg("OK"))
+ .setFinished(true);
+ return checkTxnConflictResponseBuilder.build();
+ }
+
+ @Mock
+ public Cloud.GetClusterResponse getCluster(Cloud.GetClusterRequest
request) {
+ Cloud.GetClusterResponse.Builder getClusterResponseBuilder =
Cloud.GetClusterResponse.newBuilder();
+ Cloud.ClusterPB.Builder clusterBuilder =
Cloud.ClusterPB.newBuilder();
+
clusterBuilder.setClusterId("test_id").setClusterName("test_group");
+
+ Cloud.NodeInfoPB.Builder node1 = Cloud.NodeInfoPB.newBuilder();
+ node1.setCloudUniqueId("test_cloud")
+ .setName("host1")
+ .setIp("host1")
+ .setHost("host1")
+ .setHeartbeatPort(123)
+ .setEditLogPort(125)
+ .setStatus(Cloud.NodeStatusPB.NODE_STATUS_RUNNING);
+ clusterBuilder.addNodes(node1.build());
+
getClusterResponseBuilder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder()
+ .setCode(MetaServiceCode.OK).setMsg("OK"))
+ .addCluster(clusterBuilder.build());
+ return getClusterResponseBuilder.build();
+ }
+
+ @Mock
+ public Cloud.CreateTabletsResponse
createTablets(Cloud.CreateTabletsRequest request) {
+ Cloud.CreateTabletsResponse.Builder responseBuilder =
Cloud.CreateTabletsResponse.newBuilder();
+ responseBuilder.setStatus(
+
Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK).setMsg("OK"));
+ return responseBuilder.build();
+ }
+
+ @Mock
+ public Cloud.FinishTabletJobResponse
finishTabletJob(Cloud.FinishTabletJobRequest request) {
+ Cloud.FinishTabletJobResponse.Builder responseBuilder =
Cloud.FinishTabletJobResponse.newBuilder();
+ responseBuilder.setStatus(
+
Cloud.MetaServiceResponseStatus.newBuilder().setCode(MetaServiceCode.OK).setMsg("OK"));
+ return responseBuilder.build();
+ }
+
+ @Mock
+ public Cloud.IndexResponse prepareIndex(Cloud.IndexRequest
request) {
+ Cloud.IndexResponse.Builder builder =
Cloud.IndexResponse.newBuilder();
+ builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder()
+ .setCode(MetaServiceCode.OK).setMsg("OK"));
+ return builder.build();
+ }
+
+ @Mock
+ public Cloud.IndexResponse commitIndex(Cloud.IndexRequest request)
{
+ Cloud.IndexResponse.Builder builder =
Cloud.IndexResponse.newBuilder();
+ builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder()
+ .setCode(MetaServiceCode.OK).setMsg("OK"));
+ return builder.build();
+ }
+
+ @Mock
+ public Cloud.IndexResponse dropIndex(Cloud.IndexRequest request) {
+ Cloud.IndexResponse.Builder builder =
Cloud.IndexResponse.newBuilder();
+ builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder()
+ .setCode(MetaServiceCode.OK).setMsg("OK"));
+ return builder.build();
+ }
+
+ @Mock
+ public Cloud.CheckKVResponse checkKv(Cloud.CheckKVRequest request)
{
+ Cloud.CheckKVResponse.Builder builder =
Cloud.CheckKVResponse.newBuilder();
+ builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder()
+ .setCode(MetaServiceCode.OK).setMsg("OK"));
+ return builder.build();
+ }
+
+ @Mock
+ public Cloud.GetCurrentMaxTxnResponse
getCurrentMaxTxnId(Cloud.GetCurrentMaxTxnRequest request) {
+ Cloud.GetCurrentMaxTxnResponse.Builder builder =
Cloud.GetCurrentMaxTxnResponse.newBuilder();
+ builder.setStatus(Cloud.MetaServiceResponseStatus.newBuilder()
+ .setCode(MetaServiceCode.OK).setMsg("OK"))
+ .setCurrentMaxTxnId(1000);
+ return builder.build();
+ }
+ };
+
+ Config.cloud_unique_id = "test_cloud";
+ Config.meta_service_endpoint =
MockedMetaServerFactory.METASERVER_DEFAULT_IP + ":" + 20121;
+
+ EnvFactory envFactory = EnvFactory.getInstance();
+ masterEnv = envFactory.createEnv(false);
+ SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo();
+ fakeEnv = new FakeEnv();
+ FakeEnv.setSystemInfo(cloudSystemInfo);
+
+ fakeEditLog = new FakeEditLog();
+ testEditLog = null; // Will be set by MockUp
+ FakeEnv.setEnv(masterEnv);
+
+ ctx = new ConnectContext();
+ ctx.setEnv(masterEnv);
+ ctx.setQualifiedUser("root");
+ UserIdentity rootUser = new UserIdentity("root", "%");
+ rootUser.setIsAnalyzed();
+ ctx.setCurrentUserIdentity(rootUser);
+ ctx.setThreadLocalInfo();
+ ctx.setCloudCluster("test_group");
+ Assert.assertTrue(envFactory instanceof CloudEnvFactory);
+ Assert.assertTrue(masterEnv instanceof CloudEnv);
+ new MockUp<Env>() {
+ @Mock
+ public Env getCurrentEnv() {
+ return masterEnv;
+ }
+
+ @Mock
+ public EditLog getEditLog() {
+ if (testEditLog == null) {
+ // Create a mock EditLog using a no-op approach
+ testEditLog = new EditLog("test") {
+ // Override to avoid initialization issues
+ };
+ }
+ return testEditLog;
+ }
+
+ @Mock
+ public ComputeGroupMgr getComputeGroupMgr() {
+ return new ComputeGroupMgr(Env.getCurrentSystemInfo());
+ }
+
+ @Mock
+ public SchemaChangeHandler getSchemaChangeHandler() {
+ // Create a new independent SchemaChangeHandler for each call
+ return schemaChangeHandler;
+ }
+
+ @Mock
+ public AccessControllerManager getAccessManager() {
+ return new AccessControllerManager(masterEnv.getAuth()) {
+ @Override
+ public boolean checkTblPriv(ConnectContext ctx, String
ctl, String db, String tbl, PrivPredicate wanted) {
+ return true; // Allow all access for test
+ }
+
+ @Override
+ public boolean checkCloudPriv(UserIdentity user, String
cluster, PrivPredicate wanted, ResourceTypeEnum resourceType) {
+ return true; // Allow all cloud privileges for test
+ }
+ };
+ }
+ };
+
+ new MockUp<Auth>() {
+ @Mock
+ public String getDefaultCloudCluster(String user) {
+ return "test_group"; // Return default cluster for test
+ }
+ };
+
+ // Mock cloud environment permissions
+ new MockUp<CloudEnv>() {
+ @Mock
+ public void checkCloudClusterPriv(String cluster) throws Exception
{
+ // Always allow for tests
+ }
+ };
+
+ // Mock ConnectContext to avoid compute group permission check
+ new MockUp<ConnectContext>() {
+ @Mock
+ public String getCloudCluster() {
+ return "test_group";
+ }
+
+ @Mock
+ public UserIdentity getCurrentUserIdentity() {
+ UserIdentity rootUser = new UserIdentity("root", "%");
+ rootUser.setIsAnalyzed();
+ return rootUser;
+ }
+ };
+
+ analyzer = new Analyzer(masterEnv, ctx);
+
+ Assert.assertTrue(Env.getCurrentSystemInfo() instanceof
CloudSystemInfoService);
+ CloudSystemInfoService systemInfo = (CloudSystemInfoService)
Env.getCurrentSystemInfo();
+ ComputeGroup pcg1 = new ComputeGroup("test_id", "test_group",
ComputeGroup.ComputeTypeEnum.COMPUTE);
+ systemInfo.addComputeGroup("test_id", pcg1);
+ Backend backend = new Backend(10001L, "host1", 123);
+ backend.setAlive(true);
+ backend.setBePort(456);
+ backend.setHttpPort(789);
+ backend.setBrpcPort(321);
+ Map<String, String> newTagMap =
org.apache.doris.resource.Tag.DEFAULT_BACKEND_TAG.toMap();
+ newTagMap.put(org.apache.doris.resource.Tag.CLOUD_CLUSTER_STATUS,
"NORMAL");
+ newTagMap.put(org.apache.doris.resource.Tag.CLOUD_CLUSTER_NAME,
"test_cluster");
+ newTagMap.put(org.apache.doris.resource.Tag.CLOUD_CLUSTER_ID,
"test_id");
+
newTagMap.put(org.apache.doris.resource.Tag.CLOUD_CLUSTER_PUBLIC_ENDPOINT, "");
+
newTagMap.put(org.apache.doris.resource.Tag.CLOUD_CLUSTER_PRIVATE_ENDPOINT, "");
+ newTagMap.put(org.apache.doris.resource.Tag.CLOUD_UNIQUE_ID,
"test_cloud");
+ backend.setTagMap(newTagMap);
+ List<Backend> backends = Lists.newArrayList(backend);
+ systemInfo.updateCloudClusterMapNoLock(backends, new ArrayList<>());
+ db = new Database(CatalogTestUtil.testDbId1, CatalogTestUtil.testDb1);
+ masterEnv.unprotectCreateDb(db);
+
+ AgentTaskQueue.clearAllTasks();
+ schemaChangeHandler = masterEnv.getSchemaChangeHandler();
+ }
+
+ @Test
+ public void testCreateNgramBfIndex() throws Exception {
+ Assert.assertTrue(Env.getCurrentSystemInfo() instanceof
CloudSystemInfoService);
+
+ SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo();
+ fakeEnv = new FakeEnv();
+ fakeEditLog = new FakeEditLog();
+ FakeEnv.setEnv(masterEnv);
+ FakeEnv.setSystemInfo(cloudSystemInfo);
+ schemaChangeHandler = (SchemaChangeHandler) new
Alter().getSchemaChangeHandler();
+
+ Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof
CloudInternalCatalog);
+ Assert.assertTrue(Env.getCurrentSystemInfo() instanceof
CloudSystemInfoService);
+ CatalogTestUtil.createDupTable(db);
+ olapTable = (OlapTable)
db.getTableOrDdlException(CatalogTestUtil.testTableId2);
+ DataSortInfo dataSortInfo = new DataSortInfo();
+ dataSortInfo.setSortType(TSortType.LEXICAL);
+ olapTable.setDataSortInfo(dataSortInfo);
+ String indexName = "ngram_bf_index";
+
+ // Add required properties for NGRAM_BF index
+ Map<String, String> properties = Maps.newHashMap();
+ properties.put("gram_size", "2");
+ properties.put("bf_size", "256");
+
+ IndexDef indexDef = new IndexDef(indexName, false,
+ Lists.newArrayList(olapTable.getBaseSchema().get(3).getName()),
+ org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF,
+ properties, "ngram bf index");
+ TableName tableName = new
TableName(masterEnv.getInternalCatalog().getName(), db.getName(),
+ olapTable.getName());
+ createIndexClause = new CreateIndexClause(tableName, indexDef, false);
+ createIndexClause.analyze(analyzer);
+ ArrayList<AlterClause> alterClauses = new ArrayList<>();
+ alterClauses.add(createIndexClause);
+ ctx.getSessionVariable().setEnableAddIndexForNewData(true);
+ schemaChangeHandler.process(alterClauses, db, olapTable);
+ Map<Long, AlterJobV2> indexChangeJobMap =
schemaChangeHandler.getAlterJobsV2();
+ Assert.assertEquals(1, indexChangeJobMap.size());
+ Assert.assertEquals(1, olapTable.getIndexes().size());
+ Assert.assertEquals("ngram_bf_index",
olapTable.getIndexes().get(0).getIndexName());
+ Assert.assertEquals(OlapTableState.NORMAL, olapTable.getState());
+
+ long createJobId =
indexChangeJobMap.values().stream().findAny().get().jobId;
+
+ // Finish the create index job first
+ SchemaChangeJobV2 createJobV2 = (SchemaChangeJobV2)
indexChangeJobMap.get(createJobId);
+ Assert.assertEquals(AlterJobV2.JobState.FINISHED,
createJobV2.getJobState());
+ }
+
+ @Test
+ public void testNormalCreateNgramBfIndex() throws Exception {
+ Assert.assertTrue(Env.getCurrentSystemInfo() instanceof
CloudSystemInfoService);
+
+ SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo();
+ fakeEnv = new FakeEnv();
+ fakeEditLog = new FakeEditLog();
+ FakeEnv.setEnv(masterEnv);
+ FakeEnv.setSystemInfo(cloudSystemInfo);
+ schemaChangeHandler = (SchemaChangeHandler) new
Alter().getSchemaChangeHandler();
+
+ Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof
CloudInternalCatalog);
+ Assert.assertTrue(Env.getCurrentSystemInfo() instanceof
CloudSystemInfoService);
+ CatalogTestUtil.createDupTable(db);
+ olapTable = (OlapTable)
db.getTableOrDdlException(CatalogTestUtil.testTableId2);
+ DataSortInfo dataSortInfo = new DataSortInfo();
+ dataSortInfo.setSortType(TSortType.LEXICAL);
+ olapTable.setDataSortInfo(dataSortInfo);
+ String indexName = "ngram_bf_index";
+
+ // Add required properties for NGRAM_BF index
+ Map<String, String> properties = Maps.newHashMap();
+ properties.put("gram_size", "2");
+ properties.put("bf_size", "256");
+
+ IndexDef indexDef = new IndexDef(indexName, false,
+ Lists.newArrayList(olapTable.getBaseSchema().get(3).getName()),
+ org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF,
+ properties, "ngram bf index");
+ TableName tableName = new
TableName(masterEnv.getInternalCatalog().getName(), db.getName(),
+ olapTable.getName());
+ createIndexClause = new CreateIndexClause(tableName, indexDef, false);
+ createIndexClause.analyze(analyzer);
+ ArrayList<AlterClause> alterClauses = new ArrayList<>();
+ alterClauses.add(createIndexClause);
+ // Set session variable to false (default)
+ ctx.getSessionVariable().setEnableAddIndexForNewData(false);
+ schemaChangeHandler.process(alterClauses, db, olapTable);
+ Map<Long, AlterJobV2> indexChangeJobMap =
schemaChangeHandler.getAlterJobsV2();
+ Assert.assertEquals(1, indexChangeJobMap.size());
+ Assert.assertEquals(OlapTableState.SCHEMA_CHANGE,
olapTable.getState());
+
+ long createJobId =
indexChangeJobMap.values().stream().findAny().get().jobId;
+
+ // Finish the create index job first
+ SchemaChangeJobV2 createJobV2 = (SchemaChangeJobV2)
indexChangeJobMap.get(createJobId);
+ schemaChangeHandler.runAfterCatalogReady();
+ Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN,
createJobV2.getJobState());
+ schemaChangeHandler.runAfterCatalogReady();
+ Assert.assertEquals(AlterJobV2.JobState.RUNNING,
createJobV2.getJobState());
+ Assert.assertEquals(1, createJobV2.schemaChangeBatchTask.getTaskNum());
+
+ List<AgentTask> tasks = AgentTaskQueue.getTask(TTaskType.ALTER);
+ Assert.assertEquals(1, tasks.size());
+ for (AgentTask agentTask : tasks) {
+ agentTask.setFinished(true);
+ }
+
+ schemaChangeHandler.runAfterCatalogReady();
+ Assert.assertEquals(AlterJobV2.JobState.FINISHED,
createJobV2.getJobState());
+ Assert.assertEquals(OlapTableState.NORMAL, olapTable.getState());
+ Assert.assertEquals(1, olapTable.getIndexes().size());
+ Assert.assertEquals("ngram_bf_index",
olapTable.getIndexes().get(0).getIndexName());
+ }
+
+ @Test
+ public void testCreateInvertedIndex() throws Exception {
+ Assert.assertTrue(Env.getCurrentSystemInfo() instanceof
CloudSystemInfoService);
+
+ SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo();
+ fakeEnv = new FakeEnv();
+ fakeEditLog = new FakeEditLog();
+ FakeEnv.setEnv(masterEnv);
+ FakeEnv.setSystemInfo(cloudSystemInfo);
+ schemaChangeHandler = (SchemaChangeHandler) new
Alter().getSchemaChangeHandler();
+
+ Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof
CloudInternalCatalog);
+ Assert.assertTrue(Env.getCurrentSystemInfo() instanceof
CloudSystemInfoService);
+ CatalogTestUtil.createDupTable(db);
+ olapTable = (OlapTable)
db.getTableOrDdlException(CatalogTestUtil.testTableId2);
+ DataSortInfo dataSortInfo = new DataSortInfo();
+ dataSortInfo.setSortType(TSortType.LEXICAL);
+ olapTable.setDataSortInfo(dataSortInfo);
+ String indexName = "raw_inverted_index";
+ // Explicitly set parser="none" for raw inverted index
+ Map<String, String> properties = Maps.newHashMap();
+ properties.put("parser", "none");
+
+ IndexDef indexDef = new IndexDef(indexName, false,
+ Lists.newArrayList(olapTable.getBaseSchema().get(3).getName()),
+ IndexType.INVERTED,
+ properties, "raw inverted index");
+ TableName tableName = new
TableName(masterEnv.getInternalCatalog().getName(), db.getName(),
+ olapTable.getName());
+ createIndexClause = new CreateIndexClause(tableName, indexDef, false);
+ createIndexClause.analyze(analyzer);
+ ArrayList<AlterClause> alterClauses = new ArrayList<>();
+ alterClauses.add(createIndexClause);
+ ctx.getSessionVariable().setEnableAddIndexForNewData(false);
+ schemaChangeHandler.process(alterClauses, db, olapTable);
+ Map<Long, AlterJobV2> indexChangeJobMap =
schemaChangeHandler.getAlterJobsV2();
+ Assert.assertEquals(1, indexChangeJobMap.size());
+
+ long createJobId =
indexChangeJobMap.values().stream().findAny().get().jobId;
+ Assert.assertEquals(OlapTableState.SCHEMA_CHANGE,
olapTable.getState());
+
+ // Finish the create index job first
+ SchemaChangeJobV2 createJobV2 = (SchemaChangeJobV2)
indexChangeJobMap.get(createJobId);
+ schemaChangeHandler.runAfterCatalogReady();
+ Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN,
createJobV2.getJobState());
+
+ schemaChangeHandler.runAfterCatalogReady();
+ Assert.assertEquals(AlterJobV2.JobState.RUNNING,
createJobV2.getJobState());
+ Assert.assertEquals(1, createJobV2.schemaChangeBatchTask.getTaskNum());
+
+ List<AgentTask> tasks = AgentTaskQueue.getTask(TTaskType.ALTER);
+ Assert.assertEquals(1, tasks.size());
+ for (AgentTask agentTask : tasks) {
+ agentTask.setFinished(true);
+ }
+
+ schemaChangeHandler.runAfterCatalogReady();
+ Assert.assertEquals(AlterJobV2.JobState.FINISHED,
createJobV2.getJobState());
+ Assert.assertEquals(OlapTableState.NORMAL, olapTable.getState());
+ Assert.assertEquals(1, olapTable.getIndexes().size());
+ Assert.assertEquals("raw_inverted_index",
olapTable.getIndexes().get(0).getIndexName());
+ }
+
+ @Test
+ public void testCreateInvertedIndexWithLightweightMode() throws Exception {
+ Assert.assertTrue(Env.getCurrentSystemInfo() instanceof
CloudSystemInfoService);
+
+ SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo();
+ fakeEnv = new FakeEnv();
+ fakeEditLog = new FakeEditLog();
+ FakeEnv.setEnv(masterEnv);
+ FakeEnv.setSystemInfo(cloudSystemInfo);
+ schemaChangeHandler = (SchemaChangeHandler) new
Alter().getSchemaChangeHandler();
+
+ Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof
CloudInternalCatalog);
+ Assert.assertTrue(Env.getCurrentSystemInfo() instanceof
CloudSystemInfoService);
+ CatalogTestUtil.createDupTable(db);
+ olapTable = (OlapTable)
db.getTableOrDdlException(CatalogTestUtil.testTableId2);
+ DataSortInfo dataSortInfo = new DataSortInfo();
+ dataSortInfo.setSortType(TSortType.LEXICAL);
+ olapTable.setDataSortInfo(dataSortInfo);
+ String indexName = "lightweight_raw_inverted_index";
+ // Explicitly set parser="none" for raw inverted index
+ Map<String, String> properties = Maps.newHashMap();
+ properties.put("parser", "none");
+ IndexDef indexDef = new IndexDef(indexName, false,
+ Lists.newArrayList(olapTable.getBaseSchema().get(3).getName()),
+ IndexType.INVERTED,
+ properties, "lightweight raw inverted index");
+ TableName tableName = new
TableName(masterEnv.getInternalCatalog().getName(), db.getName(),
+ olapTable.getName());
+ createIndexClause = new CreateIndexClause(tableName, indexDef, false);
+ createIndexClause.analyze(analyzer);
+ ArrayList<AlterClause> alterClauses = new ArrayList<>();
+ alterClauses.add(createIndexClause);
+ // Test with enable_add_index_for_new_data = true, should use
lightweight mode
+ ctx.getSessionVariable().setEnableAddIndexForNewData(true);
+ schemaChangeHandler.process(alterClauses, db, olapTable);
+ Map<Long, AlterJobV2> indexChangeJobMap =
schemaChangeHandler.getAlterJobsV2();
+ // Lightweight mode should not create any schema change jobs
+ Assert.assertEquals(1, indexChangeJobMap.size());
+ Assert.assertEquals(1, olapTable.getIndexes().size());
+ Assert.assertEquals("lightweight_raw_inverted_index",
olapTable.getIndexes().get(0).getIndexName());
+ Assert.assertEquals(OlapTableState.NORMAL, olapTable.getState());
+ // Verify the index properties
+ Assert.assertEquals("none",
olapTable.getIndexes().get(0).getProperties().get("parser"));
+ }
+
+ @Test
+ public void testCreateTokenizedInvertedIndex() throws Exception {
+ Assert.assertTrue(Env.getCurrentSystemInfo() instanceof
CloudSystemInfoService);
+
+ SystemInfoService cloudSystemInfo = Env.getCurrentSystemInfo();
+ fakeEnv = new FakeEnv();
+ fakeEditLog = new FakeEditLog();
+ FakeEnv.setEnv(masterEnv);
+ FakeEnv.setSystemInfo(cloudSystemInfo);
+ schemaChangeHandler = (SchemaChangeHandler) new
Alter().getSchemaChangeHandler();
+
+ Assert.assertTrue(Env.getCurrentInternalCatalog() instanceof
CloudInternalCatalog);
+ Assert.assertTrue(Env.getCurrentSystemInfo() instanceof
CloudSystemInfoService);
+ CatalogTestUtil.createDupTable(db);
+ olapTable = (OlapTable)
db.getTableOrDdlException(CatalogTestUtil.testTableId2);
+ DataSortInfo dataSortInfo = new DataSortInfo();
+ dataSortInfo.setSortType(TSortType.LEXICAL);
+ olapTable.setDataSortInfo(dataSortInfo);
+
+ // Set inverted index file storage format to V2 for cloud mode
+
olapTable.setInvertedIndexFileStorageFormat(TInvertedIndexFileStorageFormat.V2);
+
+ String indexName = "tokenized_inverted_index";
+ Map<String, String> properties = Maps.newHashMap();
+ properties.put("parser", "english");
+ properties.put("support_phrase", "true");
+ properties.put("lower_case", "true");
+
+ // Use VARCHAR column v1 (index 2) for string type support
+ IndexDef indexDef = new IndexDef(indexName, false,
+ Lists.newArrayList(olapTable.getBaseSchema().get(2).getName()),
+ IndexType.INVERTED,
+ properties, "tokenized inverted index with english parser");
+ TableName tableName = new
TableName(masterEnv.getInternalCatalog().getName(), db.getName(),
+ olapTable.getName());
+ createIndexClause = new CreateIndexClause(tableName, indexDef, false);
+ createIndexClause.analyze(analyzer);
+ ArrayList<AlterClause> alterClauses = new ArrayList<>();
+ alterClauses.add(createIndexClause);
+ schemaChangeHandler.process(alterClauses, db, olapTable);
+ Map<Long, AlterJobV2> indexChangeJobMap =
schemaChangeHandler.getAlterJobsV2();
+ Assert.assertEquals(1, indexChangeJobMap.size());
+ Assert.assertEquals(OlapTableState.SCHEMA_CHANGE,
olapTable.getState());
+
+ SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2)
indexChangeJobMap.values().stream()
+ .findFirst()
+ .orElse(null);
+ Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum());
+
+ // This should be a heavyweight schema change for tokenized index
+ schemaChangeHandler.runAfterCatalogReady();
+ Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN,
jobV2.getJobState());
+ Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum());
+
+ schemaChangeHandler.runAfterCatalogReady();
+ Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState());
+ Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum());
+
+ List<AgentTask> tasks = AgentTaskQueue.getTask(TTaskType.ALTER);
+ Assert.assertEquals(1, tasks.size());
+ for (AgentTask agentTask : tasks) {
+ agentTask.setFinished(true);
+ }
+
+ schemaChangeHandler.runAfterCatalogReady();
+ Assert.assertEquals(AlterJobV2.JobState.FINISHED, jobV2.getJobState());
+
+ Assert.assertEquals(1, olapTable.getIndexes().size());
+ Assert.assertEquals("tokenized_inverted_index",
olapTable.getIndexes().get(0).getIndexName());
+
+ // Verify that the index has the correct properties
+ Assert.assertEquals("english",
olapTable.getIndexes().get(0).getProperties().get("parser"));
+ Assert.assertEquals("true",
olapTable.getIndexes().get(0).getProperties().get("support_phrase"));
+ Assert.assertEquals("true",
olapTable.getIndexes().get(0).getProperties().get("lower_case"));
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java
b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java
index 6b0943c306f..29aed7e050d 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/alter/IndexChangeJobTest.java
@@ -46,6 +46,7 @@ import org.apache.doris.common.FeConstants;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.PropertyAnalyzer;
+import org.apache.doris.qe.ConnectContext;
import org.apache.doris.task.AgentTask;
import org.apache.doris.task.AgentTaskQueue;
import org.apache.doris.thrift.TStatusCode;
@@ -55,6 +56,8 @@ import org.apache.doris.transaction.GlobalTransactionMgr;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import mockit.Mock;
+import mockit.MockUp;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
@@ -85,6 +88,9 @@ public class IndexChangeJobTest {
private static BuildIndexClause buildIndexClause;
private static DropIndexClause dropIndexClause;
private static CancelAlterTableStmt cancelAlterTableStmt;
+ private static TableName tableName;
+ private static String indexName;
+ private static ConnectContext ctx;
@Rule
public ExpectedException expectedEx = ExpectedException.none();
@@ -108,25 +114,40 @@ public class IndexChangeJobTest {
db =
masterEnv.getInternalCatalog().getDbOrDdlException(CatalogTestUtil.testDbId1);
olapTable = (OlapTable)
db.getTableOrDdlException(CatalogTestUtil.testTableId1);
+ new MockUp<Env>() {
+ @Mock
+ public Env getCurrentEnv() {
+ return masterEnv;
+ }
+ };
+
+ // Initialize ConnectContext
+ ctx = new ConnectContext();
+ new MockUp<ConnectContext>() {
+ @Mock
+ public ConnectContext get() {
+ return ctx;
+ }
+ };
+
// set mow table property
Map<String, String> properties = Maps.newHashMap();
properties.put(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE,
"false");
TableProperty tableProperty = new TableProperty(properties);
olapTable.setTableProperty(tableProperty);
- TableName tableName = new
TableName(masterEnv.getInternalCatalog().getName(), db.getName(),
+ tableName = new TableName(masterEnv.getInternalCatalog().getName(),
db.getName(),
olapTable.getName());
- IndexDef indexDef = new IndexDef("index1", false,
+ indexName = "index1";
+ IndexDef indexDef = new IndexDef(indexName, false,
Lists.newArrayList(olapTable.getBaseSchema().get(1).getName()),
IndexDef.IndexType.INVERTED,
Maps.newHashMap(), "balabala");
+ FakeEnv.setEnv(masterEnv);
createIndexClause = new CreateIndexClause(tableName, indexDef, false);
createIndexClause.analyze(analyzer);
- buildIndexClause = new BuildIndexClause(tableName, indexDef, false);
- buildIndexClause.analyze(analyzer);
-
- dropIndexClause = new DropIndexClause("index1", false, tableName,
false);
+ dropIndexClause = new DropIndexClause(indexName, false, tableName,
false);
dropIndexClause.analyze(analyzer);
cancelAlterTableStmt = new
CancelAlterTableStmt(ShowAlterStmt.AlterType.INDEX, tableName);
@@ -163,6 +184,8 @@ public class IndexChangeJobTest {
Assert.assertEquals(olapTable.getIndexes().size(), 1);
Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(),
"index1");
alterClauses.clear();
+ buildIndexClause = new BuildIndexClause(tableName, indexName, null,
false);
+ buildIndexClause.analyze(analyzer);
alterClauses.add(buildIndexClause);
schemaChangeHandler.process(alterClauses, db, olapTable);
Map<Long, IndexChangeJob> indexChangeJobMap =
schemaChangeHandler.getIndexChangeJobs();
@@ -203,6 +226,8 @@ public class IndexChangeJobTest {
Assert.assertEquals(olapTable.getIndexes().size(), 1);
Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(),
"index1");
alterClauses.clear();
+ buildIndexClause = new BuildIndexClause(tableName, indexName, null,
false);
+ buildIndexClause.analyze(analyzer);
alterClauses.add(buildIndexClause);
schemaChangeHandler.process(alterClauses, db, olapTable);
Map<Long, IndexChangeJob> indexChangeJobMap =
schemaChangeHandler.getIndexChangeJobs();
@@ -292,6 +317,8 @@ public class IndexChangeJobTest {
Assert.assertEquals(olapTable.getIndexes().size(), 1);
Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(),
"index1");
alterClauses.clear();
+ buildIndexClause = new BuildIndexClause(tableName, indexName, null,
false);
+ buildIndexClause.analyze(analyzer);
alterClauses.add(buildIndexClause);
schemaChangeHandler.process(alterClauses, db, olapTable);
Map<Long, IndexChangeJob> indexChangeJobMap =
schemaChangeHandler.getIndexChangeJobs();
@@ -339,6 +366,8 @@ public class IndexChangeJobTest {
Assert.assertEquals(olapTable.getIndexes().size(), 1);
Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(),
"index1");
alterClauses.clear();
+ buildIndexClause = new BuildIndexClause(tableName, indexName, null,
false);
+ buildIndexClause.analyze(analyzer);
alterClauses.add(buildIndexClause);
schemaChangeHandler.process(alterClauses, db, olapTable);
Map<Long, IndexChangeJob> indexChangeJobMap =
schemaChangeHandler.getIndexChangeJobs();
@@ -467,6 +496,8 @@ public class IndexChangeJobTest {
Assert.assertEquals(olapTable.getIndexes().size(), 1);
Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(),
"index1");
alterClauses.clear();
+ buildIndexClause = new BuildIndexClause(tableName, indexName, null,
false);
+ buildIndexClause.analyze(analyzer);
alterClauses.add(buildIndexClause);
schemaChangeHandler.process(alterClauses, db, olapTable);
Map<Long, IndexChangeJob> indexChangeJobMap =
schemaChangeHandler.getIndexChangeJobs();
@@ -519,6 +550,8 @@ public class IndexChangeJobTest {
Assert.assertEquals(olapTable.getIndexes().size(), 1);
Assert.assertEquals(olapTable.getIndexes().get(0).getIndexName(),
"index1");
alterClauses.clear();
+ buildIndexClause = new BuildIndexClause(tableName, indexName, null,
false);
+ buildIndexClause.analyze(analyzer);
alterClauses.add(buildIndexClause);
schemaChangeHandler.process(alterClauses, db, olapTable);
Map<Long, IndexChangeJob> indexChangeJobMap =
schemaChangeHandler.getIndexChangeJobs();
@@ -526,13 +559,13 @@ public class IndexChangeJobTest {
Assert.assertEquals(OlapTableState.NORMAL, olapTable.getState());
IndexChangeJob indexChangejob =
indexChangeJobMap.values().stream().findAny().get();
-
Assert.assertEquals(indexChangejob.invertedIndexBatchTask.getTaskNum(), 0);
+ Assert.assertEquals(0,
indexChangejob.invertedIndexBatchTask.getTaskNum());
Assert.assertEquals(IndexChangeJob.JobState.WAITING_TXN,
indexChangejob.getJobState());
// run waiting txn job
schemaChangeHandler.runAfterCatalogReady();
Assert.assertEquals(IndexChangeJob.JobState.RUNNING,
indexChangejob.getJobState());
-
Assert.assertEquals(indexChangejob.invertedIndexBatchTask.getTaskNum(), 3);
+ Assert.assertEquals(3,
indexChangejob.invertedIndexBatchTask.getTaskNum());
// run running job
schemaChangeHandler.runAfterCatalogReady();
Assert.assertEquals(IndexChangeJob.JobState.RUNNING,
indexChangejob.getJobState());
@@ -565,17 +598,135 @@ public class IndexChangeJobTest {
fakeEditLog = new FakeEditLog();
FakeEnv.setEnv(masterEnv);
- IndexDef indexDef = new IndexDef("ngram_bf_index", false,
- Lists.newArrayList(olapTable.getBaseSchema().get(1).getName()),
+ OlapTable table = (OlapTable)
db.getTableOrDdlException(CatalogTestUtil.testTableId2);
+ String indexName = "ngram_bf_index";
+ IndexDef indexDef = new IndexDef(indexName, false,
+ Lists.newArrayList(table.getBaseSchema().get(3).getName()),
org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF,
Maps.newHashMap(), "ngram bf index");
TableName tableName = new
TableName(masterEnv.getInternalCatalog().getName(), db.getName(),
- olapTable.getName());
+ table.getName());
createIndexClause = new CreateIndexClause(tableName, indexDef, false);
createIndexClause.analyze(analyzer);
+ SchemaChangeHandler schemaChangeHandler =
Env.getCurrentEnv().getSchemaChangeHandler();
+ ArrayList<AlterClause> alterClauses = new ArrayList<>();
+ alterClauses.add(createIndexClause);
+
+ // Test with enable_add_index_for_new_data = true
+ ConnectContext context = ConnectContext.get();
+ context.getSessionVariable().setEnableAddIndexForNewData(true);
+ schemaChangeHandler.process(alterClauses, db, table);
+ Map<Long, AlterJobV2> indexChangeJobMap =
schemaChangeHandler.getAlterJobsV2();
+ Assert.assertEquals(1, indexChangeJobMap.size());
+ Assert.assertEquals(1, table.getIndexes().size());
+ Assert.assertEquals("ngram_bf_index",
table.getIndexes().get(0).getIndexName());
+
+ SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2)
indexChangeJobMap.values().stream()
+ .findFirst()
+ .orElse(null);
+ Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum());
+ Assert.assertEquals(AlterJobV2.JobState.FINISHED, jobV2.getJobState());
+
+ // Clean up for next test
+ table.setIndexes(Lists.newArrayList());
+ indexChangeJobMap.clear();
+ AgentTaskQueue.clearAllTasks();
+
+ // Test with enable_add_index_for_new_data = false
+ context.getSessionVariable().setEnableAddIndexForNewData(false);
+ String indexName2 = "ngram_bf_index2";
+ IndexDef indexDef2 = new IndexDef(indexName2, false,
+ Lists.newArrayList(table.getBaseSchema().get(3).getName()),
+ org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF,
+ Maps.newHashMap(), "ngram bf index2");
+
+ createIndexClause = new CreateIndexClause(tableName, indexDef2, false);
+ createIndexClause.analyze(analyzer);
+ ArrayList<AlterClause> alterClauses2 = new ArrayList<>();
+ alterClauses2.add(createIndexClause);
+ schemaChangeHandler.process(alterClauses2, db, table);
+ indexChangeJobMap = schemaChangeHandler.getAlterJobsV2();
+ Assert.assertEquals(1, indexChangeJobMap.size());
+ Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState());
+
+ jobV2 = (SchemaChangeJobV2) indexChangeJobMap.values().stream()
+ .findFirst()
+ .orElse(null);
+ Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum());
+
+ schemaChangeHandler.runAfterCatalogReady();
+ Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN,
jobV2.getJobState());
+ Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum());
+
+ schemaChangeHandler.runAfterCatalogReady();
+ Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState());
+ Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum());
+
+ schemaChangeHandler.runAfterCatalogReady();
+ Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState());
+ Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum());
- buildIndexClause = new BuildIndexClause(tableName, indexDef, false);
-
org.junit.jupiter.api.Assertions.assertThrows(org.apache.doris.common.AnalysisException.class,
- () -> buildIndexClause.analyze(analyzer));
+ List<AgentTask> tasks = AgentTaskQueue.getTask(TTaskType.ALTER);
+ Assert.assertEquals(1, tasks.size());
+ for (AgentTask agentTask : tasks) {
+ agentTask.setFinished(true);
+ }
+
+ schemaChangeHandler.runAfterCatalogReady();
+ Assert.assertEquals(AlterJobV2.JobState.FINISHED, jobV2.getJobState());
+ Assert.assertEquals(1, table.getIndexes().size());
+ Assert.assertEquals("ngram_bf_index2",
table.getIndexes().get(0).getIndexName());
+ }
+
+ @Test
+ public void testCancelNgramBfBuildIndex() throws UserException {
+ fakeEnv = new FakeEnv();
+ fakeEditLog = new FakeEditLog();
+ FakeEnv.setEnv(masterEnv);
+
+ OlapTable table = (OlapTable)
db.getTableOrDdlException(CatalogTestUtil.testTableId2);
+ String indexName = "ngram_bf_index";
+ IndexDef indexDef = new IndexDef(indexName, false,
+ Lists.newArrayList(table.getBaseSchema().get(3).getName()),
+ org.apache.doris.analysis.IndexDef.IndexType.NGRAM_BF,
+ Maps.newHashMap(), "ngram bf index");
+ TableName tableName = new
TableName(masterEnv.getInternalCatalog().getName(), db.getName(),
+ table.getName());
+ createIndexClause = new CreateIndexClause(tableName, indexDef, false);
+ createIndexClause.analyze(analyzer);
+ SchemaChangeHandler schemaChangeHandler =
Env.getCurrentEnv().getSchemaChangeHandler();
+ ArrayList<AlterClause> alterClauses = new ArrayList<>();
+ alterClauses.add(createIndexClause);
+
+ //cancel test can only with enable_add_index_for_new_data = false
+ ctx.getSessionVariable().setEnableAddIndexForNewData(false);
+ schemaChangeHandler.process(alterClauses, db, table);
+ Map<Long, AlterJobV2> indexChangeJobMap =
schemaChangeHandler.getAlterJobsV2();
+ Assert.assertEquals(1, indexChangeJobMap.size());
+ Assert.assertEquals(OlapTableState.SCHEMA_CHANGE, table.getState());
+
+ SchemaChangeJobV2 jobV2 = (SchemaChangeJobV2)
indexChangeJobMap.values().stream()
+ .findFirst()
+ .orElse(null);
+ Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum());
+
+ schemaChangeHandler.runAfterCatalogReady();
+ Assert.assertEquals(AlterJobV2.JobState.WAITING_TXN,
jobV2.getJobState());
+ Assert.assertEquals(0, jobV2.schemaChangeBatchTask.getTaskNum());
+
+ schemaChangeHandler.runAfterCatalogReady();
+ Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState());
+ Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum());
+
+ schemaChangeHandler.runAfterCatalogReady();
+ Assert.assertEquals(AlterJobV2.JobState.RUNNING, jobV2.getJobState());
+ Assert.assertEquals(1, jobV2.schemaChangeBatchTask.getTaskNum());
+
+ cancelAlterTableStmt = new
CancelAlterTableStmt(ShowAlterStmt.AlterType.COLUMN, tableName);
+ cancelAlterTableStmt.analyze(analyzer);
+ schemaChangeHandler.cancel(cancelAlterTableStmt);
+
+ schemaChangeHandler.runAfterCatalogReady();
+ Assert.assertEquals(AlterJobV2.JobState.CANCELLED,
jobV2.getJobState());
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java
index b87ae340f1b..77693efa6e4 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/alter/SchemaChangeHandlerTest.java
@@ -850,9 +850,8 @@ public class SchemaChangeHandlerTest extends
TestWithFeService {
waitAlterJobDone(alterJobs);
String buildNgramBfIndexStmtStr = "BUILD INDEX idx_error_msg on
test.sc_dup ";
- AlterTableStmt buildNgramBfIndexStmt = (AlterTableStmt)
parseAndAnalyzeStmt(buildNgramBfIndexStmtStr);
-
org.junit.jupiter.api.Assertions.assertThrows(org.apache.doris.common.DdlException.class,
- () ->
Env.getCurrentEnv().getAlterInstance().processAlterTable(buildNgramBfIndexStmt));
+
Assertions.assertThrows(org.apache.doris.common.AnalysisException.class,
+ () -> parseAndAnalyzeStmt(buildNgramBfIndexStmtStr));
tbl.readLock();
try {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java
index 590b6563e11..b16d3d15cf7 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java
@@ -22,7 +22,8 @@ import org.apache.doris.analysis.PartitionValue;
import org.apache.doris.analysis.SinglePartitionDesc;
import org.apache.doris.catalog.MaterializedIndex.IndexExtState;
import org.apache.doris.catalog.MaterializedIndex.IndexState;
-import org.apache.doris.catalog.Replica.ReplicaState;
+import org.apache.doris.cloud.catalog.CloudReplica;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.persist.EditLog;
@@ -175,13 +176,26 @@ public class CatalogTestUtil {
long version) {
Env.getCurrentInvertedIndex().clear();
- // replica
- Replica replica1 = new Replica(testReplicaId1, testBackendId1,
version, 0, 0L, 0L, 0L,
- ReplicaState.NORMAL, -1, 0);
- Replica replica2 = new Replica(testReplicaId2, testBackendId2,
version, 0, 0L, 0L, 0L,
- ReplicaState.NORMAL, -1, 0);
- Replica replica3 = new Replica(testReplicaId3, testBackendId3,
version, 0, 0L, 0L, 0L,
- ReplicaState.NORMAL, -1, 0);
+ Replica replica1;
+ Replica replica2;
+ Replica replica3;
+ if (Config.isCloudMode()) {
+ // In cloud mode we must create CloudReplica instances to avoid
ClassCastException
+ replica1 = new CloudReplica(testReplicaId1, testBackendId1,
Replica.ReplicaState.NORMAL, version,
+ /*schemaHash*/ 0, dbId, tableId, partitionId, indexId,
/*idx*/ 0);
+ replica2 = new CloudReplica(testReplicaId2, testBackendId2,
Replica.ReplicaState.NORMAL, version,
+ 0, dbId, tableId, partitionId, indexId, 1);
+ replica3 = new CloudReplica(testReplicaId3, testBackendId3,
Replica.ReplicaState.NORMAL, version,
+ 0, dbId, tableId, partitionId, indexId, 2);
+ } else {
+ replica1 = new Replica(testReplicaId1, testBackendId1, version, 0,
0L, 0L, 0L,
+ Replica.ReplicaState.NORMAL, -1, 0);
+ replica2 = new Replica(testReplicaId2, testBackendId2, version, 0,
0L, 0L, 0L,
+ Replica.ReplicaState.NORMAL, -1, 0);
+ replica3 = new Replica(testReplicaId3, testBackendId3, version, 0,
0L, 0L, 0L,
+ Replica.ReplicaState.NORMAL, -1, 0);
+ }
+
// tablet
Tablet tablet = new Tablet(tabletId);
@@ -244,10 +258,14 @@ public class CatalogTestUtil {
}
public static void createDupTable(Database db) {
-
- // replica
- Replica replica = new Replica(testReplicaId4, testBackendId1,
testStartVersion, 0, 0L, 0L, 0L,
- ReplicaState.NORMAL, -1, 0);
+ Replica replica;
+ if (Config.isCloudMode()) {
+ replica = new CloudReplica(testReplicaId4, testBackendId1,
Replica.ReplicaState.NORMAL, testStartVersion,
+ 0, db.getId(), testTableId2, testPartitionId2,
testIndexId2, 0);
+ } else {
+ replica = new Replica(testReplicaId4, testBackendId1,
testStartVersion, 0, 0L, 0L, 0L,
+ Replica.ReplicaState.NORMAL, -1, 0);
+ }
// tablet
Tablet tablet = new Tablet(testTabletId2);
diff --git
a/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out
b/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out
new file mode 100644
index 00000000000..49570b96d29
Binary files /dev/null and
b/regression-test/data/index_p0/test_ngram_bloomfilter_index_change.out differ
diff --git a/regression-test/pipeline/cloud_p1/conf/fe_custom.conf
b/regression-test/pipeline/cloud_p1/conf/fe_custom.conf
index 0e43d458f38..5bba5437f73 100644
--- a/regression-test/pipeline/cloud_p1/conf/fe_custom.conf
+++ b/regression-test/pipeline/cloud_p1/conf/fe_custom.conf
@@ -34,6 +34,5 @@ cloud_http_port=18030
meta_service_endpoint=127.0.0.1:5000
arrow_flight_sql_port = 8081
cloud_unique_id=cloud_unique_id_sql_server00
-enable_light_index_change=false
enable_advance_next_id = true
enable_job_schedule_second_for_test = true
\ No newline at end of file
diff --git
a/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy
b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy
new file mode 100644
index 00000000000..402f72ddc36
--- /dev/null
+++ b/regression-test/suites/index_p0/test_ngram_bloomfilter_index_change.groovy
@@ -0,0 +1,325 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+import groovy.json.JsonSlurper
+
+suite("test_ngram_bloomfilter_index_change") {
+ def tableName = 'test_ngram_bloomfilter_index_change'
+ def timeout = 60000
+ def delta_time = 1000
+ def alter_res = "null"
+ def useTime = 0
+
+ def wait_for_latest_op_on_table_finish = { table_name, OpTimeout ->
+ for(int t = delta_time; t <= OpTimeout; t += delta_time){
+ alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName =
"${table_name}" ORDER BY CreateTime DESC LIMIT 1;"""
+ alter_res = alter_res.toString()
+ if(alter_res.contains("FINISHED")) {
+ sleep(3000) // wait change table state to normal
+ logger.info(table_name + " latest alter job finished, detail:
" + alter_res)
+ break
+ }
+ useTime = t
+ sleep(delta_time)
+ }
+ assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish
timeout")
+ }
+
+ // Function to insert test data batch
+ def insertTestData = { ->
+ // insert 10 records
+ sql "INSERT INTO ${tableName} VALUES (1001, '2023-10-06 15:00:00',
'Laptop', 'John Smith', 199.99, 'North');"
+ sql "INSERT INTO ${tableName} VALUES (1002, '2023-10-09 17:05:00',
'Smartphone', 'Emily Johnson', 299.99, 'South');"
+ sql "INSERT INTO ${tableName} VALUES (1003, '2023-10-12 19:10:00',
'Headphones', 'Michael Brown', 399.99, 'East');"
+ sql "INSERT INTO ${tableName} VALUES (1004, '2023-10-15 21:15:00',
'Monitor', 'Jessica Davis', 499.99, 'West');"
+ sql "INSERT INTO ${tableName} VALUES (1005, '2023-10-18 23:20:00',
'Keyboard', 'David Wilson', 89.99, 'North');"
+ sql "INSERT INTO ${tableName} VALUES (1006, '2023-10-21 07:25:00',
'Mouse', 'Sarah Taylor', 699.99, 'South');"
+ sql "INSERT INTO ${tableName} VALUES (1007, '2023-10-24 09:30:00',
'Printer', 'Thomas Anderson', 799.99, 'East');"
+ sql "INSERT INTO ${tableName} VALUES (1008, '2023-10-27 11:35:00',
'Speaker', 'Jennifer Martin', 899.99, 'West');"
+ sql "INSERT INTO ${tableName} VALUES (1009, '2023-10-02 13:40:00',
'External SSD', 'Robert Clark', 999.99, 'North');"
+ sql "INSERT INTO ${tableName} VALUES (1010, '2023-10-05 15:45:00',
'Webcam', 'Amanda Lewis', 89.99, 'South');"
+ sql "sync"
+ }
+
+ // Test settings
+ sql "set enable_function_pushdown=true"
+ sql "set enable_profile=true"
+ sql "set profile_level=2"
+
+ // Define test query
+ def query = "SELECT /*+SET_VAR(enable_function_pushdown = true,
enable_profile = true, profile_level = 2)*/ * FROM ${tableName} WHERE
customer_name LIKE '%xxxx%' ORDER BY sale_id"
+ // Test Case 1: Test with enable_add_index_for_new_data = true
+ logger.info("=== Test Case 1: enable_add_index_for_new_data = true ===")
+ // Create table
+ sql "DROP TABLE IF EXISTS ${tableName}"
+ sql """
+ CREATE TABLE ${tableName} (
+ `sale_id` int NULL,
+ `sale_date` datetime NULL,
+ `product_name` varchar(100) NULL,
+ `customer_name` varchar(100) NULL,
+ `amount` decimal(10,2) NULL,
+ `region` char(50) NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`sale_id`)
+ PARTITION BY RANGE(`sale_date`) (
+ PARTITION p202310 VALUES [('2023-10-01 00:00:00'), ('2023-11-01
00:00:00')),
+ PARTITION p202311 VALUES [('2023-11-01 00:00:00'), ('2023-12-01
00:00:00')),
+ PARTITION p202312 VALUES [('2023-12-01 00:00:00'), ('2024-01-01 00:00:00'))
+ )
+ DISTRIBUTED BY HASH(`sale_id`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "storage_format" = "V2",
+ "light_schema_change" = "true",
+ "disable_auto_compaction" = "true"
+ );
+ """
+
+ // Insert test data
+ insertTestData()
+ // Verify data loaded correctly
+ qt_select_light_mode_init "SELECT * FROM ${tableName} ORDER BY sale_id"
+
+ // Test without NGRAM Bloom Filter index
+ profile("sql_select_like_without_ngram_index_light_mode") {
+ run {
+ sql "/* sql_select_like_without_ngram_index_light_mode */ ${query}"
+ sleep(1000)
+ }
+
+ check { profileString, exception ->
+ log.info(profileString)
+ assertTrue(profileString.contains("RowsBloomFilterFiltered: 0"))
+ }
+ }
+ sql "set enable_add_index_for_new_data = true"
+
+ // Add NGRAM Bloom Filter index (should be immediate in light mode)
+ sql "ALTER TABLE ${tableName} ADD INDEX
idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' =
'1024', 'gram_size' = '3');"
+
+ // In light mode, the index should be effective immediately, no need to
wait for alter job
+ // But let's give it a moment to ensure metadata is updated
+ sleep(2000)
+
+ // Insert more data after index added
+ insertTestData()
+ // Verify more data loaded correctly
+ qt_select_light_mode_more_data "SELECT * FROM ${tableName} ORDER BY
sale_id"
+
+ // Test with more data (should still filter correctly)
+ profile("sql_select_like_with_ngram_index_light_mode_more_data") {
+ run {
+ sql "/* sql_select_like_with_ngram_index_light_mode_more_data */
${query}"
+ sleep(1000)
+ }
+
+ check { profileString, exception ->
+ log.info(profileString)
+ assertTrue(profileString.contains("RowsBloomFilterFiltered: 10"))
+ }
+ }
+
+ // Drop index
+ sql "DROP INDEX idx_ngram_customer_name ON ${tableName};"
+ wait_for_latest_op_on_table_finish(tableName, timeout)
+
+ // Test after dropping index
+ profile("sql_select_like_with_ngram_index_light_mode_dropped") {
+ run {
+ sql "/* sql_select_like_with_ngram_index_light_mode_dropped */
${query}"
+ sleep(1000)
+ }
+
+ check { profileString, exception ->
+ log.info(profileString)
+ assertTrue(profileString.contains("RowsBloomFilterFiltered: 0"))
+ }
+ }
+
+ // Test Case 2: Test with enable_add_index_for_new_data = false (schema
change mode)
+ logger.info("=== Test Case 2: enable_add_index_for_new_data = false ===")
+ // Set enable_add_index_for_new_data = false
+ sql "set enable_add_index_for_new_data = false"
+ // Create new table
+ sql "DROP TABLE IF EXISTS ${tableName}"
+ sql """
+ CREATE TABLE ${tableName} (
+ `sale_id` int NULL,
+ `sale_date` datetime NULL,
+ `product_name` varchar(100) NULL,
+ `customer_name` varchar(100) NULL,
+ `amount` decimal(10,2) NULL,
+ `region` char(50) NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`sale_id`)
+ PARTITION BY RANGE(`sale_date`) (
+ PARTITION p202310 VALUES [('2023-10-01 00:00:00'), ('2023-11-01
00:00:00')),
+ PARTITION p202311 VALUES [('2023-11-01 00:00:00'), ('2023-12-01
00:00:00')),
+ PARTITION p202312 VALUES [('2023-12-01 00:00:00'), ('2024-01-01 00:00:00'))
+ )
+ DISTRIBUTED BY HASH(`sale_id`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "storage_format" = "V2",
+ "light_schema_change" = "true",
+ "disable_auto_compaction" = "true"
+ );
+ """
+ // Insert test data
+ insertTestData()
+ // Verify data loaded correctly
+ qt_select_schema_change_mode_init "SELECT * FROM ${tableName} ORDER BY
sale_id"
+
+ // Test without NGRAM Bloom Filter index
+ profile("sql_select_like_without_ngram_index_schema_change_mode") {
+ run {
+ sql "/* sql_select_like_without_ngram_index_schema_change_mode */
${query}"
+ sleep(1000)
+ }
+
+ check { profileString, exception ->
+ log.info(profileString)
+ assertTrue(profileString.contains("RowsBloomFilterFiltered: 0"))
+ }
+ }
+
+ // Add NGRAM Bloom Filter index (will trigger schema change in this mode)
+ sql "ALTER TABLE ${tableName} ADD INDEX
idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' =
'1024', 'gram_size' = '3');"
+ wait_for_latest_op_on_table_finish(tableName, timeout)
+
+ // Test after adding NGRAM Bloom Filter index (should filter existing data)
+ profile("sql_select_like_with_ngram_index_schema_change_mode_added") {
+ run {
+ sql "/* sql_select_like_with_ngram_index_schema_change_mode_added
*/ ${query}"
+ sleep(1000)
+ }
+
+ check { profileString, exception ->
+ log.info(profileString)
+ assertTrue(profileString.contains("RowsBloomFilterFiltered: 10"))
+ }
+ }
+
+ // Insert more data after index is built
+ insertTestData()
+ // Verify more data loaded correctly
+ qt_select_schema_change_mode_more_data "SELECT * FROM ${tableName} ORDER
BY sale_id"
+
+ // Test with more data (should filter all data)
+ profile("sql_select_like_with_ngram_index_schema_change_mode_more_data") {
+ run {
+ sql "/*
sql_select_like_with_ngram_index_schema_change_mode_more_data */ ${query}"
+ sleep(1000)
+ }
+
+ check { profileString, exception ->
+ log.info(profileString)
+ assertTrue(profileString.contains("RowsBloomFilterFiltered: 20"))
+ }
+ }
+
+ // Drop index
+ sql "DROP INDEX idx_ngram_customer_name ON ${tableName};"
+ wait_for_latest_op_on_table_finish(tableName, timeout)
+
+ // Test after dropping index
+ profile("sql_select_like_with_ngram_index_schema_change_mode_dropped") {
+ run {
+ sql "/*
sql_select_like_with_ngram_index_schema_change_mode_dropped */ ${query}"
+ sleep(1000)
+ }
+
+ check { profileString, exception ->
+ log.info(profileString)
+ assertTrue(profileString.contains("RowsBloomFilterFiltered: 0"))
+ }
+ }
+
+ // Test Case 3: Test different scenarios for index lifecycle
+ logger.info("=== Test Case 3: Index lifecycle with
enable_add_index_for_new_data = true ===")
+ // Set enable_add_index_for_new_data = true
+ sql "set enable_add_index_for_new_data = true"
+ // Create table and add index before inserting data
+ sql "DROP TABLE IF EXISTS ${tableName}"
+ sql """
+ CREATE TABLE ${tableName} (
+ `sale_id` int NULL,
+ `sale_date` datetime NULL,
+ `product_name` varchar(100) NULL,
+ `customer_name` varchar(100) NULL,
+ `amount` decimal(10,2) NULL,
+ `region` char(50) NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`sale_id`)
+ PARTITION BY RANGE(`sale_date`) (
+ PARTITION p202310 VALUES [('2023-10-01 00:00:00'), ('2023-11-01
00:00:00')),
+ PARTITION p202311 VALUES [('2023-11-01 00:00:00'), ('2023-12-01
00:00:00')),
+ PARTITION p202312 VALUES [('2023-12-01 00:00:00'), ('2024-01-01 00:00:00'))
+ )
+ DISTRIBUTED BY HASH(`sale_id`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "storage_format" = "V2",
+ "light_schema_change" = "true",
+ "disable_auto_compaction" = "true"
+ );
+ """
+
+ // Add ngram bf index before data insertion
+ sql "ALTER TABLE ${tableName} ADD INDEX
idx_ngram_customer_name(customer_name) USING NGRAM_BF PROPERTIES('bf_size' =
'1024', 'gram_size' = '3');"
+ wait_for_latest_op_on_table_finish(tableName, timeout)
+
+ // Insert data after index creation
+ insertTestData()
+ // Verify data loaded correctly
+ qt_select_lifecycle_after_data "SELECT * FROM ${tableName} ORDER BY
sale_id"
+
+ // Test filtering with index added before data insertion
+ profile("sql_select_like_with_ngram_index_lifecycle_test") {
+ run {
+ sql "/* sql_select_like_with_ngram_index_lifecycle_test */
${query}"
+ sleep(1000)
+ }
+
+ check { profileString, exception ->
+ log.info(profileString)
+ assertTrue(profileString.contains("RowsBloomFilterFiltered: 10"))
+ }
+ }
+
+ // Insert more data
+ insertTestData()
+ // Verify more data loaded correctly
+ qt_select_lifecycle_final "SELECT * FROM ${tableName} ORDER BY sale_id"
+
+ // Test filtering with more data
+ profile("sql_select_like_with_ngram_index_lifecycle_final") {
+ run {
+ sql "/* sql_select_like_with_ngram_index_lifecycle_final */
${query}"
+ sleep(1000)
+ }
+
+ check { profileString, exception ->
+ log.info(profileString)
+ assertTrue(profileString.contains("RowsBloomFilterFiltered: 20"))
+ }
+ }
+
+ // Final cleanup
+ sql "DROP INDEX idx_ngram_customer_name ON ${tableName};"
+ sleep(2000)
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
