This is an automated email from the ASF dual-hosted git repository.
luwei16 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 42f8f2e5cdc [improvement](ddl) Restrict state types to aggregate
tables (#66963)
42f8f2e5cdc is described below
commit 42f8f2e5cdc185beb1f5442e7f24085b4f13928d
Author: Luwei <[email protected]>
AuthorDate: Thu Sep 24 14:26:29 2026 +0800
[improvement](ddl) Restrict state types to aggregate tables (#66963)
### What problem does this PR solve?
Issue Number: close #65865
Related PR: #66664
Problem Summary: HLL, QUANTILE_STATE, and AGG_STATE columns in
non-aggregate table models can enter row conversion paths that do not
support nullable aggregate-state values and fail at runtime. Reject new
definitions of these types in Duplicate Key and Unique Key tables by
default, while retaining a temporary FE compatibility switch for
migration. Existing Aggregate Key tables remain supported.
### Release note
HLL, QUANTILE_STATE, and AGG_STATE columns are restricted to Aggregate
Key tables by default. The temporary FE config
`enable_non_aggregate_table_state_types=true` can restore the previous
definition behavior during migration and will be removed after the
compatibility transition period.
### Check List (For Author)
- Test: Unit Test / Build
- `./run-fe-ut.sh --run
org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinitionTest`
- `./build.sh --fe`
- `./build.sh --be`
- Regression test added but not run because an isolated cluster rebuild
requires separate destructive-action approval
- Behavior changed: Yes. New HLL, QUANTILE_STATE, and AGG_STATE columns
are rejected in non-Aggregate Key tables by default
- Does this need documentation: No. The compatibility switch is
temporary and carries its removal notice in the FE config description
---
.../main/java/org/apache/doris/common/Config.java | 5 +
.../java/org/apache/doris/mtmv/MTMVPlanUtil.java | 2 +-
.../plans/commands/info/ColumnDefinition.java | 24 ++++-
.../trees/plans/commands/info/CreateTableInfo.java | 4 +-
.../doris/alter/InternalSchemaAlterTest.java | 10 ++
.../org/apache/doris/catalog/CreateTableTest.java | 28 +++++
.../CreateTableWithBloomFilterIndexTest.java | 31 +++---
.../org/apache/doris/mtmv/MTMVPlanUtilTest.java | 16 +++
.../plans/commands/info/ColumnDefinitionTest.java | 93 +++++++++++++++++
.../suites/correctness_p0/test_default_hll.groovy | 6 +-
.../duplicate/storage/test_duplicate_hll.groovy | 4 +
.../storage/test_duplicate_quantile_state.groovy | 4 +
...test_state_types_only_in_aggregate_table.groovy | 113 +++++++++++++++++++++
.../data_model_p0/unique/test_unique_hll.groovy | 4 +
.../unique/test_unique_quantile_state.groovy | 4 +
.../test_remote_doris_unique_table_select.groovy | 4 +
.../mv_p0/mv_negative/dup_negative_test.groovy | 4 +
.../mv_p0/mv_negative/mor_negative_test.groovy | 4 +
.../mv_p0/mv_negative/mow_negative_test.groovy | 4 +
.../support_type/any_value/any_value.groovy | 6 +-
.../suites/query_p0/join/test_join_on.groovy | 4 +
21 files changed, 355 insertions(+), 19 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 1704a3fa252..8fcf7661eb5 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1859,6 +1859,11 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, masterOnly = true)
public static boolean enable_quantile_state_type = true;
+ @ConfField(mutable = true, masterOnly = true, description = "Temporary
compatibility switch that allows HLL, "
+ + "QUANTILE_STATE, and AGG_STATE columns in non-aggregate key
tables. Disabled by default. This switch "
+ + "is intended only for migration and will be removed after the
compatibility transition period.")
+ public static boolean enable_non_aggregate_table_state_types = false;
+
/*---------------------- JOB CONFIG START------------------------*/
/**
* The number of threads used to dispatch timer job.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java
index 45c528449a8..b937c93065a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java
@@ -698,7 +698,7 @@ public class MTMVPlanUtil {
if (col.getType().isVarBinaryType()) {
throw new AnalysisException("MTMV do not support varbinary
type : " + col.getName());
}
- col.validate(true, keysSet, Sets.newHashSet(),
finalEnableMergeOnWrite, keysType);
+ col.validate(true, keysSet, Sets.newHashSet(),
finalEnableMergeOnWrite, keysType, true);
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
index f10d9c517a0..63b3713fde4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
@@ -24,6 +24,7 @@ import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.common.CaseSensibility;
+import org.apache.doris.common.Config;
import org.apache.doris.common.FeNameFormat;
import org.apache.doris.common.util.SqlUtils;
import org.apache.doris.nereids.exceptions.AnalysisException;
@@ -311,6 +312,10 @@ public class ColumnDefinition {
return sb.toString();
}
+ private boolean isAggregateTableOnlyType() {
+ return type.isHllType() || type.isQuantileStateType() ||
type.isAggStateType();
+ }
+
private DataType updateCharacterTypeLength(DataType dataType) {
if (dataType instanceof ArrayType) {
return ArrayType.of(updateCharacterTypeLength(((ArrayType)
dataType).getItemType()));
@@ -384,7 +389,13 @@ public class ColumnDefinition {
*/
public void validate(boolean isOlap, Set<String> keysSet, Set<String>
clusterKeySet, boolean isEnableMergeOnWrite,
KeysType keysType) {
- validateInternal(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite,
keysType, false);
+ validate(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite,
keysType, false);
+ }
+
+ public void validate(boolean isOlap, Set<String> keysSet, Set<String>
clusterKeySet, boolean isEnableMergeOnWrite,
+ KeysType keysType, boolean isSystemGeneratedTable) {
+ validateInternal(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite,
keysType, false,
+ isSystemGeneratedTable);
}
/**
@@ -392,11 +403,11 @@ public class ColumnDefinition {
*/
public void validateNestedColumn(boolean isOlap, Set<String> keysSet,
Set<String> clusterKeySet,
boolean isEnableMergeOnWrite, KeysType keysType) {
- validateInternal(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite,
keysType, true);
+ validateInternal(isOlap, keysSet, clusterKeySet, isEnableMergeOnWrite,
keysType, true, false);
}
private void validateInternal(boolean isOlap, Set<String> keysSet,
Set<String> clusterKeySet,
- boolean isEnableMergeOnWrite, KeysType keysType, boolean
nestedColumn) {
+ boolean isEnableMergeOnWrite, KeysType keysType, boolean
nestedColumn, boolean isSystemGeneratedTable) {
try {
// if enableAddHiddenColumn is true, can add hidden column.
// So does not check if the column name starts with __DORIS_
@@ -414,6 +425,13 @@ public class ColumnDefinition {
}
type.validateDataType();
type = updateCharacterTypeLength(type);
+ if (!isSystemGeneratedTable && isOlap && keysType != KeysType.AGG_KEYS
&& isAggregateTableOnlyType()
+ && !Config.enable_non_aggregate_table_state_types) {
+ throw new AnalysisException(String.format(
+ "%s type is only supported in aggregate key tables,
column: %s. "
+ + "Set FE config
'enable_non_aggregate_table_state_types' to true to temporarily allow it",
+ type.toSql(), name));
+ }
if (type.isArrayType()) {
int depth = 0;
DataType curType = type;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
index 8307f965bf8..b32ad6af120 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
@@ -726,8 +726,10 @@ public class CreateTableInfo {
keysSet.addAll(keys);
Set<String> orderKeySet =
Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
orderKeySet.addAll(sortOrderFields.stream().map(SortFieldInfo::getColumnName).collect(Collectors.toSet()));
+ // Internal statistics tables need state columns. The internal-query
flag can also be set by user SHOWs.
+ boolean isSystemGeneratedTable = targetIsInternalCatalog &&
FeConstants.INTERNAL_DB_NAME.equals(dbName);
columns.forEach(c -> c.validate(targetIsInternalCatalog, keysSet,
orderKeySet, finalEnableMergeOnWrite,
- keysType));
+ keysType, isSystemGeneratedTable));
try {
invertedIndexFileStorageFormat =
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/alter/InternalSchemaAlterTest.java
b/fe/fe-core/src/test/java/org/apache/doris/alter/InternalSchemaAlterTest.java
index 3ad88c49c40..af4901864d9 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/alter/InternalSchemaAlterTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/alter/InternalSchemaAlterTest.java
@@ -25,6 +25,7 @@ import org.apache.doris.catalog.InternalSchemaInitializer;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.PartitionInfo;
+import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.FeConstants;
@@ -89,4 +90,13 @@ public class InternalSchemaAlterTest extends
TestWithFeService {
Assertions.assertNotNull(table.getColumn(def.getName()));
}
}
+
+ @Test
+ public void testCheckPartitionStatisticsTable() throws AnalysisException {
+ Database db = Env.getCurrentEnv().getCatalogMgr()
+
.getInternalCatalog().getDbNullable(FeConstants.INTERNAL_DB_NAME);
+ Assertions.assertNotNull(db);
+ OlapTable table =
db.getOlapTableOrAnalysisException(StatisticConstants.PARTITION_STATISTIC_TBL_NAME);
+ Assertions.assertEquals(PrimitiveType.HLL,
table.getColumn("ndv").getType().getPrimitiveType());
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
index 5cc2899958c..37b7afd5961 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
@@ -52,6 +52,34 @@ public class CreateTableTest extends TestWithFeService {
createDatabase("test");
}
+ @Test
+ public void testInternalQueryStateDoesNotExemptUserTable() {
+ boolean originalAllowStateTypes =
Config.enable_non_aggregate_table_state_types;
+ boolean originalInternal = connectContext.getState().isInternal();
+ boolean originalEnableAggState =
connectContext.getSessionVariable().enableAggState;
+ Config.enable_non_aggregate_table_state_types = false;
+ connectContext.getSessionVariable().enableAggState = true;
+ // An ordinary SHOW can leave the internal-query flag set on a user
connection.
+ connectContext.getState().setInternal(true);
+ try {
+ for (String keysType : new String[] {"DUPLICATE", "UNIQUE"}) {
+ for (String type : new String[] {"HLL NOT NULL",
"QUANTILE_STATE NOT NULL",
+ "AGG_STATE<sum(INT NOT NULL)>"}) {
+ AnalysisException exception =
Assertions.assertThrows(AnalysisException.class,
+ () -> createTable("CREATE TABLE
test.user_state_type (k INT, v " + type + ") "
+ + keysType + " KEY(k) DISTRIBUTED BY
HASH(k) BUCKETS 1 "
+ + "PROPERTIES('replication_num'='1')"));
+ Assertions.assertTrue(exception.getMessage().contains(
+ "type is only supported in aggregate key tables"));
+ }
+ }
+ } finally {
+ Config.enable_non_aggregate_table_state_types =
originalAllowStateTypes;
+ connectContext.getState().setInternal(originalInternal);
+ connectContext.getSessionVariable().enableAggState =
originalEnableAggState;
+ }
+ }
+
@Test
public void testDuplicateCreateTable() throws Exception {
// test
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableWithBloomFilterIndexTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableWithBloomFilterIndexTest.java
index c12843cf21a..21436bb6da7 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableWithBloomFilterIndexTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableWithBloomFilterIndexTest.java
@@ -19,6 +19,7 @@ package org.apache.doris.catalog;
import org.apache.doris.alter.AlterJobV2;
import org.apache.doris.catalog.info.IndexType;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.FeConstants;
@@ -446,18 +447,24 @@ public class CreateTableWithBloomFilterIndexTest extends
TestWithFeService {
@Test
public void testCreateTableWithHllBloomFilterIndex() {
- ExceptionChecker.expectThrowsWithMsg(DdlException.class,
- " HLL is not supported in bloom filter index. invalid column:
k1",
- () -> createTable("CREATE TABLE test.tbl_hll_bf (\n"
- + "v1 INT,\n"
- + "k1 HLL\n"
- + ") ENGINE=OLAP\n"
- + "DUPLICATE KEY(v1)\n"
- + "DISTRIBUTED BY HASH(v1) BUCKETS 1\n"
- + "PROPERTIES (\n"
- + "\"bloom_filter_columns\" = \"k1\",\n"
- + "\"replication_num\" = \"1\"\n"
- + ");"));
+ boolean originalValue = Config.enable_non_aggregate_table_state_types;
+ Config.enable_non_aggregate_table_state_types = true;
+ try {
+ ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+ " HLL is not supported in bloom filter index. invalid
column: k1",
+ () -> createTable("CREATE TABLE test.tbl_hll_bf (\n"
+ + "v1 INT,\n"
+ + "k1 HLL\n"
+ + ") ENGINE=OLAP\n"
+ + "DUPLICATE KEY(v1)\n"
+ + "DISTRIBUTED BY HASH(v1) BUCKETS 1\n"
+ + "PROPERTIES (\n"
+ + "\"bloom_filter_columns\" = \"k1\",\n"
+ + "\"replication_num\" = \"1\"\n"
+ + ");"));
+ } finally {
+ Config.enable_non_aggregate_table_state_types = originalValue;
+ }
}
@Test
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java
index c2b0f83d44f..d661d2df7f3 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java
@@ -352,6 +352,22 @@ public class MTMVPlanUtilTest extends SqlTestBase {
Assertions.assertTrue(mtmvAnalyzeQueryInfo.getColumnDefinitions().size() == 2);
}
+ @Test
+ public void testCreateMTMVWithAggStateColumn() throws Exception {
+ boolean originalEnableAggState =
connectContext.getSessionVariable().enableAggState;
+ connectContext.getSessionVariable().enableAggState = true;
+ connectContext.setThreadLocalInfo();
+ try {
+ Assertions.assertDoesNotThrow(() -> createMvByNereids(
+ "create materialized view mv_with_agg_state BUILD DEFERRED
REFRESH COMPLETE ON MANUAL\n"
+ + "DISTRIBUTED BY RANDOM BUCKETS 1\n"
+ + "PROPERTIES ('replication_num' = '1')\n"
+ + "as select id, sum_union(sum_state(score)) from
test.T1 group by id"));
+ } finally {
+ connectContext.getSessionVariable().enableAggState =
originalEnableAggState;
+ }
+ }
+
@Test
public void testEnsureMTMVQueryUsable() throws Exception {
createMvByNereids("create materialized view mv1 BUILD DEFERRED REFRESH
COMPLETE ON MANUAL\n"
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java
index 6cbb4073df2..bcd54023faa 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinitionTest.java
@@ -17,13 +17,38 @@
package org.apache.doris.nereids.trees.plans.commands.info;
+import org.apache.doris.catalog.AggregateType;
+import org.apache.doris.catalog.KeysType;
+import org.apache.doris.common.Config;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.types.AggStateType;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.HllType;
+import org.apache.doris.nereids.types.IntegerType;
+import org.apache.doris.nereids.types.QuantileStateType;
import org.apache.doris.nereids.types.StringType;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import java.util.Optional;
+
public class ColumnDefinitionTest {
+ @BeforeEach
+ public void setUp() {
+ Config.enable_non_aggregate_table_state_types = false;
+ }
+
+ @AfterEach
+ public void tearDown() {
+ Config.enable_non_aggregate_table_state_types = false;
+ }
+
@Test
public void testNameEquals() {
ColumnDefinition columnDefinition = new ColumnDefinition("col1", null,
false, null, false, null, null);
@@ -43,4 +68,72 @@ public class ColumnDefinitionTest {
String sql = columnDefinition.toSql();
Assertions.assertTrue(sql.endsWith("COMMENT \"\""));
}
+
+ @Test
+ public void testStateTypesRequireAggregateKeyTableByDefault() {
+ for (KeysType keysType : ImmutableList.of(KeysType.DUP_KEYS,
KeysType.UNIQUE_KEYS)) {
+ for (DataType type : aggregateTableOnlyTypes()) {
+ ColumnDefinition column = new ColumnDefinition(
+ "v", type, false, null, false, Optional.empty(), "");
+
+ AnalysisException exception =
Assertions.assertThrows(AnalysisException.class,
+ () -> validateColumn(column, keysType));
+ Assertions.assertTrue(exception.getMessage().contains(
+ type.toSql() + " type is only supported in aggregate
key tables"));
+ }
+ }
+ }
+
+ @Test
+ public void testTemporaryConfigAllowsStateTypesInNonAggregateTable() {
+ Config.enable_non_aggregate_table_state_types = true;
+
+ for (KeysType keysType : ImmutableList.of(KeysType.DUP_KEYS,
KeysType.UNIQUE_KEYS)) {
+ for (DataType type : aggregateTableOnlyTypes()) {
+ ColumnDefinition column = new ColumnDefinition(
+ "v", type, false, null, false, Optional.empty(), "");
+ Assertions.assertDoesNotThrow(() -> validateColumn(column,
keysType));
+ }
+ }
+ }
+
+ @Test
+ public void testStateTypesRemainSupportedInAggregateKeyTable() {
+ Assertions.assertDoesNotThrow(() -> validateColumn(new
ColumnDefinition(
+ "v", HllType.INSTANCE, false, AggregateType.HLL_UNION, false,
Optional.empty(), ""),
+ KeysType.AGG_KEYS));
+ Assertions.assertDoesNotThrow(() -> validateColumn(new
ColumnDefinition(
+ "v", QuantileStateType.INSTANCE, false,
AggregateType.QUANTILE_UNION, false, Optional.empty(), ""),
+ KeysType.AGG_KEYS));
+ Assertions.assertDoesNotThrow(() -> validateColumn(new
ColumnDefinition(
+ "v", aggStateType(), false, AggregateType.GENERIC, false,
Optional.empty(), ""),
+ KeysType.AGG_KEYS));
+ }
+
+ @Test
+ public void testSystemGeneratedTableAllowsStateTypesInNonAggregateTable() {
+ for (KeysType keysType : ImmutableList.of(KeysType.DUP_KEYS,
KeysType.UNIQUE_KEYS)) {
+ for (DataType type : aggregateTableOnlyTypes()) {
+ ColumnDefinition column = new ColumnDefinition(
+ "v", type, false, null, false, Optional.empty(), "");
+ Assertions.assertDoesNotThrow(() ->
validateSystemGeneratedColumn(column, keysType));
+ }
+ }
+ }
+
+ private static ImmutableList<DataType> aggregateTableOnlyTypes() {
+ return ImmutableList.of(HllType.INSTANCE, QuantileStateType.INSTANCE,
aggStateType());
+ }
+
+ private static AggStateType aggStateType() {
+ return new AggStateType("sum", ImmutableList.of(IntegerType.INSTANCE),
ImmutableList.of(false), false);
+ }
+
+ private static void validateColumn(ColumnDefinition column, KeysType
keysType) {
+ column.validate(true, ImmutableSet.of("k"), ImmutableSet.of(), true,
keysType);
+ }
+
+ private static void validateSystemGeneratedColumn(ColumnDefinition column,
KeysType keysType) {
+ column.validate(true, ImmutableSet.of("k"), ImmutableSet.of(), true,
keysType, true);
+ }
}
diff --git a/regression-test/suites/correctness_p0/test_default_hll.groovy
b/regression-test/suites/correctness_p0/test_default_hll.groovy
index b21869e30e3..dc7c612bdf2 100644
--- a/regression-test/suites/correctness_p0/test_default_hll.groovy
+++ b/regression-test/suites/correctness_p0/test_default_hll.groovy
@@ -16,6 +16,8 @@
// under the License.
suite("test_default_hll") {
+ withGlobalLock("enable_non_aggregate_table_state_types") {
+ setFeConfigTemporary([enable_non_aggregate_table_state_types: true]) {
def tableName = "test_default_hll"
sql """ DROP TABLE IF EXISTS ${tableName} """
@@ -96,4 +98,6 @@ suite("test_default_hll") {
qt_stream_load_csv1 """ select HLL_CARDINALITY(h1) from ${tableName} order
by k; """
-}
\ No newline at end of file
+ }
+ }
+}
diff --git
a/regression-test/suites/data_model_p0/duplicate/storage/test_duplicate_hll.groovy
b/regression-test/suites/data_model_p0/duplicate/storage/test_duplicate_hll.groovy
index 0c88f276a06..3c61f7b0fa9 100644
---
a/regression-test/suites/data_model_p0/duplicate/storage/test_duplicate_hll.groovy
+++
b/regression-test/suites/data_model_p0/duplicate/storage/test_duplicate_hll.groovy
@@ -16,6 +16,8 @@
// under the License.
suite("test_duplicate_table_hll") {
+ withGlobalLock("enable_non_aggregate_table_state_types") {
+ setFeConfigTemporary([enable_non_aggregate_table_state_types: true]) {
sql "sync;"
@@ -68,4 +70,6 @@ suite("test_duplicate_table_hll") {
DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num"
= "1"); """
exception "Key column can not set complex type:k"
}
+ }
+ }
}
diff --git
a/regression-test/suites/data_model_p0/duplicate/storage/test_duplicate_quantile_state.groovy
b/regression-test/suites/data_model_p0/duplicate/storage/test_duplicate_quantile_state.groovy
index 9c4e07094b6..1715bbacbdd 100644
---
a/regression-test/suites/data_model_p0/duplicate/storage/test_duplicate_quantile_state.groovy
+++
b/regression-test/suites/data_model_p0/duplicate/storage/test_duplicate_quantile_state.groovy
@@ -16,6 +16,8 @@
// under the License.
suite("test_duplicate_table_quantile_state") {
+ withGlobalLock("enable_non_aggregate_table_state_types") {
+ setFeConfigTemporary([enable_non_aggregate_table_state_types: true]) {
sql "sync;"
@@ -64,4 +66,6 @@ suite("test_duplicate_table_quantile_state") {
DISTRIBUTED BY HASH(k) BUCKETS 1 properties("replication_num"
= "1"); """
exception "Key column can not set complex type:k"
}
+ }
+ }
}
diff --git
a/regression-test/suites/data_model_p0/test_state_types_only_in_aggregate_table.groovy
b/regression-test/suites/data_model_p0/test_state_types_only_in_aggregate_table.groovy
new file mode 100644
index 00000000000..f74f026e7c3
--- /dev/null
+++
b/regression-test/suites/data_model_p0/test_state_types_only_in_aggregate_table.groovy
@@ -0,0 +1,113 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_state_types_only_in_aggregate_table") {
+ context.reconnectToMasterFe()
+ withGlobalLock("enable_non_aggregate_table_state_types") {
+ setFeConfigTemporary([enable_non_aggregate_table_state_types: false]) {
+ sql "set enable_agg_state=true"
+ // SHOW executes an internal query, but must not exempt subsequent
user DDL.
+ sql "show table status"
+
+ sql "drop table if exists state_type_dup_hll"
+ test {
+ sql """
+ create table state_type_dup_hll (
+ k int,
+ v hll not null
+ ) duplicate key(k)
+ distributed by hash(k) buckets 1
+ properties("replication_num" = "1")
+ """
+ exception "type is only supported in aggregate key tables"
+ }
+
+ sql "drop table if exists state_type_unique_quantile"
+ test {
+ sql """
+ create table state_type_unique_quantile (
+ k int,
+ v quantile_state not null
+ ) unique key(k)
+ distributed by hash(k) buckets 1
+ properties("replication_num" = "1")
+ """
+ exception "type is only supported in aggregate key tables"
+ }
+
+ sql "drop table if exists state_type_dup_agg_state"
+ test {
+ sql """
+ create table state_type_dup_agg_state (
+ k int,
+ v agg_state<sum(int not null)> generic
+ ) duplicate key(k)
+ distributed by hash(k) buckets 1
+ properties("replication_num" = "1")
+ """
+ exception "DUP_KEYS table should not specify aggregate type"
+ }
+
+ sql "drop table if exists state_type_alter_dup"
+ sql """
+ create table state_type_alter_dup (
+ k int,
+ v int
+ ) duplicate key(k)
+ distributed by hash(k) buckets 1
+ properties("replication_num" = "1")
+ """
+ test {
+ sql "alter table state_type_alter_dup add column h hll not
null"
+ exception "type is only supported in aggregate key tables"
+ }
+ test {
+ sql "alter table state_type_alter_dup add column q
quantile_state not null"
+ exception "type is only supported in aggregate key tables"
+ }
+ test {
+ sql "alter table state_type_alter_dup add column a
agg_state<sum(int not null)> generic"
+ exception "type is only supported in aggregate key tables"
+ }
+
+ sql "drop table if exists state_type_aggregate"
+ sql """
+ create table state_type_aggregate (
+ k int,
+ h hll hll_union not null,
+ q quantile_state quantile_union not null,
+ a agg_state<sum(int not null)> generic
+ ) aggregate key(k)
+ distributed by hash(k) buckets 1
+ properties("replication_num" = "1")
+ """
+
+ setFeConfigTemporary([enable_non_aggregate_table_state_types:
true]) {
+ sql "drop table if exists state_type_compatibility_dup"
+ sql """
+ create table state_type_compatibility_dup (
+ k int,
+ h hll not null,
+ q quantile_state not null
+ ) duplicate key(k)
+ distributed by hash(k) buckets 1
+ properties("replication_num" = "1")
+ """
+ }
+ }
+ }
+}
diff --git a/regression-test/suites/data_model_p0/unique/test_unique_hll.groovy
b/regression-test/suites/data_model_p0/unique/test_unique_hll.groovy
index 035f6b1cb37..a26e266abd2 100644
--- a/regression-test/suites/data_model_p0/unique/test_unique_hll.groovy
+++ b/regression-test/suites/data_model_p0/unique/test_unique_hll.groovy
@@ -16,6 +16,8 @@
// under the License.
suite("test_unique_table_hll") {
+ withGlobalLock("enable_non_aggregate_table_state_types") {
+ setFeConfigTemporary([enable_non_aggregate_table_state_types: true]) {
for (def enable_mow : [true, false]) {
sql "sync;"
@@ -70,4 +72,6 @@ suite("test_unique_table_hll") {
exception "Key column can not set complex type:k"
}
}
+ }
+ }
}
diff --git
a/regression-test/suites/data_model_p0/unique/test_unique_quantile_state.groovy
b/regression-test/suites/data_model_p0/unique/test_unique_quantile_state.groovy
index 9f2b2a5475a..70d23a34b17 100644
---
a/regression-test/suites/data_model_p0/unique/test_unique_quantile_state.groovy
+++
b/regression-test/suites/data_model_p0/unique/test_unique_quantile_state.groovy
@@ -16,6 +16,8 @@
// under the License.
suite("test_unique_table_quantile_state") {
+ withGlobalLock("enable_non_aggregate_table_state_types") {
+ setFeConfigTemporary([enable_non_aggregate_table_state_types: true]) {
for (def enable_mow : [true, false]) {
sql "sync;"
@@ -66,4 +68,6 @@ suite("test_unique_table_quantile_state") {
exception "Key column can not set complex type:k"
}
}
+ }
+ }
}
diff --git
a/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_unique_table_select.groovy
b/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_unique_table_select.groovy
index 768deb9c81b..0d6a00026f6 100644
---
a/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_unique_table_select.groovy
+++
b/regression-test/suites/external_table_p0/remote_doris/test_remote_doris_unique_table_select.groovy
@@ -16,6 +16,8 @@
// under the License.
suite("test_remote_doris_unique_table_select", "p0,external") {
+ withGlobalLock("enable_non_aggregate_table_state_types") {
+ setFeConfigTemporary([enable_non_aggregate_table_state_types: true]) {
String remote_doris_host =
context.config.otherConfigs.get("extArrowFlightSqlHost")
String remote_doris_arrow_port =
context.config.otherConfigs.get("extArrowFlightSqlPort")
String remote_doris_http_port =
context.config.otherConfigs.get("extArrowFlightHttpPort")
@@ -235,4 +237,6 @@ suite("test_remote_doris_unique_table_select",
"p0,external") {
sql """ DROP DATABASE IF EXISTS `${db_name}` """
sql """ DROP CATALOG IF EXISTS `${catalog_name}` """
+ }
+ }
}
diff --git a/regression-test/suites/mv_p0/mv_negative/dup_negative_test.groovy
b/regression-test/suites/mv_p0/mv_negative/dup_negative_test.groovy
index 446954d6fdb..149cee3568f 100644
--- a/regression-test/suites/mv_p0/mv_negative/dup_negative_test.groovy
+++ b/regression-test/suites/mv_p0/mv_negative/dup_negative_test.groovy
@@ -16,6 +16,8 @@
// under the License.
suite("dup_negative_mv_test", "mv_negative") {
+ withGlobalLock("enable_non_aggregate_table_state_types") {
+ setFeConfigTemporary([enable_non_aggregate_table_state_types: true]) {
// this mv rewrite would not be rewritten in RBO phase, so set TRY_IN_RBO
explicitly to make case stable
sql "set pre_materialized_view_rewrite_strategy = TRY_IN_RBO"
@@ -153,4 +155,6 @@ suite("dup_negative_mv_test", "mv_negative") {
}
+ }
+ }
}
diff --git a/regression-test/suites/mv_p0/mv_negative/mor_negative_test.groovy
b/regression-test/suites/mv_p0/mv_negative/mor_negative_test.groovy
index 5cd3264d6a7..e806507e039 100644
--- a/regression-test/suites/mv_p0/mv_negative/mor_negative_test.groovy
+++ b/regression-test/suites/mv_p0/mv_negative/mor_negative_test.groovy
@@ -16,6 +16,8 @@
// under the License.
suite("mor_negative_mv_test", "mv_negative") {
+ withGlobalLock("enable_non_aggregate_table_state_types") {
+ setFeConfigTemporary([enable_non_aggregate_table_state_types: true]) {
String db = context.config.getDbNameByFile(context.file)
def prefix_str = "mv_mor_negative"
@@ -157,4 +159,6 @@ suite("mor_negative_mv_test", "mv_negative") {
}
+ }
+ }
}
diff --git a/regression-test/suites/mv_p0/mv_negative/mow_negative_test.groovy
b/regression-test/suites/mv_p0/mv_negative/mow_negative_test.groovy
index 760614a2038..7d598e3aabf 100644
--- a/regression-test/suites/mv_p0/mv_negative/mow_negative_test.groovy
+++ b/regression-test/suites/mv_p0/mv_negative/mow_negative_test.groovy
@@ -16,6 +16,8 @@
// under the License.
suite("mow_negative_mv_test", "mv_negative") {
+ withGlobalLock("enable_non_aggregate_table_state_types") {
+ setFeConfigTemporary([enable_non_aggregate_table_state_types: true]) {
String db = context.config.getDbNameByFile(context.file)
def prefix_str = "mv_mow_negative"
@@ -158,4 +160,6 @@ suite("mow_negative_mv_test", "mv_negative") {
}
+ }
+ }
}
diff --git
a/regression-test/suites/query_p0/aggregate/support_type/any_value/any_value.groovy
b/regression-test/suites/query_p0/aggregate/support_type/any_value/any_value.groovy
index 68df1a9b57d..b43ae350e66 100644
---
a/regression-test/suites/query_p0/aggregate/support_type/any_value/any_value.groovy
+++
b/regression-test/suites/query_p0/aggregate/support_type/any_value/any_value.groovy
@@ -16,6 +16,8 @@
// under the License.
suite("any_value") {
+ withGlobalLock("enable_non_aggregate_table_state_types") {
+ setFeConfigTemporary([enable_non_aggregate_table_state_types: true]) {
sql "set enable_decimal256 = true;"
sql """
drop table if exists d_table;
@@ -102,4 +104,6 @@ suite("any_value") {
qt_sql_bitmap """select bitmap_to_string(any_value(col_bitmap)) from
d_table;"""
qt_sql_hll """select hll_cardinality(any_value(col_hll)) from d_table;"""
qt_sql_quantile_state """select
QUANTILE_PERCENT(any_value(col_quantile_state), 0.5) from d_table;"""
-}
\ No newline at end of file
+ }
+ }
+}
diff --git a/regression-test/suites/query_p0/join/test_join_on.groovy
b/regression-test/suites/query_p0/join/test_join_on.groovy
index 042e16b1b2c..577ff414a25 100644
--- a/regression-test/suites/query_p0/join/test_join_on.groovy
+++ b/regression-test/suites/query_p0/join/test_join_on.groovy
@@ -16,6 +16,8 @@
// under the License.
suite("test_join_on", "query_p0") {
+ withGlobalLock("enable_non_aggregate_table_state_types") {
+ setFeConfigTemporary([enable_non_aggregate_table_state_types: true]) {
sql "DROP TABLE IF EXISTS join_on"
sql """
@@ -49,4 +51,6 @@ suite("test_join_on", "query_p0") {
sql """select * from (select cast('' as variant) as a) t1 join (select
cast('' as variant) as a) t2 on t1.a = t2.a"""
exception "could not used in ComparisonPredicate (a = a)"
}
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
