This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit d1acd42db4d1ae010c4e0b3d0fd033ac6884ac96
Author: amory <wangqian...@selectdb.com>
AuthorDate: Wed Jan 17 17:41:40 2024 +0800
[Improve](config)delete confused config for nested complex type (#29988)
---
.../main/java/org/apache/doris/common/Config.java | 11 -----
.../java/org/apache/doris/analysis/TypeDef.java | 11 -----
.../plans/commands/info/ColumnDefinition.java | 14 ------
...ocal_tvf_with_complex_type_insertinto_doris.out | 3 --
.../three_level_nestedtypes_with_s3data.groovy | 2 +-
.../two_level_nestedtypes_with_s3data.groovy | 1 -
.../suites/datatype_p0/nested_types/load.groovy | 2 +-
.../map_functions/test_basic_map_function.groovy | 1 -
.../query/test_nested_type_with_count.groovy | 1 -
...st_nested_types_insert_into_with_literal.groovy | 1 -
.../test_nested_types_insert_into_with_s3.groovy | 1 -
...test_nestedtypes_csv_insert_into_with_s3.groovy | 1 -
.../test_nestedtypes_insert_into_select.groovy | 1 -
...est_nestedtypes_json_insert_into_with_s3.groovy | 1 -
...l_tvf_with_complex_type_insertinto_doris.groovy | 2 +-
.../test_load_with_map_nested_array.groovy | 1 -
.../aggregate/map_agg_nested_insert_doris.groovy | 2 +-
.../show/test_nested_complex_switch.groovy | 56 ----------------------
18 files changed, 4 insertions(+), 108 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index f5865e1b7ba..01fd3596895 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2089,17 +2089,6 @@ public class Config extends ConfigBase {
@ConfField(mutable = true)
public static boolean disable_datev1 = true;
- /**
- * Now we not fully support array/struct/map nesting complex type in many
situation,
- * so just disable creating nesting complex data type when create table.
- * We can make it able after we fully support
- */
- @ConfField(mutable = true, masterOnly = true, description = {
- "当前默认设置为 true，不支持建表时创建复杂类型(array/struct/map)嵌套复杂类型,
仅支持array类型自身嵌套。",
- "Now default set to true, not support create complex
type(array/struct/map) nested complex type "
- + "when we create table, only support array type nested
array"})
- public static boolean disable_nested_complex_type = true;
-
/*
* This variable indicates the number of digits by which to increase the
scale
* of the result of division operations performed with the `/` operator.
The
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java
index c7199b82b28..2e8c1291b8b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java
@@ -28,7 +28,6 @@ import org.apache.doris.catalog.StructField;
import org.apache.doris.catalog.StructType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.Config;
import org.apache.doris.qe.SessionVariable;
import org.apache.doris.thrift.TColumnDesc;
import org.apache.doris.thrift.TPrimitiveType;
@@ -130,18 +129,10 @@ public class TypeDef implements ParseNode {
Type itemType = ((ArrayType) type).getItemType();
if (itemType instanceof ScalarType) {
analyzeNestedType(type, (ScalarType) itemType);
- } else if (Config.disable_nested_complex_type && !(itemType
instanceof ArrayType)) {
- // now we can array nesting array
- throw new AnalysisException("Unsupported data type:
ARRAY<" + itemType.toSql() + ">");
}
}
if (type.isMapType()) {
MapType mt = (MapType) type;
- if (Config.disable_nested_complex_type && (!(mt.getKeyType()
instanceof ScalarType)
- || !(mt.getValueType() instanceof ScalarType))) {
- throw new AnalysisException("Unsupported data type: MAP<"
+ mt.getKeyType().toSql() + ","
- + mt.getValueType().toSql() + ">");
- }
if (mt.getKeyType() instanceof ScalarType) {
analyzeNestedType(type, (ScalarType) mt.getKeyType());
}
@@ -160,8 +151,6 @@ public class TypeDef implements ParseNode {
throw new AnalysisException("Duplicate field name "
+ field.getName() + " in struct " +
type.toSql());
}
- } else if (Config.disable_nested_complex_type) {
- throw new AnalysisException("Unsupported field type: "
+ fieldType.toSql() + " for STRUCT");
}
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
index 5832a0534c0..bf85d405503 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
@@ -24,7 +24,6 @@ import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
-import org.apache.doris.common.Config;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.types.ArrayType;
import org.apache.doris.nereids.types.BigIntType;
@@ -392,21 +391,11 @@ public class ColumnDefinition {
Type itemType = ((org.apache.doris.catalog.ArrayType)
catalogType).getItemType();
if (itemType instanceof ScalarType) {
validateNestedType(catalogType, (ScalarType) itemType);
- } else if (Config.disable_nested_complex_type
- && !(itemType instanceof
org.apache.doris.catalog.ArrayType)) {
- // now we can array nesting array
- throw new AnalysisException(
- "Unsupported data type: ARRAY<" + itemType.toSql()
+ ">");
}
}
if (catalogType.isMapType()) {
org.apache.doris.catalog.MapType mt =
(org.apache.doris.catalog.MapType) catalogType;
- if (Config.disable_nested_complex_type && (!(mt.getKeyType()
instanceof ScalarType)
- || !(mt.getValueType() instanceof ScalarType))) {
- throw new AnalysisException("Unsupported data type: MAP<"
- + mt.getKeyType().toSql() + "," +
mt.getValueType().toSql() + ">");
- }
if (mt.getKeyType() instanceof ScalarType) {
validateNestedType(catalogType, (ScalarType)
mt.getKeyType());
}
@@ -426,9 +415,6 @@ public class ColumnDefinition {
throw new AnalysisException("Duplicate field name
" + field.getName()
+ " in struct " + catalogType.toSql());
}
- } else if (Config.disable_nested_complex_type) {
- throw new AnalysisException(
- "Unsupported field type: " + fieldType.toSql()
+ " for STRUCT");
}
}
}
diff --git
a/regression-test/data/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.out
b/regression-test/data/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.out
index ebc6fbe02b8..8e99380d460 100644
---
a/regression-test/data/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.out
+++
b/regression-test/data/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.out
@@ -2,9 +2,6 @@
-- !sql --
0
--- !sql --
-0
-
-- !sql --
2 {1:[0.35815922932906263, 0.0011899152357573994, 0.28749219850167373,
0.93512930168283781, 0.1552584991620739, 0.73308976093672584,
0.52815960653805338, 0.92936404769642733, 0.89084215295591418,
0.21986459138832559], 2:[0.25076205844319044, 0.54003619330849928,
0.70661164863002113, 0.99472899095144263, 0.078314941019622886,
0.44206845606243961, 0.30857433265707379, 0.048661247184825784,
0.76954870938240083, 0.27253204080482074], 3:[0.687619207224633,
0.81614795840545351, 0.8866348164721 [...]
diff --git
a/regression-test/suites/datatype_p0/nested_types/base_cases/three_level_nestedtypes_with_s3data.groovy
b/regression-test/suites/datatype_p0/nested_types/base_cases/three_level_nestedtypes_with_s3data.groovy
index 4258c30e1d4..66897b161d3 100644
---
a/regression-test/suites/datatype_p0/nested_types/base_cases/three_level_nestedtypes_with_s3data.groovy
+++
b/regression-test/suites/datatype_p0/nested_types/base_cases/three_level_nestedtypes_with_s3data.groovy
@@ -20,7 +20,7 @@ import org.apache.commons.lang3.StringUtils
suite("three_level_nestedtypes_with_s3data") {
sql """set enable_nereids_planner=false"""
sql """ set enable_fallback_to_original_planner=true;"""
- sql """ ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' =
'false');"""
+
// this test case aim to test one-level nested type with s3 data
diff --git
a/regression-test/suites/datatype_p0/nested_types/base_cases/two_level_nestedtypes_with_s3data.groovy
b/regression-test/suites/datatype_p0/nested_types/base_cases/two_level_nestedtypes_with_s3data.groovy
index c00ccfcf7d6..d4b701b94b8 100644
---
a/regression-test/suites/datatype_p0/nested_types/base_cases/two_level_nestedtypes_with_s3data.groovy
+++
b/regression-test/suites/datatype_p0/nested_types/base_cases/two_level_nestedtypes_with_s3data.groovy
@@ -20,7 +20,6 @@ import org.apache.commons.lang3.StringUtils
suite("two_level_nestedtypes_with_s3data") {
sql """set enable_nereids_planner=false"""
sql """ set enable_fallback_to_original_planner=true;"""
- sql """ ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' =
'false');"""
// this test case aim to test one-level nested type with s3 data
diff --git a/regression-test/suites/datatype_p0/nested_types/load.groovy
b/regression-test/suites/datatype_p0/nested_types/load.groovy
index 3ceffc30cd4..2bf559425d1 100644
--- a/regression-test/suites/datatype_p0/nested_types/load.groovy
+++ b/regression-test/suites/datatype_p0/nested_types/load.groovy
@@ -17,7 +17,7 @@
suite("load") {
// ddl begin
- sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' =
'false')"""
+
sql """set enable_nereids_planner=false"""
def dataFile = """test_scalar_types_100.csv"""
diff --git
a/regression-test/suites/datatype_p0/nested_types/query/map_functions/test_basic_map_function.groovy
b/regression-test/suites/datatype_p0/nested_types/query/map_functions/test_basic_map_function.groovy
index 88993c5aeb0..1b4a4e8a919 100644
---
a/regression-test/suites/datatype_p0/nested_types/query/map_functions/test_basic_map_function.groovy
+++
b/regression-test/suites/datatype_p0/nested_types/query/map_functions/test_basic_map_function.groovy
@@ -16,7 +16,6 @@
// under the License.
suite("test_basic_map_function", "p0") {
- sql """ ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' =
'false'); """
sql """set enable_nereids_planner=false"""
// ============ sum(map-value) ============
qt_sql """ SELECT "sum-map-value" """
diff --git
a/regression-test/suites/datatype_p0/nested_types/query/test_nested_type_with_count.groovy
b/regression-test/suites/datatype_p0/nested_types/query/test_nested_type_with_count.groovy
index 2b5bf97795e..1bc8285257e 100644
---
a/regression-test/suites/datatype_p0/nested_types/query/test_nested_type_with_count.groovy
+++
b/regression-test/suites/datatype_p0/nested_types/query/test_nested_type_with_count.groovy
@@ -20,7 +20,6 @@ import org.apache.commons.lang3.StringUtils
suite("test_nested_type_with_count") {
// this test case aim to test nested type with old planner
sql """set enable_nereids_planner=false"""
- sql """ ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' =
'false');"""
def table_names = ["test_array_one_level", "test_map_one_level",
"test_struct_one_level"]
diff --git
a/regression-test/suites/datatype_p0/nested_types/query/test_nested_types_insert_into_with_literal.groovy
b/regression-test/suites/datatype_p0/nested_types/query/test_nested_types_insert_into_with_literal.groovy
index da3262858ae..08e794b23f3 100644
---
a/regression-test/suites/datatype_p0/nested_types/query/test_nested_types_insert_into_with_literal.groovy
+++
b/regression-test/suites/datatype_p0/nested_types/query/test_nested_types_insert_into_with_literal.groovy
@@ -25,7 +25,6 @@ suite("test_nested_types_insert_into_with_literal", "p0") {
// old planner does not support cast empty
sql 'set enable_nereids_planner=true'
sql 'set enable_fallback_to_original_planner=false'
- sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' =
'false')"""
def table_names = [
"two_level_array_array_a",
diff --git
a/regression-test/suites/datatype_p0/nested_types/query/test_nested_types_insert_into_with_s3.groovy
b/regression-test/suites/datatype_p0/nested_types/query/test_nested_types_insert_into_with_s3.groovy
index 2b59e76aa29..fc2d3d25988 100644
---
a/regression-test/suites/datatype_p0/nested_types/query/test_nested_types_insert_into_with_s3.groovy
+++
b/regression-test/suites/datatype_p0/nested_types/query/test_nested_types_insert_into_with_s3.groovy
@@ -24,7 +24,6 @@ suite("test_nested_types_insert_into_with_s3", "p0") {
sql 'use regression_test_datatype_p0_nested_types'
sql 'set enable_nereids_planner=false'
sql 'set max_allowed_packet=4194304'
- sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' =
'false')"""
String ak = getS3AK()
String sk = getS3SK()
diff --git
a/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_csv_insert_into_with_s3.groovy
b/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_csv_insert_into_with_s3.groovy
index ab9dff9ffb2..40f76575bcf 100644
---
a/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_csv_insert_into_with_s3.groovy
+++
b/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_csv_insert_into_with_s3.groovy
@@ -25,7 +25,6 @@ suite("test_nestedtypes_csv_insert_into_with_s3", "p0") {
sql 'set enable_nereids_planner=false'
sql 'set max_allowed_packet=4194304'
sql 'set topn_opt_limit_threshold=10000'
- sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' =
'false')"""
String ak = getS3AK()
String sk = getS3SK()
diff --git
a/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.groovy
b/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.groovy
index a74687af64e..bc21c18a0a9 100644
---
a/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.groovy
+++
b/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_insert_into_select.groovy
@@ -22,7 +22,6 @@ import org.codehaus.groovy.runtime.IOGroovyMethods
suite("test_nestedtypes_insert_into_select", "p0") {
sql "set enable_nereids_planner=false"
- sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' =
'false')"""
// create array struct
sql "DROP TABLE IF EXISTS ast;"
diff --git
a/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_json_insert_into_with_s3.groovy
b/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_json_insert_into_with_s3.groovy
index fb83de7a340..66d5c6bc5ee 100644
---
a/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_json_insert_into_with_s3.groovy
+++
b/regression-test/suites/datatype_p0/nested_types/query/test_nestedtypes_json_insert_into_with_s3.groovy
@@ -25,7 +25,6 @@ suite("test_nestedtypes_json_insert_into_with_s3", "p0") {
sql 'set enable_nereids_planner=false'
sql 'set max_allowed_packet=4194304'
sql 'set topn_opt_limit_threshold=10000'
- sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' =
'false')"""
String ak = getS3AK()
String sk = getS3SK()
diff --git
a/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy
b/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy
index 04feab78ddb..c8bd8a8c2d6 100644
---
a/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy
+++
b/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy
@@ -43,7 +43,7 @@ suite("test_local_tvf_with_complex_type_insertinto_doris",
"p0") {
}
}
- qt_sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' =
'false')"""
+
// create doris table
sql """ DROP TABLE IF EXISTS ${table_name} """
diff --git
a/regression-test/suites/load_p0/stream_load/test_load_with_map_nested_array.groovy
b/regression-test/suites/load_p0/stream_load/test_load_with_map_nested_array.groovy
index 7e7e676bf3a..312905bc868 100644
---
a/regression-test/suites/load_p0/stream_load/test_load_with_map_nested_array.groovy
+++
b/regression-test/suites/load_p0/stream_load/test_load_with_map_nested_array.groovy
@@ -17,7 +17,6 @@
suite("test_load_with_map_nested_array", "p0") {
def tableName = "test_load_with_map_nested_array"
- sql """ ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' =
'false'); """
sql """ DROP TABLE IF EXISTS ${tableName} """
sql """
diff --git
a/regression-test/suites/query_p0/aggregate/map_agg_nested_insert_doris.groovy
b/regression-test/suites/query_p0/aggregate/map_agg_nested_insert_doris.groovy
index a4b0410cba3..8d4ae13b1cc 100644
---
a/regression-test/suites/query_p0/aggregate/map_agg_nested_insert_doris.groovy
+++
b/regression-test/suites/query_p0/aggregate/map_agg_nested_insert_doris.groovy
@@ -20,7 +20,7 @@ suite("map_agg_nested_insert_doris", "p0") {
def tb_doris = "test_map_agg_nested_insert_target"
sql "DROP TABLE IF EXISTS `${tb_base}`;"
sql "DROP TABLE IF EXISTS `${tb_doris}`;"
- sql """ ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' =
'false'); """
+
sql """
CREATE TABLE `${tb_base}` (
diff --git
a/regression-test/suites/query_p0/show/test_nested_complex_switch.groovy
b/regression-test/suites/query_p0/show/test_nested_complex_switch.groovy
index af39d8a9c32..9b06a00eb42 100644
--- a/regression-test/suites/query_p0/show/test_nested_complex_switch.groovy
+++ b/regression-test/suites/query_p0/show/test_nested_complex_switch.groovy
@@ -137,63 +137,7 @@ suite("test_nested_complex_switch", "query") {
sql "DROP TABLE IF EXISTS ${testTable_m}"
sql "DROP TABLE IF EXISTS ${testTable_a}"
sql "DROP TABLE IF EXISTS ${testTable_s}"
- sql "ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' =
'true')"
- // map
- test {
- sql sql_m_s
- exception "java.sql.SQLException: errCode = 2, detailMessage =
Unsupported data type: MAP<TEXT,STRUCT<f1:TINYINT>>"
- }
-
- test {
- sql sql_m_a
- exception "java.sql.SQLException: errCode = 2, detailMessage =
Unsupported data type: MAP<ARRAY<INT>,TEXT>"
- }
-
- test {
- sql sql_m_m
- exception "java.sql.SQLException: errCode = 2, detailMessage =
Unsupported data type: MAP<TEXT,MAP<TEXT,INT>>"
- }
-
- // array
- test {
- sql sql_a_s
- exception "java.sql.SQLException: errCode = 2, detailMessage =
Unsupported data type: ARRAY<STRUCT<f1:TINYINT>>"
- }
-
-
- test {
- sql sql_a_m
- exception "java.sql.SQLException: errCode = 2, detailMessage =
Unsupported data type: ARRAY<MAP<TEXT,INT>>"
- }
-
- // struct
- test {
- sql sql_s_s
- exception "java.sql.SQLException: errCode = 2, detailMessage =
Unsupported"
- }
-
- test {
- sql sql_s_a
- exception "java.sql.SQLException: errCode = 2, detailMessage =
Unsupported"
- }
-
- test {
- sql sql_s_m
- exception "java.sql.SQLException: errCode = 2, detailMessage =
Unsupported"
- }
-
- } finally {
- try_sql("DROP TABLE IF EXISTS ${testTable_m}")
- try_sql("DROP TABLE IF EXISTS ${testTable_a}")
- try_sql("DROP TABLE IF EXISTS ${testTable_s}")
- }
-
- try {
- sql "DROP TABLE IF EXISTS ${testTable_m}"
- sql "DROP TABLE IF EXISTS ${testTable_a}"
- sql "DROP TABLE IF EXISTS ${testTable_s}"
- sql "ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' =
'false')"
// map
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
