This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push: new 221e7d69256 [fix](test) fix some tests in external p0 (#36127) (#36149) 221e7d69256 is described below commit 221e7d692569ff20f24e414432323fa09933fa0a Author: Mingyu Chen <morning...@163.com> AuthorDate: Wed Jun 19 22:51:09 2024 +0800 [fix](test) fix some tests in external p0 (#36127) (#36149) Also move the analysis exception of "Not support insert with partition spec in hive catalog." from create sink phase to bind sink phase. So that when `set enable_fallback_to_original_planner=false;`, the return error will be correct. branch-2.1: #36127 #36262 #36284 --- .../doris/nereids/analyzer/UnboundTableSinkCreator.java | 3 --- .../apache/doris/nereids/rules/analysis/BindSink.java | 6 +++++- .../hive/write/test_hive_write_insert.out | 16 ---------------- .../hive/ddl/test_hive_write_type.groovy | 16 ---------------- .../hive/write/test_hive_write_insert.groovy | 5 ----- .../hive/test_external_catalog_hive.groovy | 5 +++-- 6 files changed, 8 insertions(+), 43 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSinkCreator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSinkCreator.java index 66222fcf3d6..6c361c36f05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSinkCreator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSinkCreator.java @@ -105,9 +105,6 @@ public class UnboundTableSinkCreator { isPartialUpdate, dmlCommandType, Optional.empty(), Optional.empty(), plan); } else if (curCatalog instanceof HMSExternalCatalog && !isAutoDetectPartition) { - if (!partitions.isEmpty()) { - throw new AnalysisException("Not support insert with partition spec in hive catalog."); - } return new UnboundHiveTableSink<>(nameParts, colNames, hints, partitions, dmlCommandType, Optional.empty(), Optional.empty(), plan); } else if (curCatalog instanceof IcebergExternalCatalog && !isAutoDetectPartition) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java index 63f6db8fad9..0c0f9e2e25f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java @@ -391,6 +391,10 @@ public class BindSink implements AnalysisRuleFactory { HMSExternalTable table = pair.second; LogicalPlan child = ((LogicalPlan) sink.child()); + if (!sink.getPartitions().isEmpty()) { + throw new AnalysisException("Not support insert with partition spec in hive catalog."); + } + List<Column> bindColumns; if (sink.getColNames().isEmpty()) { bindColumns = table.getBaseSchema(true).stream().collect(ImmutableList.toImmutableList()); @@ -399,7 +403,7 @@ public class BindSink implements AnalysisRuleFactory { Column column = table.getColumn(cn); if (column == null) { throw new AnalysisException(String.format("column %s is not found in table %s", - cn, table.getName())); + cn, table.getName())); } return column; }).collect(ImmutableList.toImmutableList()); diff --git a/regression-test/data/external_table_p0/hive/write/test_hive_write_insert.out b/regression-test/data/external_table_p0/hive/write/test_hive_write_insert.out index a2902ef187e..014d4598224 100644 --- a/regression-test/data/external_table_p0/hive/write/test_hive_write_insert.out +++ b/regression-test/data/external_table_p0/hive/write/test_hive_write_insert.out @@ -22,7 +22,6 @@ false -7 -15 16 -9223372036854775808 -123.45 -123456.789 123456789 -1234.5678 -1 \N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240321 -- !q06 -- -\N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240321 -- !q01 -- false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123456 2024-03-21T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890 [...] @@ -54,9 +53,6 @@ true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5 \N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -- !q05 -- -\N \N \N \N \N -123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {2:20} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [3.4567, 4.5678] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -\N \N \N \N \N -123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {3:20} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [8.4567, 4.5678] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -\N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -- !q01 -- true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789 [...] @@ -138,7 +134,6 @@ false -7 -15 16 -9223372036854775808 -123.45 -123456.789 123456789 -1234.5678 -1 \N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240321 -- !q06 -- -\N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240321 -- !q01 -- false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123456 2024-03-21T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890 [...] @@ -170,9 +165,6 @@ true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5 \N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -- !q05 -- -\N \N \N \N \N -123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {2:20} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [3.4567, 4.5678] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -\N \N \N \N \N -123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {3:20} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [8.4567, 4.5678] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -\N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -- !q01 -- true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789 [...] @@ -254,7 +246,6 @@ false -7 -15 16 -9223372036854775808 -123.45 -123456.789 123456789 -1234.5678 -1 \N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240321 -- !q06 -- -\N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240321 -- !q01 -- false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123456 2024-03-21T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890 [...] @@ -286,9 +277,6 @@ true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5 \N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -- !q05 -- -\N \N \N \N \N -123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {2:20} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [3.4567, 4.5678] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -\N \N \N \N \N -123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {3:20} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [8.4567, 4.5678] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -\N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -- !q01 -- true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789 [...] @@ -370,7 +358,6 @@ false -7 -15 16 -9223372036854775808 -123.45 -123456.789 123456789 -1234.5678 -1 \N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240321 -- !q06 -- -\N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N 20240321 -- !q01 -- false -128 -32768 -2147483648 -9223372036854775808 -123.45 -123456.789 -123456789 -1234.5678 -123456.789012 -123456789.012345678901 string_value binary_value 2024-03-21 2024-03-21T12:00 2024-03-21T12:00:00.123456 2024-03-21T12:00:00.123456 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"x":"y"} {2:20} {2:200000000000} {2.2:20.2} {2.2:20.2} {0:1} {2.2:2.2} {2.34:2.34} {2.3456:2.3456} {2.34567890:2.34567890} {2.34567890 [...] @@ -402,9 +389,6 @@ true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5 \N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -- !q05 -- -\N \N \N \N \N -123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {2:20} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [3.4567, 4.5678] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -\N \N \N \N \N -123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {3:20} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [8.4567, 4.5678] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -\N \N \N \N \N 123.45 \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N {1:10} \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N \N [1.2345, 2.3456] \N \N \N \N \N \N \N \N \N \N [null, "value1", "value2"] \N \N \N \N -- !q01 -- true 127 32767 2147483647 9223372036854775807 123.45 123456.789 123456789 1234.5678 123456.789012 123456789.012345678901 string_value binary_value 2024-03-20 2024-03-20T12:00 2024-03-20T12:00:00.123457 2024-03-20T12:00:00.123457 char_value1 char_value2 char_value3 varchar_value1 varchar_value2 varchar_value3 {"key1":"value1"} {"key1":"value1"} {"a":"b"} {1:10} {1:100000000000} {1.1:10.1} {1.1:10.1} {1:0} {1.1:1.1} {1.23:1.23} {1.2345:1.2345} {1.23456789:1.23456789} {1.23456789:1.23456789 [...] diff --git a/regression-test/suites/external_table_p0/hive/ddl/test_hive_write_type.groovy b/regression-test/suites/external_table_p0/hive/ddl/test_hive_write_type.groovy index 62b8b730e2b..6bf428b592f 100644 --- a/regression-test/suites/external_table_p0/hive/ddl/test_hive_write_type.groovy +++ b/regression-test/suites/external_table_p0/hive/ddl/test_hive_write_type.groovy @@ -241,14 +241,6 @@ suite("test_hive_write_type", "p0,external,hive,external_docker,external_docker_ exception "errCode = 2, detailMessage = Unknown column 'pt00' in target table." } - test { - sql """ INSERT INTO ex_tbl_${file_format} partition(`pt1`,`pt2`) (`col1`, `col9`) - VALUES - ('abcdefghij', 'error', true, 123); - """ - exception "errCode = 2, detailMessage = Not support insert with partition spec in hive catalog" - } - // TODO: support partition spec test { sql """ INSERT INTO ex_tbl_${file_format} partition(`pt1`,`pt2`) (`col3`, `col6`, `col9`) @@ -258,14 +250,6 @@ suite("test_hive_write_type", "p0,external,hive,external_docker,external_docker_ exception "errCode = 2, detailMessage = Not support insert with partition spec in hive catalog" } - test { - sql """ INSERT INTO ex_tbl_${file_format} partition(`pt0`, `pt1`,`pt3`) (`col3`, `col6`, `col9`) - VALUES - ('err', 'err', 'err', 9876543210, 6.28, 'error'); - """ - exception "errCode = 2, detailMessage = Not support insert with partition spec in hive catalog" - } - sql """ DROP TABLE ${catalog_name}.test_hive_ex.ex_tbl_${file_format} """ sql """ DROP DATABASE ${catalog_name}.test_hive_ex """ } diff --git a/regression-test/suites/external_table_p0/hive/write/test_hive_write_insert.groovy b/regression-test/suites/external_table_p0/hive/write/test_hive_write_insert.groovy index 30b6d55c61c..0b6fab86b2b 100644 --- a/regression-test/suites/external_table_p0/hive/write/test_hive_write_insert.groovy +++ b/regression-test/suites/external_table_p0/hive/write/test_hive_write_insert.groovy @@ -393,8 +393,6 @@ suite("test_hive_write_insert", "p0,external,hive,external_docker,external_docke logger.info("hive sql: " + """ truncate table all_types_${format_compression}; """) hive_docker """ truncate table all_types_${format_compression}; """ - order_qt_q06 """ select * from all_types_${format_compression}; - """ } def q02 = { String format_compression, String catalog_name -> @@ -444,9 +442,6 @@ suite("test_hive_write_insert", "p0,external,hive,external_docker,external_docke logger.info("hive sql: " + """ truncate table all_types_${format_compression}; """) hive_docker """ truncate table all_types_${format_compression}; """ - order_qt_q05 """ - select * from all_types_${format_compression}; - """ } def q03 = { String format_compression, String catalog_name -> logger.info("hive sql: " + """ DROP TABLE IF EXISTS all_types_par_${format_compression}_${catalog_name}_q03; """) diff --git a/regression-test/suites/external_table_p2/hive/test_external_catalog_hive.groovy b/regression-test/suites/external_table_p2/hive/test_external_catalog_hive.groovy index 7bc2979b914..813f316df29 100644 --- a/regression-test/suites/external_table_p2/hive/test_external_catalog_hive.groovy +++ b/regression-test/suites/external_table_p2/hive/test_external_catalog_hive.groovy @@ -23,6 +23,7 @@ suite("test_external_catalog_hive", "p2,external,hive,external_remote,external_r String extHiveHmsPort = context.config.otherConfigs.get("extHiveHmsPort") String catalog_name = "test_external_catalog_hive" + sql """set enable_fallback_to_original_planner=false""" sql """drop catalog if exists ${catalog_name};""" sql """ @@ -47,7 +48,7 @@ suite("test_external_catalog_hive", "p2,external,hive,external_remote,external_r sql """switch ${catalog_name};""" // test small table(text format) def q01 = { - qt_q01 """ select name, count(1) as c from student group by name order by c desc;""" + qt_q01 """ select name, count(1) as c from student group by name order by name desc;""" qt_q02 """ select lo_orderkey, count(1) as c from lineorder group by lo_orderkey order by lo_orderkey asc;""" qt_q03 """ select * from test1 order by col_1;""" qt_q04 """ select * from string_table order by p_partkey desc;""" @@ -148,7 +149,7 @@ suite("test_external_catalog_hive", "p2,external,hive,external_remote,external_r 'type'='hms', 'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}', 'access_controller.properties.ranger.service.name' = 'hive_wrong', - 'access_controller.class' = 'org.apache.doris.catalog.authorizer.RangerHiveAccessControllerFactory' + 'access_controller.class' = 'org.apache.doris.catalog.authorizer.ranger.hive.RangerHiveAccessControllerFactory' ); """ exception "Failed to init access controller: bound must be positive" --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org