This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 0e638f69c399e78bbe6e6a4e2a2efd3a88501990
Author: Siyang Tang <82279870+tangsiyang2...@users.noreply.github.com>
AuthorDate: Sun Sep 10 10:51:07 2023 +0800
[fix](broker-load) make sequence column name case insensitive (#24071)
---
.../org/apache/doris/analysis/DataDescription.java | 4 +-
.../data/load_p0/broker_load/test_seq_load.out | 4 ++
.../load_p0/broker_load/ddl/uniq_tbl_basic.sql | 58 +++++++++++++++++
.../broker_load/ddl/uniq_tbl_basic_drop.sql | 1 +
.../load_p0/broker_load/test_seq_load.groovy | 76 ++++++++++++++++++++++
5 files changed, 141 insertions(+), 2 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java
index 2ee09f3c52..c60cbb76af 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java
@@ -939,7 +939,7 @@ public class DataDescription implements InsertStmt.DataDesc
{
boolean hasSourceSequenceCol = false;
if (!parsedColumnExprList.isEmpty()) {
for (ImportColumnDesc importColumnDesc : parsedColumnExprList) {
- if (importColumnDesc.getColumnName().equals(sequenceCol)) {
+ if
(importColumnDesc.getColumnName().equalsIgnoreCase(sequenceCol)) {
hasSourceSequenceCol = true;
break;
}
@@ -947,7 +947,7 @@ public class DataDescription implements InsertStmt.DataDesc
{
} else {
List<Column> columns = olapTable.getBaseSchema();
for (Column column : columns) {
- if (column.getName().equals(sequenceCol)) {
+ if (column.getName().equalsIgnoreCase(sequenceCol)) {
hasSourceSequenceCol = true;
break;
}
diff --git a/regression-test/data/load_p0/broker_load/test_seq_load.out
b/regression-test/data/load_p0/broker_load/test_seq_load.out
new file mode 100644
index 0000000000..3b2c909248
--- /dev/null
+++ b/regression-test/data/load_p0/broker_load/test_seq_load.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+20
+
diff --git a/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic.sql
b/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic.sql
new file mode 100644
index 0000000000..36d5ac1e05
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic.sql
@@ -0,0 +1,58 @@
+CREATE TABLE uniq_tbl_basic
+(
+ k00 INT NOT NULL,
+ k01 DATE NOT NULL,
+ k02 BOOLEAN NULL,
+ k03 TINYINT NULL,
+ k04 SMALLINT NULL,
+ k05 INT NULL,
+ k06 BIGINT NULL,
+ k07 LARGEINT NULL,
+ k08 FLOAT NULL,
+ k09 DOUBLE NULL,
+ k10 DECIMAL(9,1) NULL,
+ k11 DECIMALV3(9,1) NULL,
+ k12 DATETIME NULL,
+ k13 DATEV2 NULL,
+ k14 DATETIMEV2 NULL,
+ k15 CHAR NULL,
+ k16 VARCHAR NULL,
+ k17 STRING NULL,
+ k18 JSON NULL,
+ kd01 BOOLEAN NOT NULL DEFAULT "TRUE",
+ kd02 TINYINT NOT NULL DEFAULT "1",
+ kd03 SMALLINT NOT NULL DEFAULT "2",
+ kd04 INT NOT NULL DEFAULT "3",
+ kd05 BIGINT NOT NULL DEFAULT "4",
+ kd06 LARGEINT NOT NULL DEFAULT "5",
+ kd07 FLOAT NOT NULL DEFAULT "6.0",
+ kd08 DOUBLE NOT NULL DEFAULT "7.0",
+ kd09 DECIMAL NOT NULL DEFAULT "888888888",
+ kd10 DECIMALV3 NOT NULL DEFAULT "999999999",
+ kd11 DATE NOT NULL DEFAULT "2023-08-24",
+ kd12 DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ kd13 DATEV2 NOT NULL DEFAULT "2023-08-24",
+ kd14 DATETIMEV2 NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ kd15 CHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd16 VARCHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd17 STRING NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd18 JSON NULL,
+
+ INDEX idx_bitmap_k104 (`k02`) USING BITMAP,
+ INDEX idx_bitmap_k110 (`kd01`) USING BITMAP,
+ INDEX idx_bitmap_k113 (`k13`) USING BITMAP,
+ INDEX idx_bitmap_k114 (`k14`) USING BITMAP,
+ INDEX idx_bitmap_k117 (`k17`) USING BITMAP
+)
+UNIQUE KEY(k00,k01)
+PARTITION BY RANGE(k01)
+(
+ PARTITION p1 VALUES [('2023-08-01'), ('2023-08-11')),
+ PARTITION p2 VALUES [('2023-08-11'), ('2023-08-21')),
+ PARTITION p3 VALUES [('2023-08-21'), ('2023-09-01'))
+)
+DISTRIBUTED BY HASH(k00) BUCKETS 32
+PROPERTIES (
+ "function_column.sequence_col" = "k12",
+ "replication_num" = "1"
+);
\ No newline at end of file
diff --git
a/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic_drop.sql
b/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic_drop.sql
new file mode 100644
index 0000000000..0e82736da9
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS uniq_tbl_basic
\ No newline at end of file
diff --git a/regression-test/suites/load_p0/broker_load/test_seq_load.groovy
b/regression-test/suites/load_p0/broker_load/test_seq_load.groovy
new file mode 100644
index 0000000000..96040c58b6
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/test_seq_load.groovy
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_seq_load", "load_p0") {
+
+ def tableName = "uniq_tbl_basic"
+
+ sql new File("""${ context.file.parent
}/ddl/uniq_tbl_basic_drop.sql""").text
+ sql new File("""${ context.file.parent }/ddl/uniq_tbl_basic.sql""").text
+
+ def label = UUID.randomUUID().toString().replace("-", "0")
+ def path =
"s3://doris-build-1308700295/regression/load/data/basic_data.csv"
+ def format_str = "CSV"
+ def ak = getS3AK()
+ def sk = getS3SK()
+ def seq_column = "K01"
+
+ def sql_str = """
+ LOAD LABEL $label (
+ DATA INFILE("$path")
+ INTO TABLE $tableName
+ COLUMNS TERMINATED BY "|"
+ FORMAT AS $format_str
+
(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)
+ ORDER BY $seq_column
+ )
+ WITH S3 (
+ "AWS_ACCESS_KEY" = "$ak",
+ "AWS_SECRET_KEY" = "$sk",
+ "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
+ "AWS_REGION" = "ap-beijing"
+ )
+ properties(
+ "use_new_load_scan_node" = "true"
+ )
+ """
+ logger.info("submit sql: ${sql_str}");
+ sql """${sql_str}"""
+ logger.info("Submit load with lable: $label, table: $tableName, path:
$path")
+
+ def max_try_milli_secs = 600000
+ while (max_try_milli_secs > 0) {
+ String[][] result = sql """ show load where label="$label" order by
createtime desc limit 1; """
+ if (result[0][2].equals("FINISHED")) {
+ logger.info("Load FINISHED " + label)
+ break
+ }
+ if (result[0][2].equals("CANCELLED")) {
+ def reason = result[0][7]
+ logger.info("load failed, reason:$reason")
+ assertTrue(1 == 2)
+ break
+ }
+ Thread.sleep(1000)
+ max_try_milli_secs -= 1000
+ if(max_try_milli_secs <= 0) {
+ assertTrue(1 == 2, "load Timeout: $label")
+ }
+ }
+
+ qt_sql """ SELECT COUNT(*) FROM ${tableName} """
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
