This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 9cd9e195d8 [test](load) add some s3 load regression test (#24399)
9cd9e195d8 is described below
commit 9cd9e195d8eae0dd1d56db3d84d7a2019a6c071c
Author: Siyang Tang <82279870+tangsiyang2...@users.noreply.github.com>
AuthorDate: Sat Sep 23 22:40:45 2023 +0800
[test](load) add some s3 load regression test (#24399)
---
.../data/load_p0/broker_load/test_s3_load.out | 22 +
.../load_p0/broker_load/ddl/agg_tbl_basic.sql | 59 +++
.../load_p0/broker_load/ddl/agg_tbl_basic_drop.sql | 1 +
.../load_p0/broker_load/ddl/dup_tbl_array.sql | 42 ++
.../load_p0/broker_load/ddl/dup_tbl_array_drop.sql | 1 +
.../ddl/{uniq_tbl_basic.sql => dup_tbl_basic.sql} | 25 +-
.../load_p0/broker_load/ddl/dup_tbl_basic_drop.sql | 1 +
.../load_p0/broker_load/ddl/mow_tbl_array.sql | 44 ++
.../load_p0/broker_load/ddl/mow_tbl_array_drop.sql | 1 +
.../ddl/{uniq_tbl_basic.sql => mow_tbl_basic.sql} | 35 +-
.../load_p0/broker_load/ddl/mow_tbl_basic_drop.sql | 1 +
.../load_p0/broker_load/ddl/uniq_tbl_array.sql | 43 ++
.../broker_load/ddl/uniq_tbl_array_drop.sql | 1 +
.../load_p0/broker_load/ddl/uniq_tbl_basic.sql | 2 +-
.../broker_load/ddl/uniq_tbl_basic_drop.sql | 2 +-
.../suites/load_p0/broker_load/test_s3_load.groovy | 513 +++++++++++++++++++++
.../load_p0/broker_load/test_seq_load.groovy | 68 ++-
17 files changed, 833 insertions(+), 28 deletions(-)
diff --git a/regression-test/data/load_p0/broker_load/test_s3_load.out
b/regression-test/data/load_p0/broker_load/test_s3_load.out
new file mode 100644
index 0000000000..6bc8c5ca75
--- /dev/null
+++ b/regression-test/data/load_p0/broker_load/test_s3_load.out
@@ -0,0 +1,22 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select --
+48
+
+-- !select --
+180
+
+-- !select --
+200
+
+-- !select --
+18
+
+-- !select --
+48
+
+-- !select --
+18
+
+-- !select --
+48
+
diff --git a/regression-test/suites/load_p0/broker_load/ddl/agg_tbl_basic.sql
b/regression-test/suites/load_p0/broker_load/ddl/agg_tbl_basic.sql
new file mode 100644
index 0000000000..23a061b254
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/agg_tbl_basic.sql
@@ -0,0 +1,59 @@
+CREATE TABLE agg_tbl_basic
+(
+ k00 INT NOT NULL,
+ k01 DATE NOT NULL,
+ k02 BOOLEAN REPLACE NULL,
+ k03 TINYINT SUM NULL,
+ k04 SMALLINT SUM NULL,
+ k05 INT SUM NULL,
+ k06 BIGINT SUM NULL,
+ k07 LARGEINT SUM NULL,
+ k08 FLOAT SUM NULL,
+ k09 DOUBLE SUM NULL,
+ k10 DECIMAL(9,1) SUM NULL,
+ k11 DECIMALV3(9,1) SUM NULL,
+ k12 DATETIME REPLACE NULL,
+ k13 DATEV2 REPLACE NULL,
+ k14 DATETIMEV2 REPLACE NULL,
+ k15 CHAR(300) REPLACE NULL,
+ k16 VARCHAR(300) REPLACE NULL,
+ k17 STRING REPLACE NULL,
+ k18 JSON REPLACE NULL,
+ k19 BITMAP BITMAP_UNION NULL,
+ k20 HLL HLL_UNION NULL,
+ k21 QUANTILE_STATE QUANTILE_UNION NULL,
+ kd01 BOOLEAN REPLACE NOT NULL DEFAULT "TRUE",
+ kd02 TINYINT SUM NOT NULL DEFAULT "1",
+ kd03 SMALLINT SUM NOT NULL DEFAULT "2",
+ kd04 INT SUM NOT NULL DEFAULT "3",
+ kd05 BIGINT SUM NOT NULL DEFAULT "4",
+ kd06 LARGEINT SUM NOT NULL DEFAULT "5",
+ kd07 FLOAT SUM NOT NULL DEFAULT "6.0",
+ kd08 DOUBLE SUM NOT NULL DEFAULT "7.0",
+ kd09 DECIMAL SUM NOT NULL DEFAULT "888888888",
+ kd10 DECIMALV3 SUM NOT NULL DEFAULT "999999999",
+ kd11 DATE REPLACE NOT NULL DEFAULT "2023-08-24",
+ kd12 DATETIME REPLACE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ kd13 DATEV2 REPLACE NOT NULL DEFAULT "2023-08-24",
+ kd14 DATETIMEV2 REPLACE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ kd15 CHAR(300) REPLACE NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd16 VARCHAR(300) REPLACE NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd17 STRING REPLACE NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd18 JSON REPLACE NULL,
+ kd19 BITMAP BITMAP_UNION NULL,
+ kd20 HLL HLL_UNION NULL,
+ kd21 QUANTILE_STATE QUANTILE_UNION NULL,
+
+ INDEX idx_bitmap_k104 (`k01`) USING BITMAP
+)
+ AGGREGATE KEY(k00,k01)
+PARTITION BY RANGE(k01)
+(
+ PARTITION p1 VALUES [('2023-08-01'), ('2023-08-11')),
+ PARTITION p2 VALUES [('2023-08-11'), ('2023-08-21')),
+ PARTITION p3 VALUES [('2023-08-21'), ('2023-09-01'))
+)
+DISTRIBUTED BY HASH(k00) BUCKETS 32
+PROPERTIES (
+ "replication_num" = "1"
+);
diff --git
a/regression-test/suites/load_p0/broker_load/ddl/agg_tbl_basic_drop.sql
b/regression-test/suites/load_p0/broker_load/ddl/agg_tbl_basic_drop.sql
new file mode 100644
index 0000000000..77062523a3
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/agg_tbl_basic_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS agg_tbl_basic
diff --git a/regression-test/suites/load_p0/broker_load/ddl/dup_tbl_array.sql
b/regression-test/suites/load_p0/broker_load/ddl/dup_tbl_array.sql
new file mode 100644
index 0000000000..d9c31be665
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/dup_tbl_array.sql
@@ -0,0 +1,42 @@
+CREATE TABLE dup_tbl_array
+(
+ k00 INT NOT NULL,
+ k01 array<BOOLEAN> NULL,
+ k02 array<TINYINT> NULL,
+ k03 array<SMALLINT> NULL,
+ k04 array<INT> NULL,
+ k05 array<BIGINT> NULL,
+ k06 array<LARGEINT> NULL,
+ k07 array<FLOAT> NULL,
+ k08 array<DOUBLE> NULL,
+ k09 array<DECIMAL> NULL,
+ k10 array<DECIMALV3> NULL,
+ k11 array<DATE> NULL,
+ k12 array<DATETIME> NULL,
+ k13 array<DATEV2> NULL,
+ k14 array<DATETIMEV2> NULL,
+ k15 array<CHAR> NULL,
+ k16 array<VARCHAR> NULL,
+ k17 array<STRING> NULL,
+ kd01 array<BOOLEAN> NOT NULL DEFAULT "[]",
+ kd02 array<TINYINT> NOT NULL DEFAULT "[]",
+ kd03 array<SMALLINT> NOT NULL DEFAULT "[]",
+ kd04 array<INT> NOT NULL DEFAULT "[]",
+ kd05 array<BIGINT> NOT NULL DEFAULT "[]",
+ kd06 array<LARGEINT> NOT NULL DEFAULT "[]",
+ kd07 array<FLOAT> NOT NULL DEFAULT "[]",
+ kd08 array<DOUBLE> NOT NULL DEFAULT "[]",
+ kd09 array<DECIMAL> NOT NULL DEFAULT "[]",
+ kd10 array<DECIMALV3> NOT NULL DEFAULT "[]",
+ kd11 array<DATE> NOT NULL DEFAULT "[]",
+ kd12 array<DATETIME> NOT NULL DEFAULT "[]",
+ kd13 array<DATEV2> NOT NULL DEFAULT "[]",
+ kd14 array<DATETIMEV2> NOT NULL DEFAULT "[]",
+ kd15 array<CHAR> NOT NULL DEFAULT "[]",
+ kd16 array<VARCHAR> NOT NULL DEFAULT "[]",
+)
+ DUPLICATE KEY(k00)
+DISTRIBUTED BY HASH(k00) BUCKETS 32
+PROPERTIES (
+ "replication_num" = "1"
+);
diff --git
a/regression-test/suites/load_p0/broker_load/ddl/dup_tbl_array_drop.sql
b/regression-test/suites/load_p0/broker_load/ddl/dup_tbl_array_drop.sql
new file mode 100644
index 0000000000..1c792f291f
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/dup_tbl_array_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS dup_tbl_array
diff --git a/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic.sql
b/regression-test/suites/load_p0/broker_load/ddl/dup_tbl_basic.sql
similarity index 71%
copy from regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic.sql
copy to regression-test/suites/load_p0/broker_load/ddl/dup_tbl_basic.sql
index 36d5ac1e05..0fa50ada8e 100644
--- a/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic.sql
+++ b/regression-test/suites/load_p0/broker_load/ddl/dup_tbl_basic.sql
@@ -1,4 +1,4 @@
-CREATE TABLE uniq_tbl_basic
+CREATE TABLE dup_tbl_basic
(
k00 INT NOT NULL,
k01 DATE NOT NULL,
@@ -10,8 +10,8 @@ CREATE TABLE uniq_tbl_basic
k07 LARGEINT NULL,
k08 FLOAT NULL,
k09 DOUBLE NULL,
- k10 DECIMAL(9,1) NULL,
- k11 DECIMALV3(9,1) NULL,
+ k10 DECIMAL(9,1) NULL,
+ k11 DECIMALV3(9,1) NULL,
k12 DATETIME NULL,
k13 DATEV2 NULL,
k14 DATETIMEV2 NULL,
@@ -38,13 +38,20 @@ CREATE TABLE uniq_tbl_basic
kd17 STRING NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
kd18 JSON NULL,
+ INDEX idx_inverted_k104 (`k05`) USING INVERTED,
+ INDEX idx_inverted_k110 (`k11`) USING INVERTED,
+ INDEX idx_inverted_k113 (`k13`) USING INVERTED,
+ INDEX idx_inverted_k114 (`k14`) USING INVERTED,
+ INDEX idx_inverted_k117 (`k17`) USING INVERTED PROPERTIES("parser" =
"english"),
+ INDEX idx_ngrambf_k115 (`k15`) USING NGRAM_BF PROPERTIES("gram_size"="3",
"bf_size"="256"),
+ INDEX idx_ngrambf_k116 (`k16`) USING NGRAM_BF PROPERTIES("gram_size"="3",
"bf_size"="256"),
+ INDEX idx_ngrambf_k117 (`k17`) USING NGRAM_BF PROPERTIES("gram_size"="3",
"bf_size"="256"),
+
INDEX idx_bitmap_k104 (`k02`) USING BITMAP,
INDEX idx_bitmap_k110 (`kd01`) USING BITMAP,
- INDEX idx_bitmap_k113 (`k13`) USING BITMAP,
- INDEX idx_bitmap_k114 (`k14`) USING BITMAP,
- INDEX idx_bitmap_k117 (`k17`) USING BITMAP
+
)
-UNIQUE KEY(k00,k01)
+ DUPLICATE KEY(k00)
PARTITION BY RANGE(k01)
(
PARTITION p1 VALUES [('2023-08-01'), ('2023-08-11')),
@@ -53,6 +60,6 @@ PARTITION BY RANGE(k01)
)
DISTRIBUTED BY HASH(k00) BUCKETS 32
PROPERTIES (
- "function_column.sequence_col" = "k12",
+ "bloom_filter_columns"="k05",
"replication_num" = "1"
-);
\ No newline at end of file
+);
diff --git
a/regression-test/suites/load_p0/broker_load/ddl/dup_tbl_basic_drop.sql
b/regression-test/suites/load_p0/broker_load/ddl/dup_tbl_basic_drop.sql
new file mode 100644
index 0000000000..2fb0edc549
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/dup_tbl_basic_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS dup_tbl_basic
diff --git a/regression-test/suites/load_p0/broker_load/ddl/mow_tbl_array.sql
b/regression-test/suites/load_p0/broker_load/ddl/mow_tbl_array.sql
new file mode 100644
index 0000000000..0109119299
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/mow_tbl_array.sql
@@ -0,0 +1,44 @@
+CREATE TABLE mow_tbl_array
+(
+ k00 INT NOT NULL,
+ k01 array<BOOLEAN> NULL,
+ k02 array<TINYINT> NULL,
+ k03 array<SMALLINT> NULL,
+ k04 array<INT> NULL,
+ k05 array<BIGINT> NULL,
+ k06 array<LARGEINT> NULL,
+ k07 array<FLOAT> NULL,
+ k08 array<DOUBLE> NULL,
+ k09 array<DECIMAL> NULL,
+ k10 array<DECIMALV3> NULL,
+ k11 array<DATE> NULL,
+ k12 array<DATETIME> NULL,
+ k13 array<DATEV2> NULL,
+ k14 array<DATETIMEV2> NULL,
+ k15 array<CHAR> NULL,
+ k16 array<VARCHAR> NULL,
+ k17 array<STRING> NULL,
+ kd01 array<BOOLEAN> NOT NULL DEFAULT "[]",
+ kd02 array<TINYINT> NOT NULL DEFAULT "[]",
+ kd03 array<SMALLINT> NOT NULL DEFAULT "[]",
+ kd04 array<INT> NOT NULL DEFAULT "[]",
+ kd05 array<BIGINT> NOT NULL DEFAULT "[]",
+ kd06 array<LARGEINT> NOT NULL DEFAULT "[]",
+ kd07 array<FLOAT> NOT NULL DEFAULT "[]",
+ kd08 array<DOUBLE> NOT NULL DEFAULT "[]",
+ kd09 array<DECIMAL> NOT NULL DEFAULT "[]",
+ kd10 array<DECIMALV3> NOT NULL DEFAULT "[]",
+ kd11 array<DATE> NOT NULL DEFAULT "[]",
+ kd12 array<DATETIME> NOT NULL DEFAULT "[]",
+ kd13 array<DATEV2> NOT NULL DEFAULT "[]",
+ kd14 array<DATETIMEV2> NOT NULL DEFAULT "[]",
+ kd15 array<CHAR> NOT NULL DEFAULT "[]",
+ kd16 array<VARCHAR> NOT NULL DEFAULT "[]",
+ kd17 array<STRING> NOT NULL DEFAULT "[]"
+)
+ UNIQUE KEY(k00)
+DISTRIBUTED BY HASH(k00) BUCKETS 32
+PROPERTIES (
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "true"
+);
diff --git
a/regression-test/suites/load_p0/broker_load/ddl/mow_tbl_array_drop.sql
b/regression-test/suites/load_p0/broker_load/ddl/mow_tbl_array_drop.sql
new file mode 100644
index 0000000000..76e3725e41
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/mow_tbl_array_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS mow_tbl_array
diff --git a/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic.sql
b/regression-test/suites/load_p0/broker_load/ddl/mow_tbl_basic.sql
similarity index 59%
copy from regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic.sql
copy to regression-test/suites/load_p0/broker_load/ddl/mow_tbl_basic.sql
index 36d5ac1e05..efc1ad2cab 100644
--- a/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic.sql
+++ b/regression-test/suites/load_p0/broker_load/ddl/mow_tbl_basic.sql
@@ -1,7 +1,7 @@
-CREATE TABLE uniq_tbl_basic
+CREATE TABLE mow_tbl_basic
(
k00 INT NOT NULL,
- k01 DATE NOT NULL,
+ k01 DATE NULL,
k02 BOOLEAN NULL,
k03 TINYINT NULL,
k04 SMALLINT NULL,
@@ -10,8 +10,8 @@ CREATE TABLE uniq_tbl_basic
k07 LARGEINT NULL,
k08 FLOAT NULL,
k09 DOUBLE NULL,
- k10 DECIMAL(9,1) NULL,
- k11 DECIMALV3(9,1) NULL,
+ k10 DECIMAL(9,1) NULL,
+ k11 DECIMALV3(9,1) NULL,
k12 DATETIME NULL,
k13 DATEV2 NULL,
k14 DATETIMEV2 NULL,
@@ -33,18 +33,26 @@ CREATE TABLE uniq_tbl_basic
kd12 DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
kd13 DATEV2 NOT NULL DEFAULT "2023-08-24",
kd14 DATETIMEV2 NOT NULL DEFAULT CURRENT_TIMESTAMP,
- kd15 CHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
- kd16 VARCHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd15 CHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd16 VARCHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
kd17 STRING NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
kd18 JSON NULL,
- INDEX idx_bitmap_k104 (`k02`) USING BITMAP,
- INDEX idx_bitmap_k110 (`kd01`) USING BITMAP,
+ INDEX idx_inverted_k104 (`k05`) USING INVERTED,
+ INDEX idx_inverted_k110 (`k11`) USING INVERTED,
+ INDEX idx_inverted_k113 (`k13`) USING INVERTED,
+ INDEX idx_inverted_k114 (`k14`) USING INVERTED,
+ INDEX idx_inverted_k117 (`k17`) USING INVERTED PROPERTIES("parser" =
"english"),
+ INDEX idx_bitmap_k104 (`k05`) USING BITMAP,
+ INDEX idx_bitmap_k110 (`k11`) USING BITMAP,
INDEX idx_bitmap_k113 (`k13`) USING BITMAP,
INDEX idx_bitmap_k114 (`k14`) USING BITMAP,
- INDEX idx_bitmap_k117 (`k17`) USING BITMAP
+ INDEX idx_bitmap_k117 (`k17`) USING BITMAP,
+ INDEX idx_ngrambf_k115 (`k15`) USING NGRAM_BF PROPERTIES("gram_size"="3",
"bf_size"="256"),
+ INDEX idx_ngrambf_k116 (`k16`) USING NGRAM_BF PROPERTIES("gram_size"="3",
"bf_size"="256"),
+ INDEX idx_ngrambf_k117 (`k17`) USING NGRAM_BF PROPERTIES("gram_size"="3",
"bf_size"="256")
)
-UNIQUE KEY(k00,k01)
+ UNIQUE KEY(k00,k01)
PARTITION BY RANGE(k01)
(
PARTITION p1 VALUES [('2023-08-01'), ('2023-08-11')),
@@ -53,6 +61,7 @@ PARTITION BY RANGE(k01)
)
DISTRIBUTED BY HASH(k00) BUCKETS 32
PROPERTIES (
- "function_column.sequence_col" = "k12",
- "replication_num" = "1"
-);
\ No newline at end of file
+ "bloom_filter_columns"="k05",
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "true"
+);
diff --git
a/regression-test/suites/load_p0/broker_load/ddl/mow_tbl_basic_drop.sql
b/regression-test/suites/load_p0/broker_load/ddl/mow_tbl_basic_drop.sql
new file mode 100644
index 0000000000..5a446bd3d6
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/mow_tbl_basic_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS mow_tbl_basic
diff --git a/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_array.sql
b/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_array.sql
new file mode 100644
index 0000000000..69e166aa52
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_array.sql
@@ -0,0 +1,43 @@
+CREATE TABLE uniq_tbl_array
+(
+ k00 INT NOT NULL,
+ k01 array<BOOLEAN> NULL,
+ k02 array<TINYINT> NULL,
+ k03 array<SMALLINT> NULL,
+ k04 array<INT> NULL,
+ k05 array<BIGINT> NULL,
+ k06 array<LARGEINT> NULL,
+ k07 array<FLOAT> NULL,
+ k08 array<DOUBLE> NULL,
+ k09 array<DECIMAL> NULL,
+ k10 array<DECIMALV3> NULL,
+ k11 array<DATE> NULL,
+ k12 array<DATETIME> NULL,
+ k13 array<DATEV2> NULL,
+ k14 array<DATETIMEV2> NULL,
+ k15 array<CHAR> NULL,
+ k16 array<VARCHAR> NULL,
+ k17 array<STRING> NULL,
+ kd01 array<BOOLEAN> NOT NULL DEFAULT "[]",
+ kd02 array<TINYINT> NOT NULL DEFAULT "[]",
+ kd03 array<SMALLINT> NOT NULL DEFAULT "[]",
+ kd04 array<INT> NOT NULL DEFAULT "[]",
+ kd05 array<BIGINT> NOT NULL DEFAULT "[]",
+ kd06 array<LARGEINT> NOT NULL DEFAULT "[]",
+ kd07 array<FLOAT> NOT NULL DEFAULT "[]",
+ kd08 array<DOUBLE> NOT NULL DEFAULT "[]",
+ kd09 array<DECIMAL> NOT NULL DEFAULT "[]",
+ kd10 array<DECIMALV3> NOT NULL DEFAULT "[]",
+ kd11 array<DATE> NOT NULL DEFAULT "[]",
+ kd12 array<DATETIME> NOT NULL DEFAULT "[]",
+ kd13 array<DATEV2> NOT NULL DEFAULT "[]",
+ kd14 array<DATETIMEV2> NOT NULL DEFAULT "[]",
+ kd15 array<CHAR> NOT NULL DEFAULT "[]",
+ kd16 array<VARCHAR> NOT NULL DEFAULT "[]",
+ kd17 array<STRING> NOT NULL DEFAULT "[]",
+)
+ UNIQUE KEY(k00)
+DISTRIBUTED BY HASH(k00) BUCKETS 32
+PROPERTIES (
+ "replication_num" = "1"
+);
diff --git
a/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_array_drop.sql
b/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_array_drop.sql
new file mode 100644
index 0000000000..419c9c6d53
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_array_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS uniq_tbl_array
diff --git a/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic.sql
b/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic.sql
index 36d5ac1e05..e111e95721 100644
--- a/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic.sql
+++ b/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic.sql
@@ -55,4 +55,4 @@ DISTRIBUTED BY HASH(k00) BUCKETS 32
PROPERTIES (
"function_column.sequence_col" = "k12",
"replication_num" = "1"
-);
\ No newline at end of file
+);
diff --git
a/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic_drop.sql
b/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic_drop.sql
index 0e82736da9..272bc65549 100644
--- a/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic_drop.sql
+++ b/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic_drop.sql
@@ -1 +1 @@
-DROP TABLE IF EXISTS uniq_tbl_basic
\ No newline at end of file
+DROP TABLE IF EXISTS uniq_tbl_basic
diff --git a/regression-test/suites/load_p0/broker_load/test_s3_load.groovy
b/regression-test/suites/load_p0/broker_load/test_s3_load.groovy
new file mode 100644
index 0000000000..69d193a19b
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/test_s3_load.groovy
@@ -0,0 +1,513 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_s3_load", "load_p0") {
+
+ def tables = [
+ "agg_tbl_basic",
+ "dup_tbl_array",
+ "dup_tbl_basic",
+ "mow_tbl_array",
+ "mow_tbl_basic",
+ "uniq_tbl_array",
+ "uniq_tbl_basic"
+ ]
+
+ //deal with agg tables in separate
+ def basicTables = [
+ "dup_tbl_basic",
+ "mow_tbl_basic",
+ "uniq_tbl_basic",
+ ]
+
+ def arrayTables = [
+ "dup_tbl_array",
+ "uniq_tbl_array",
+ "mow_tbl_array"
+ ]
+
+ def uniqTables = [
+ "uniq_tbl_basic"
+ ]
+
+ def attributesList = [
+
+ ]
+
+ /* ========================================================== normal
========================================================== */
+ for (String table : basicTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "", "", ""))
+ }
+
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\" ", "FORMAT AS \"csv\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", ""))
+
+ for (String table : arrayTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
+ "", "", "", "", ""))
+ }
+
+ for (String table : basicTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(K00,K01,K02,K03,K04,K05,K06,K07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17,K18)",
+ "", "", "", "", ""))
+ }
+
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\" ", "FORMAT AS \"csv\"",
"(K00,K01,K02,K03,K04,K05,K06,k07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17,K18)",
+ "", "", "SET
(K19=to_bitmap(k04),K20=HLL_HASH(k04),K21=TO_QUANTILE_STATE(K04,1.0),Kd19=to_bitmap(K05),kd20=HLL_HASH(K05),KD21=TO_QUANTILE_STATE(K05,1.0))",
"", ""))
+
+ for (String table : arrayTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(K00,K01,K02,K03,K04,K05,K06,K07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17)",
+ "", "", "", "", ""))
+ }
+ // TODO: should be success ?
+// for (String table : basicTables) {
+// attributesList.add(new
LoadAttributes("s3://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_data.csv",
+// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+// "", "", "", "", "").withPathStyle())
+// }
+//
+// attributesList.add(new
LoadAttributes("s3://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_data.csv",
+// "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\" ", "FORMAT AS \"csv\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+// "", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", "").withPathStyle())
+//
+// for (String table : arrayTables) {
+// attributesList.add(new
LoadAttributes("s3://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_array_data.csv",
+// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
+// "", "", "", "", "").withPathStyle())
+// }
+//
+// for (String table : basicTables) {
+// attributesList.add(new
LoadAttributes("s3://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_data.csv",
+// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+// "", "", "", "", "").withPathStyle())
+// }
+//
+// attributesList.add(new
LoadAttributes("s3://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_data.csv",
+// "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\" ", "FORMAT AS \"csv\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+// "", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", "").withPathStyle())
+//
+// for (String table : arrayTables) {
+// attributesList.add(new
LoadAttributes("s3://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_array_data.csv",
+// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
+// "", "", "", "", "").withPathStyle())
+// }
+
+ /* ========================================================== error
========================================================== */
+ for (String table : basicTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "", "", "", true))
+ }
+
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+ "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\" ", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", "", true))
+
+ for (String table : arrayTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data_with_errors.csv",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"csv\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
+ "", "", "", "", "", true))
+ }
+
+// has problem, should be success
+// for(String table: basicTables) {
+// attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+// "", "", "", "","").addProperties("max_filter_ratio", "0.5"))
+// }
+//
+// attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+// "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\" ", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+// "", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", "").addProperties("max_filter_ratio", "0.5"))
+//
+// for(String table : arrayTables) {
+// attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data_with_errors.csv",
+// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
+// "", "", "", "","").addProperties("max_filter_ratio", "0.5"))
+// }
+
+// for(String table: basicTables) {
+// attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+// "", "", "", "","", true).addProperties("max_filter_ratio",
"0.4"))
+// }
+//
+// attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+// "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\" ", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+// "", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", "", true).addProperties("max_filter_ratio", "0.4"))
+//
+// for(String table : arrayTables) {
+// attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data_with_errors.csv",
+// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
+// "", "", "", "","", true).addProperties("max_filter_ratio",
"0.4"))
+// }
+
+ // skip lines
+// for(String table: basicTables) {
+// attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+// "", "", "", "","").addProperties("skip_lines", "10"))
+// }
+//
+// attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+// "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\" ", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+// "", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", "").addProperties("skip_lines", "10"))
+//
+// for(String table : arrayTables) {
+// attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data_with_errors.csv",
+// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
+// "", "", "", "","").addProperties("skip_lines", "10"))
+// }
+
+ /* ========================================================== wrong column
sep ========================================================== */
+ for (String table : basicTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \",\"", "FORMAT AS \"csv\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "", "", "", true))
+ }
+
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \",\" ", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", "", true))
+
+ for (String table : arrayTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \",\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
+ "", "", "", "", "", true))
+ }
+
+ /* ========================================================== wrong line
delim ========================================================== */
+ for (String table : basicTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ "${table}", "LINES TERMINATED BY \"\t\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "", "", "", true))
+ }
+
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ "agg_tbl_basic", "LINES TERMINATED BY \"\t\"", "COLUMNS TERMINATED
BY \"|\" ", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", "", true))
+
+ for (String table : arrayTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv",
+ "${table}", "LINES TERMINATED BY \"\t\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
+ "", "", "", "", "", true))
+ }
+
+ /* ========================================================== strict mode
========================================================== */
+ for(String table: basicTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "", "","", true).addProperties("strict_mode", "true"))
+ }
+
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+ "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\" ", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", "", true).addProperties("strict_mode","true"))
+
+ for(String table : arrayTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data_with_errors.csv",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
+ "", "", "", "","", true).addProperties("strict_mode", "true"))
+ }
+
+ /* ========================================================== timezone
========================================================== */
+
+ for(String table: basicTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "SET (k00=unix_timestamp('2023-09-01 12:00:00'))",
"","").addProperties("timezone", "Asia/Shanghai"))
+ }
+
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\" ", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "SET (k00=unix_timestamp('2023-09-01
12:00:00'),k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", "").addProperties("timezone", "Asia/Shanghai"))
+
+ for(String table : arrayTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
+ "", "", "SET (k00=unix_timestamp('2023-09-01 12:00:00'))",
"","").addProperties("timezone", "Asia/Shanghai"))
+ }
+
+ for(String table: basicTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "SET (k00=unix_timestamp('2023-09-01 12:00:00'))",
"","").addProperties("timezone", "America/Chicago"))
+ }
+
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\" ", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "SET (k00=unix_timestamp('2023-09-01
12:00:00'),k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", "").addProperties("timezone", "America/Chicago"))
+
+ for(String table : arrayTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
+ "", "", "SET (k00=unix_timestamp('2023-09-01 12:00:00'))",
"","").addProperties("timezone", "America/Chicago"))
+ }
+
+ /* ========================================================== compress
type ========================================================== */
+ for (String table : basicTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "", "", ""))
+ }
+
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz",
+ "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\" ", "FORMAT AS \"csv\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", ""))
+
+ for (String table : arrayTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv.gz",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
+ "", "", "", "", ""))
+ }
+
+ for (String table : basicTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "", "", ""))
+ }
+
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2",
+ "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\" ", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", ""))
+
+ for (String table : arrayTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv.bz2",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
+ "", "", "", "", ""))
+ }
+
+ for (String table : basicTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "", "", ""))
+ }
+
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4",
+ "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\" ", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", ""))
+
+ for (String table : arrayTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv.lz4",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
+ "", "", "", "", ""))
+ }
+
+
+ for (String table : basicTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "", "", ""))
+ }
+
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz",
+ "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\" ", "",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", ""))
+
+ for (String table : arrayTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv.gz",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
+ "", "", "", "", ""))
+ }
+
+ for (String table : basicTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "", "", ""))
+ }
+
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2",
+ "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\" ", "",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", ""))
+
+ for (String table : arrayTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv.bz2",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
+ "", "", "", "", ""))
+ }
+
+ for (String table : basicTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "", "", ""))
+ }
+
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4",
+ "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\" ", "",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", ""))
+
+ for (String table : arrayTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv.lz4",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
+ "", "", "", "", ""))
+ }
+
+ /*========================================================== order by
==========================================================*/
+
+ for (String table : uniqTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "", "", "ORDER BY k01"))
+ }
+
+ def ak = getS3AK()
+ def sk = getS3SK()
+
+ for(String tbl : tables) {
+ sql new File("""${ context.file.parent }/ddl/${tbl}_drop.sql""").text
+ sql new File("""${ context.file.parent }/ddl/${tbl}.sql""").text
+ }
+
+ def i = 0
+ for (LoadAttributes attributes : attributesList) {
+ def label = "test_s3_load_" +
UUID.randomUUID().toString().replace("-", "_") + "_" + i
+ attributes.label = label
+ def prop = attributes.getPropertiesStr()
+
+ def sql_str = """
+ LOAD LABEL $label (
+ DATA INFILE("$attributes.dataDesc.path")
+ INTO TABLE $attributes.dataDesc.tableName
+ $attributes.dataDesc.columnTermClause
+ $attributes.dataDesc.lineTermClause
+ $attributes.dataDesc.formatClause
+ $attributes.dataDesc.columns
+ $attributes.dataDesc.columnsFromPathClause
+ $attributes.dataDesc.columnMappingClause
+ $attributes.dataDesc.precedingFilterClause
+ $attributes.dataDesc.orderByClause
+ $attributes.dataDesc.whereExpr
+ )
+ WITH S3 (
+ "AWS_ACCESS_KEY" = "$ak",
+ "AWS_SECRET_KEY" = "$sk",
+ "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
+ "AWS_REGION" = "ap-beijing",
+ "use_path_style" = "$attributes.usePathStyle"
+ )
+ ${prop}
+ """
+ logger.info("submit sql: ${sql_str}");
+ sql """${sql_str}"""
+ logger.info("Submit load with lable: $label, table:
$attributes.dataDesc.tableName, path: $attributes.dataDesc.path")
+ ++i
+ }
+
+ for (LoadAttributes attributes : attributesList) {
+ def max_try_milli_secs = 600000
+ while (max_try_milli_secs > 0) {
+ String[][] result = sql """ show load where
label="$attributes.label" order by createtime desc limit 1; """
+ if (result[0][2].equals("FINISHED")) {
+ if (attributes.isExceptFailed) {
+ assertTrue(false, "load should be failed but was success:
$result")
+ }
+ logger.info("Load FINISHED " + attributes.label + ": $result")
+ break
+ }
+ if (result[0][2].equals("CANCELLED")) {
+ if (attributes.isExceptFailed) {
+ logger.info("Load FINISHED " + attributes.label)
+ break
+ }
+ assertTrue(false, "load failed: $result")
+ break
+ }
+ Thread.sleep(1000)
+ max_try_milli_secs -= 1000
+ if (max_try_milli_secs <= 0) {
+ assertTrue(false, "load Timeout: $attributes.label")
+ }
+ }
+ }
+
+ for(String tbl : tables) {
+ qt_select """ select count(*) from ${tbl} """
+ }
+}
+
+class DataDesc {
+ public String path
+ public String tableName
+ public String lineTermClause
+ public String columnTermClause
+ public String formatClause
+ public String columns
+ public String columnsFromPathClause
+ public String precedingFilterClause
+ public String columnMappingClause
+ public String whereExpr
+ public String orderByClause
+}
+
+class LoadAttributes {
+ LoadAttributes(String path, String tableName, String lineTermClause,
String columnTermClause, String formatClause,
+ String columns, String columnsFromPathClause, String
precedingFilterClause, String columnMappingClause, String whereExpr, String
orderByClause, boolean isExceptFailed = false) {
+ this.dataDesc = new DataDesc()
+ this.dataDesc.path = path
+ this.dataDesc.tableName = tableName
+ this.dataDesc.lineTermClause = lineTermClause
+ this.dataDesc.columnTermClause = columnTermClause
+ this.dataDesc.formatClause = formatClause
+ this.dataDesc.columns = columns
+ this.dataDesc.columnsFromPathClause = columnsFromPathClause
+ this.dataDesc.precedingFilterClause = precedingFilterClause
+ this.dataDesc.columnMappingClause = columnMappingClause
+ this.dataDesc.whereExpr = whereExpr
+ this.dataDesc.orderByClause = orderByClause
+
+ this.isExceptFailed = isExceptFailed
+
+ properties = new HashMap<>()
+ properties.put("use_new_load_scan_node", "true")
+ }
+
+ LoadAttributes addProperties(String k, String v) {
+ properties.put(k, v)
+ return this
+ }
+
+ String getPropertiesStr() {
+ if (properties.isEmpty()) {
+ return ""
+ }
+ String prop = "PROPERTIES ("
+ properties.forEach (k, v) -> {
+ prop += "\"${k}\" = \"${v}\","
+ }
+ prop = prop.substring(0, prop.size() - 1)
+ prop += ")"
+ return prop
+ }
+
+ LoadAttributes withPathStyle() {
+ usePathStyle = "true"
+ return this
+ }
+
+ public DataDesc dataDesc
+ public Map<String, String> properties
+ public String label
+ public String usePathStyle = "false"
+ public boolean isExceptFailed
+}
diff --git a/regression-test/suites/load_p0/broker_load/test_seq_load.groovy
b/regression-test/suites/load_p0/broker_load/test_seq_load.groovy
index 96040c58b6..afb955cb32 100644
--- a/regression-test/suites/load_p0/broker_load/test_seq_load.groovy
+++ b/regression-test/suites/load_p0/broker_load/test_seq_load.groovy
@@ -17,17 +17,77 @@
suite("test_seq_load", "load_p0") {
- def tableName = "uniq_tbl_basic"
+ def tableName = "uniq_tbl_basic_seq"
- sql new File("""${ context.file.parent
}/ddl/uniq_tbl_basic_drop.sql""").text
- sql new File("""${ context.file.parent }/ddl/uniq_tbl_basic.sql""").text
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+
+ sql """
+ CREATE TABLE ${tableName}
+ (
+ k00 INT NOT NULL,
+ k01 DATE NOT NULL,
+ k02 BOOLEAN NULL,
+ k03 TINYINT NULL,
+ k04 SMALLINT NULL,
+ k05 INT NULL,
+ k06 BIGINT NULL,
+ k07 LARGEINT NULL,
+ k08 FLOAT NULL,
+ k09 DOUBLE NULL,
+ k10 DECIMAL(9,1) NULL,
+ k11 DECIMALV3(9,1) NULL,
+ k12 DATETIME NULL,
+ k13 DATEV2 NULL,
+ k14 DATETIMEV2 NULL,
+ k15 CHAR NULL,
+ k16 VARCHAR NULL,
+ k17 STRING NULL,
+ k18 JSON NULL,
+ kd01 BOOLEAN NOT NULL DEFAULT "TRUE",
+ kd02 TINYINT NOT NULL DEFAULT "1",
+ kd03 SMALLINT NOT NULL DEFAULT "2",
+ kd04 INT NOT NULL DEFAULT "3",
+ kd05 BIGINT NOT NULL DEFAULT "4",
+ kd06 LARGEINT NOT NULL DEFAULT "5",
+ kd07 FLOAT NOT NULL DEFAULT "6.0",
+ kd08 DOUBLE NOT NULL DEFAULT "7.0",
+ kd09 DECIMAL NOT NULL DEFAULT "888888888",
+ kd10 DECIMALV3 NOT NULL DEFAULT "999999999",
+ kd11 DATE NOT NULL DEFAULT "2023-08-24",
+ kd12 DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ kd13 DATEV2 NOT NULL DEFAULT "2023-08-24",
+ kd14 DATETIMEV2 NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ kd15 CHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd16 VARCHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd17 STRING NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd18 JSON NULL,
+
+ INDEX idx_bitmap_k104 (`k02`) USING BITMAP,
+ INDEX idx_bitmap_k110 (`kd01`) USING BITMAP,
+ INDEX idx_bitmap_k113 (`k13`) USING BITMAP,
+ INDEX idx_bitmap_k114 (`k14`) USING BITMAP,
+ INDEX idx_bitmap_k117 (`k17`) USING BITMAP
+ )
+ UNIQUE KEY(k00,k01)
+ PARTITION BY RANGE(k01)
+ (
+ PARTITION p1 VALUES [('2023-08-01'), ('2023-08-11')),
+ PARTITION p2 VALUES [('2023-08-11'), ('2023-08-21')),
+ PARTITION p3 VALUES [('2023-08-21'), ('2023-09-01'))
+ )
+ DISTRIBUTED BY HASH(k00) BUCKETS 32
+ PROPERTIES (
+ "function_column.sequence_col" = "k12",
+ "replication_num" = "1"
+ );
+ """
def label = UUID.randomUUID().toString().replace("-", "0")
def path =
"s3://doris-build-1308700295/regression/load/data/basic_data.csv"
def format_str = "CSV"
def ak = getS3AK()
def sk = getS3SK()
- def seq_column = "K01"
+ def seq_column = "K12"
def sql_str = """
LOAD LABEL $label (
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
