This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 984d387945 [Regression](load) Add broker load regression test. (#13062)
984d387945 is described below
commit 984d387945b7b546892e42730216f7a8c3ddc61b
Author: Jibing-Li <[email protected]>
AuthorDate: Tue Oct 4 21:29:05 2022 +0800
[Regression](load) Add broker load regression test. (#13062)
Add basic broker load regression test. It has been tested. But default
---
be/src/common/config.h | 4 +-
regression-test/conf/regression-conf.groovy | 3 +
.../org/apache/doris/regression/suite/Suite.groovy | 7 +-
.../load_p0/broker_load/ddl/filter_create.sql | 17 +++
.../suites/load_p0/broker_load/ddl/filter_drop.sql | 1 +
.../broker_load/ddl/null_default_create.sql | 21 ++++
.../load_p0/broker_load/ddl/null_default_drop.sql | 1 +
.../suites/load_p0/broker_load/ddl/part_create.sql | 15 +++
.../suites/load_p0/broker_load/ddl/part_drop.sql | 1 +
.../load_p0/broker_load/ddl/path_column_create.sql | 18 +++
.../load_p0/broker_load/ddl/path_column_drop.sql | 1 +
.../load_p0/broker_load/ddl/reverse_create.sql | 15 +++
.../load_p0/broker_load/ddl/reverse_drop.sql | 1 +
.../suites/load_p0/broker_load/ddl/set1_create.sql | 10 ++
.../suites/load_p0/broker_load/ddl/set1_drop.sql | 1 +
.../suites/load_p0/broker_load/ddl/set2_create.sql | 7 ++
.../suites/load_p0/broker_load/ddl/set2_drop.sql | 1 +
.../suites/load_p0/broker_load/ddl/set3_create.sql | 7 ++
.../suites/load_p0/broker_load/ddl/set3_drop.sql | 1 +
.../suites/load_p0/broker_load/ddl/set4_create.sql | 7 ++
.../suites/load_p0/broker_load/ddl/set4_drop.sql | 1 +
.../suites/load_p0/broker_load/ddl/set5_create.sql | 7 ++
.../suites/load_p0/broker_load/ddl/set5_drop.sql | 1 +
.../suites/load_p0/broker_load/ddl/set6_create.sql | 8 ++
.../suites/load_p0/broker_load/ddl/set6_drop.sql | 1 +
.../suites/load_p0/broker_load/ddl/set7_create.sql | 8 ++
.../suites/load_p0/broker_load/ddl/set7_drop.sql | 1 +
.../load_p0/broker_load/ddl/upper_case_create.sql | 15 +++
.../load_p0/broker_load/ddl/upper_case_drop.sql | 1 +
.../load_p0/broker_load/test_broker_load.groovy | 139 ++++++++++++++++++++-
30 files changed, 315 insertions(+), 6 deletions(-)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 29985ae31e..8ee1649378 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -883,10 +883,10 @@ CONF_mInt64(nodechannel_pending_queue_max_bytes,
"67108864");
CONF_mInt32(max_fragment_start_wait_time_seconds, "30");
// Temp config. True to use new file scan node to do load job. Will remove
after fully test.
-CONF_Bool(enable_new_load_scan_node, "false");
+CONF_mBool(enable_new_load_scan_node, "false");
// Temp config. True to use new file scanner. Will remove after fully test.
-CONF_Bool(enable_new_file_scanner, "false");
+CONF_mBool(enable_new_file_scanner, "false");
#ifdef BE_TEST
// test s3
diff --git a/regression-test/conf/regression-conf.groovy
b/regression-test/conf/regression-conf.groovy
index db5ac4e937..170ae8cca5 100644
--- a/regression-test/conf/regression-conf.groovy
+++ b/regression-test/conf/regression-conf.groovy
@@ -60,3 +60,6 @@ hdfsFs = "hdfs://127.0.0.1:9000"
hdfsUser = "doris-test"
hdfsPasswd = ""
brokerName = "broker_name"
+
+// broker load test config
+enableBrokerLoad=false
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index ff9a66fd34..3bf93105df 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -286,7 +286,7 @@ class Suite implements GroovyInterceptable {
}
boolean enableHdfs() {
- String enableHdfs = context.config.otherConfigs.get("enableHdfs");
+ String enableHdfs = context.config.otherConfigs.get("enableHdfs");
return enableHdfs.equals("true");
}
@@ -300,6 +300,11 @@ class Suite implements GroovyInterceptable {
return remotePath;
}
+ boolean enableBrokerLoad() {
+ String enableBrokerLoad =
context.config.otherConfigs.get("enableBrokerLoad");
+ return (enableBrokerLoad != null && enableBrokerLoad.equals("true"));
+ }
+
String getS3Region() {
String s3Region = context.config.otherConfigs.get("s3Region");
return s3Region
diff --git a/regression-test/suites/load_p0/broker_load/ddl/filter_create.sql
b/regression-test/suites/load_p0/broker_load/ddl/filter_create.sql
new file mode 100644
index 0000000000..3e3f1ec39e
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/filter_create.sql
@@ -0,0 +1,17 @@
+CREATE TABLE IF NOT EXISTS filter (
+ p_partkey int NOT NULL,
+ p_name VARCHAR(55) NOT NULL,
+ p_mfgr VARCHAR(25) NOT NULL,
+ p_brand VARCHAR(10) NOT NULL,
+ p_type VARCHAR(25) NOT NULL,
+ p_size int NOT NULL,
+ p_container VARCHAR(10) NOT NULL,
+ p_retailprice decimal(15, 2) NOT NULL,
+ p_comment VARCHAR(23) NOT NULL
+)ENGINE=OLAP
+DUPLICATE KEY(`p_partkey`)
+COMMENT "OLAP"
+DISTRIBUTED BY HASH(`p_partkey`) BUCKETS 24
+PROPERTIES (
+ "replication_num" = "1"
+);
diff --git a/regression-test/suites/load_p0/broker_load/ddl/filter_drop.sql
b/regression-test/suites/load_p0/broker_load/ddl/filter_drop.sql
new file mode 100644
index 0000000000..6583355507
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/filter_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS filter;
diff --git
a/regression-test/suites/load_p0/broker_load/ddl/null_default_create.sql
b/regression-test/suites/load_p0/broker_load/ddl/null_default_create.sql
new file mode 100644
index 0000000000..96c0b4303f
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/null_default_create.sql
@@ -0,0 +1,21 @@
+CREATE TABLE IF NOT EXISTS null_default (
+ p_partkey int NOT NULL,
+ p_name VARCHAR(55) NOT NULL,
+ p_mfgr VARCHAR(25) NOT NULL,
+ p_brand VARCHAR(10) NOT NULL,
+ p_type VARCHAR(25) NOT NULL,
+ p_size int NOT NULL,
+ p_container VARCHAR(10) NOT NULL,
+ p_retailprice decimal(15, 2) NOT NULL,
+ p_comment VARCHAR(23) NOT NULL,
+ p_add_col1 varchar(5) null,
+ p_add_col2 int null,
+ p_add_col3 int null default '100',
+ p_add_col4 varchar(5) null default 'abc'
+)ENGINE=OLAP
+DUPLICATE KEY(`p_partkey`)
+COMMENT "OLAP"
+DISTRIBUTED BY HASH(`p_partkey`) BUCKETS 24
+PROPERTIES (
+ "replication_num" = "1"
+);
diff --git
a/regression-test/suites/load_p0/broker_load/ddl/null_default_drop.sql
b/regression-test/suites/load_p0/broker_load/ddl/null_default_drop.sql
new file mode 100644
index 0000000000..158562ba01
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/null_default_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS null_default;
diff --git a/regression-test/suites/load_p0/broker_load/ddl/part_create.sql
b/regression-test/suites/load_p0/broker_load/ddl/part_create.sql
new file mode 100644
index 0000000000..51e699ed81
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/part_create.sql
@@ -0,0 +1,15 @@
+CREATE TABLE IF NOT EXISTS part (
+ `p_partkey` integer NOT NULL,
+ `p_name` varchar(55) NOT NULL,
+ `p_mfgr` char(25) NOT NULL,
+ `p_brand` char(10) NOT NULL,
+ `p_type` varchar(25) NOT NULL,
+ `p_size` integer NOT NULL,
+ `p_container` char(10) NOT NULL,
+ `p_retailprice` decimal(12, 2) NOT NULL,
+ `p_comment` varchar(23) NOT NULL
+)ENGINE=OLAP
+DISTRIBUTED BY HASH(p_partkey) BUCKETS 24
+PROPERTIES (
+ "replication_num" = "1"
+);
diff --git a/regression-test/suites/load_p0/broker_load/ddl/part_drop.sql
b/regression-test/suites/load_p0/broker_load/ddl/part_drop.sql
new file mode 100644
index 0000000000..ffd55b2078
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/part_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS part;
diff --git
a/regression-test/suites/load_p0/broker_load/ddl/path_column_create.sql
b/regression-test/suites/load_p0/broker_load/ddl/path_column_create.sql
new file mode 100644
index 0000000000..6de4deff4f
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/path_column_create.sql
@@ -0,0 +1,18 @@
+CREATE TABLE IF NOT EXISTS path_column (
+ p_partkey int NOT NULL,
+ p_name VARCHAR(55) NOT NULL,
+ p_mfgr VARCHAR(25) NOT NULL,
+ p_brand VARCHAR(10) NOT NULL,
+ p_type VARCHAR(25) NOT NULL,
+ p_size int NOT NULL,
+ p_container VARCHAR(10) NOT NULL,
+ p_retailprice decimal(15, 2) NOT NULL,
+ p_comment VARCHAR(23) NOT NULL,
+ city varchar(10) NOT NULL
+)ENGINE=OLAP
+DUPLICATE KEY(`p_partkey`)
+COMMENT "OLAP"
+DISTRIBUTED BY HASH(`p_partkey`) BUCKETS 24
+PROPERTIES (
+ "replication_num" = "1"
+);
diff --git
a/regression-test/suites/load_p0/broker_load/ddl/path_column_drop.sql
b/regression-test/suites/load_p0/broker_load/ddl/path_column_drop.sql
new file mode 100644
index 0000000000..a2e23b1eed
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/path_column_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS path_column;
diff --git a/regression-test/suites/load_p0/broker_load/ddl/reverse_create.sql
b/regression-test/suites/load_p0/broker_load/ddl/reverse_create.sql
new file mode 100644
index 0000000000..d68b73a448
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/reverse_create.sql
@@ -0,0 +1,15 @@
+CREATE TABLE IF NOT EXISTS reverse (
+ `comment` varchar(23) NOT NULL,
+ `retailprice` decimal(12, 2) NOT NULL,
+ `container` char(10) NOT NULL,
+ `size` integer NOT NULL,
+ `type` varchar(25) NOT NULL,
+ `brand` char(10) NOT NULL,
+ `mfgr` char(25) NOT NULL,
+ `name` varchar(55) NOT NULL,
+ `partkey` integer NOT NULL
+)ENGINE=OLAP
+DISTRIBUTED BY HASH(partkey) BUCKETS 24
+PROPERTIES (
+ "replication_num" = "1"
+);
diff --git a/regression-test/suites/load_p0/broker_load/ddl/reverse_drop.sql
b/regression-test/suites/load_p0/broker_load/ddl/reverse_drop.sql
new file mode 100644
index 0000000000..80e1c0997f
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/reverse_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS reverse;
diff --git a/regression-test/suites/load_p0/broker_load/ddl/set1_create.sql
b/regression-test/suites/load_p0/broker_load/ddl/set1_create.sql
new file mode 100644
index 0000000000..4e44de6a0f
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/set1_create.sql
@@ -0,0 +1,10 @@
+CREATE TABLE IF NOT EXISTS set1 (
+ `p_partkey` integer NOT NULL default '0',
+ `p_name` varchar(55) NOT NULL default 'aaa',
+ `p_size` integer NOT NULL default '0',
+ `p_greatest` integer NOT NULL
+)ENGINE=OLAP
+DISTRIBUTED BY HASH(p_partkey) BUCKETS 24
+PROPERTIES (
+ "replication_num" = "1"
+);
diff --git a/regression-test/suites/load_p0/broker_load/ddl/set1_drop.sql
b/regression-test/suites/load_p0/broker_load/ddl/set1_drop.sql
new file mode 100644
index 0000000000..be82c99e46
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/set1_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS set1;
diff --git a/regression-test/suites/load_p0/broker_load/ddl/set2_create.sql
b/regression-test/suites/load_p0/broker_load/ddl/set2_create.sql
new file mode 100644
index 0000000000..ff3b7af1d0
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/set2_create.sql
@@ -0,0 +1,7 @@
+CREATE TABLE IF NOT EXISTS set2 (
+ `p_partkey` integer NOT NULL default '0'
+)ENGINE=OLAP
+DISTRIBUTED BY HASH(p_partkey) BUCKETS 24
+PROPERTIES (
+ "replication_num" = "1"
+);
diff --git a/regression-test/suites/load_p0/broker_load/ddl/set2_drop.sql
b/regression-test/suites/load_p0/broker_load/ddl/set2_drop.sql
new file mode 100644
index 0000000000..d0f144a2a4
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/set2_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS set2;
diff --git a/regression-test/suites/load_p0/broker_load/ddl/set3_create.sql
b/regression-test/suites/load_p0/broker_load/ddl/set3_create.sql
new file mode 100644
index 0000000000..361b1e0de2
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/set3_create.sql
@@ -0,0 +1,7 @@
+CREATE TABLE IF NOT EXISTS set3 (
+ `partkey` integer NOT NULL default '0'
+)ENGINE=OLAP
+DISTRIBUTED BY HASH(partkey) BUCKETS 24
+PROPERTIES (
+ "replication_num" = "1"
+);
diff --git a/regression-test/suites/load_p0/broker_load/ddl/set3_drop.sql
b/regression-test/suites/load_p0/broker_load/ddl/set3_drop.sql
new file mode 100644
index 0000000000..50c812fc70
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/set3_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS set3;
diff --git a/regression-test/suites/load_p0/broker_load/ddl/set4_create.sql
b/regression-test/suites/load_p0/broker_load/ddl/set4_create.sql
new file mode 100644
index 0000000000..15c9897304
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/set4_create.sql
@@ -0,0 +1,7 @@
+CREATE TABLE IF NOT EXISTS set4 (
+ `partkey` integer NOT NULL default '0'
+)ENGINE=OLAP
+DISTRIBUTED BY HASH(partkey) BUCKETS 24
+PROPERTIES (
+ "replication_num" = "1"
+);
diff --git a/regression-test/suites/load_p0/broker_load/ddl/set4_drop.sql
b/regression-test/suites/load_p0/broker_load/ddl/set4_drop.sql
new file mode 100644
index 0000000000..f47d6eb4f6
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/set4_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS set4;
diff --git a/regression-test/suites/load_p0/broker_load/ddl/set5_create.sql
b/regression-test/suites/load_p0/broker_load/ddl/set5_create.sql
new file mode 100644
index 0000000000..e6c17c8374
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/set5_create.sql
@@ -0,0 +1,7 @@
+CREATE TABLE IF NOT EXISTS set5 (
+ `partkey` integer NOT NULL default '0'
+)ENGINE=OLAP
+DISTRIBUTED BY HASH(partkey) BUCKETS 24
+PROPERTIES (
+ "replication_num" = "1"
+);
diff --git a/regression-test/suites/load_p0/broker_load/ddl/set5_drop.sql
b/regression-test/suites/load_p0/broker_load/ddl/set5_drop.sql
new file mode 100644
index 0000000000..1585ffeef6
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/set5_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS set5;
diff --git a/regression-test/suites/load_p0/broker_load/ddl/set6_create.sql
b/regression-test/suites/load_p0/broker_load/ddl/set6_create.sql
new file mode 100644
index 0000000000..828347d40d
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/set6_create.sql
@@ -0,0 +1,8 @@
+CREATE TABLE IF NOT EXISTS set6 (
+ `partkey` integer NOT NULL default '0',
+ `partsize` integer NOT NULL default '0'
+)ENGINE=OLAP
+DISTRIBUTED BY HASH(partkey) BUCKETS 24
+PROPERTIES (
+ "replication_num" = "1"
+);
diff --git a/regression-test/suites/load_p0/broker_load/ddl/set6_drop.sql
b/regression-test/suites/load_p0/broker_load/ddl/set6_drop.sql
new file mode 100644
index 0000000000..912b5bd78f
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/set6_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS set6;
diff --git a/regression-test/suites/load_p0/broker_load/ddl/set7_create.sql
b/regression-test/suites/load_p0/broker_load/ddl/set7_create.sql
new file mode 100644
index 0000000000..5ae82d598f
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/set7_create.sql
@@ -0,0 +1,8 @@
+CREATE TABLE IF NOT EXISTS set7 (
+ `p_partkey` integer NOT NULL default '0',
+ `partsize` integer NOT NULL default '0'
+)ENGINE=OLAP
+DISTRIBUTED BY HASH(p_partkey) BUCKETS 24
+PROPERTIES (
+ "replication_num" = "1"
+);
diff --git a/regression-test/suites/load_p0/broker_load/ddl/set7_drop.sql
b/regression-test/suites/load_p0/broker_load/ddl/set7_drop.sql
new file mode 100644
index 0000000000..b1169625a1
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/set7_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS set7;
diff --git
a/regression-test/suites/load_p0/broker_load/ddl/upper_case_create.sql
b/regression-test/suites/load_p0/broker_load/ddl/upper_case_create.sql
new file mode 100644
index 0000000000..dcfa3ce2f7
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/upper_case_create.sql
@@ -0,0 +1,15 @@
+CREATE TABLE IF NOT EXISTS upper_case (
+ `P_PARTKEY` integer NOT NULL,
+ `P_NAME` varchar(55) NOT NULL,
+ `P_MFGR` char(25) NOT NULL,
+ `P_BRAND` char(10) NOT NULL,
+ `P_TYPE` varchar(25) NOT NULL,
+ `P_SIZE` integer NOT NULL,
+ `P_CONTAINER` char(10) NOT NULL,
+ `P_RETAILPRICE` decimal(12, 2) NOT NULL,
+ `P_COMMENT` varchar(23) NOT NULL
+)ENGINE=OLAP
+DISTRIBUTED BY HASH(p_partkey) BUCKETS 24
+PROPERTIES (
+ "replication_num" = "1"
+);
diff --git a/regression-test/suites/load_p0/broker_load/ddl/upper_case_drop.sql
b/regression-test/suites/load_p0/broker_load/ddl/upper_case_drop.sql
new file mode 100644
index 0000000000..6e1a793776
--- /dev/null
+++ b/regression-test/suites/load_p0/broker_load/ddl/upper_case_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS upper_case;
diff --git a/regression-test/suites/load_p0/broker_load/test_broker_load.groovy
b/regression-test/suites/load_p0/broker_load/test_broker_load.groovy
index a03d4ad1ec..8cfc912a40 100644
--- a/regression-test/suites/load_p0/broker_load/test_broker_load.groovy
+++ b/regression-test/suites/load_p0/broker_load/test_broker_load.groovy
@@ -15,7 +15,140 @@
// specific language governing permissions and limitations
// under the License.
-suite("test_broker_load") {
- // todo: test broker load/cancel, need broker & hdfs or others
- sql "show load"
+suite("test_broker_load", "p0") {
+
+ def tables = ["part", "upper_case", "reverse", "set1", "set2", "set3",
"set4", "set5", "set6",
+ "set7", "null_default", "filter", "path_column"]
+ def paths = ["s3://doris-community-test-1308700295/load/data/part*",
+ "s3://doris-community-test-1308700295/load/data/part*",
+ "s3://doris-community-test-1308700295/load/data/part*",
+ "s3://doris-community-test-1308700295/load/data/part*",
+ "s3://doris-community-test-1308700295/load/data/part*",
+ "s3://doris-community-test-1308700295/load/data/part*",
+ "s3://doris-community-test-1308700295/load/data/part*",
+ "s3://doris-community-test-1308700295/load/data/part*",
+ "s3://doris-community-test-1308700295/load/data/part*",
+ "s3://doris-community-test-1308700295/load/data/part*",
+ "s3://doris-community-test-1308700295/load/data/part*",
+ "s3://doris-community-test-1308700295/load/data/part*",
+ "s3://doris-community-test-1308700295/load/data/path/*/part*"
+ ]
+ def columns_list = ["""p_partkey, p_name, p_mfgr, p_brand, p_type, p_size,
p_container, p_retailprice, p_comment""",
+ """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size,
p_container, p_retailprice, p_comment""",
+ """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size,
p_container, p_retailprice, p_comment""",
+ """p_partkey, p_name, p_size""",
+ """p_partkey""",
+ """p_partkey""",
+ """p_partkey, p_size""",
+ """p_partkey""",
+ """p_partkey, p_size""",
+ """p_partkey, p_size""",
+ """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size,
p_container, p_retailprice, p_comment""",
+ """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size,
p_container, p_retailprice, p_comment""",
+ """p_partkey, p_name, p_mfgr, p_brand, p_type, p_size,
p_container, p_retailprice, p_comment"""]
+ def column_in_paths = ["", "", "", "", "", "", "", "", "", "", "", "",
"COLUMNS FROM PATH AS (city)"]
+ def preceding_filters = ["", "", "", "", "", "", "", "", "", "", "",
"preceding filter p_size < 10", ""]
+ def set_values = ["", "",
+ "SET(comment=p_comment, retailprice=p_retailprice,
container=p_container, size=p_size, type=p_type, brand=p_brand, mfgr=p_mfgr,
name=p_name, partkey=p_partkey)",
+
"set(p_name=upper(p_name),p_greatest=greatest(cast(p_partkey as int),
cast(p_size as int)))",
+ "set(p_partkey = p_partkey + 100)",
+ "set(partkey = p_partkey + 100)",
+ "set(partkey = p_partkey + p_size)",
+ "set(tmpk = p_partkey + 1, partkey = tmpk*2)",
+ "set(partkey = p_partkey + 1, partsize = p_size*2)",
+ "set(partsize = p_partkey + p_size)", "", "", ""
+ ]
+ def where_exprs = ["", "", "", "", "", "", "", "", "", "", "", "where
p_partkey>10", ""]
+
+ String ak = getS3AK()
+ String sk = getS3SK()
+ String enabled = context.config.otherConfigs.get("enableBrokerLoad")
+
+ def do_load_job = {uuid, path, table, columns, column_in_path,
preceding_filter,
+ set_value, where_expr ->
+ sql """
+ LOAD LABEL $uuid (
+ DATA INFILE("$path")
+ INTO TABLE $table
+ FORMAT AS "PARQUET"
+ ($columns)
+ $column_in_path
+ $preceding_filter
+ $set_value
+ $where_expr
+ )
+ WITH S3 (
+ "AWS_ACCESS_KEY" = "$ak",
+ "AWS_SECRET_KEY" = "$sk",
+ "AWS_ENDPOINT" = "cos.ap-hongkong.myqcloud.com",
+ "AWS_REGION" = "ap-hongkong"
+ );
+ """
+ }
+
+ def set_be_config = { ->
+ String[][] backends = sql """ show backends; """
+ assertTrue(backends.size() > 0)
+ for (String[] backend in backends) {
+ StringBuilder setConfigCommand = new StringBuilder();
+ setConfigCommand.append("curl -X POST http://";)
+ setConfigCommand.append(backend[2])
+ setConfigCommand.append(":")
+ setConfigCommand.append(backend[5])
+ setConfigCommand.append("/api/update_config?")
+ String command1 = setConfigCommand.toString() +
"enable_new_load_scan_node=true"
+ logger.info(command1)
+ String command2 = setConfigCommand.toString() +
"enable_new_file_scanner=true"
+ logger.info(command2)
+ def process1 = command1.execute()
+ int code = process1.waitFor()
+ assertEquals(code, 0)
+ def process2 = command2.execute()
+ code = process1.waitFor()
+ assertEquals(code, 0)
+ }
+ }
+
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ def uuids = []
+ sql """ADMIN SET FRONTEND CONFIG ("enable_new_load_scan_node" =
"true");"""
+ set_be_config.call()
+ try {
+ i = 0
+ for (String table in tables) {
+ sql new
File("""${context.file.parent}/ddl/${table}_drop.sql""").text
+ sql new
File("""${context.file.parent}/ddl/${table}_create.sql""").text
+
+ def uuid = UUID.randomUUID().toString().replace("-", "0")
+ uuids.add(uuid)
+ do_load_job.call(uuid, paths[i], table, columns_list[i],
column_in_paths[i], preceding_filters[i],
+ set_values[i], where_exprs[i])
+ i++
+ }
+
+ for (String label in uuids) {
+ max_try_milli_secs = 600000
+ while (max_try_milli_secs > 0) {
+ String[][] result = sql """ show load where
label="$label"; """
+ if (result[0][2].equals("FINISHED")) {
+ logger.info("Load FINISHED " + label)
+ break;
+ }
+ if (result[0][2].equals("CANCELLED")) {
+ assertTrue(1 == 2, "Load failed.")
+ }
+ Thread.sleep(1000)
+ max_try_milli_secs -= 1000
+ if(max_try_milli_secs <= 0) {
+ assertTrue(1 == 2, "Load Timeout.")
+ }
+ }
+ }
+ } finally {
+ for (String table in tables) {
+ sql new
File("""${context.file.parent}/ddl/${table}_drop.sql""").text
+ }
+ }
+ }
}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
