This is an automated email from the ASF dual-hosted git repository.
hellostephen pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 6f3db5807f7 [test](fix) replace hardcode s3BucketName (#37835)
6f3db5807f7 is described below
commit 6f3db5807f712259b497ecd1066c581dd6931cb8
Author: Dongyang Li <hello_step...@qq.com>
AuthorDate: Tue Jul 16 10:02:58 2024 +0800
[test](fix) replace hardcode s3BucketName (#37835)
## Proposed changes
Issue Number: close #xxx
<!--Describe your changes.-->
Co-authored-by: stephen <hello-step...@qq.com>
---
.../suites/export_p0/test_outfile_exception.groovy | 23 ++--
.../mysql/test_external_catalog_mysql.groovy | 2 +-
.../mysql/test_external_resource_mysql.groovy | 2 +-
.../test_external_resource_mysql_nereids.groovy | 2 +-
.../external_table_p2/pg/test_external_pg.groovy | 2 +-
.../pg/test_external_pg_nereids.groovy | 2 +-
.../suites/github_events_p2/load.groovy | 2 +-
.../load_p0/broker_load/test_compress_type.groovy | 29 ++--
.../load_p0/broker_load/test_etl_failed.groovy | 6 +-
.../suites/load_p0/broker_load/test_s3_load.groovy | 150 +++++++++++----------
.../load_p0/broker_load/test_seq_load.groovy | 7 +-
.../suites/load_p0/tvf/test_s3_tvf.groovy | 94 ++++++-------
.../load_p2/broker_load/test_broker_load.groovy | 140 +++++++++----------
.../test_parquet_large_metadata_load.groovy | 16 ++-
.../broker_load/test_tvf_based_broker_load.groovy | 134 +++++++++---------
.../outfile/test_outfile_exception.groovy | 23 ++--
regression-test/suites/tpcds_sf1000_p2/load.groovy | 4 +-
17 files changed, 326 insertions(+), 312 deletions(-)
diff --git a/regression-test/suites/export_p0/test_outfile_exception.groovy
b/regression-test/suites/export_p0/test_outfile_exception.groovy
index 7ad177bc519..fb8754ee480 100644
--- a/regression-test/suites/export_p0/test_outfile_exception.groovy
+++ b/regression-test/suites/export_p0/test_outfile_exception.groovy
@@ -22,7 +22,8 @@ import java.nio.file.Files
import java.nio.file.Paths
suite("test_outfile_exception") {
-
+ def s3Endpoint = getS3Endpoint()
+ def s3Region = getS3Region()
def tableName = "outfile_exception_test"
sql """ DROP TABLE IF EXISTS ${tableName} """
sql """
@@ -70,8 +71,8 @@ suite("test_outfile_exception") {
into outfile "s3://ftw-datalake-test/test_outfile/exp_"
format as parquet
properties(
- "s3.endpoint" = "https://cos.ap-beijing.myqcloud.com";,
- "s3.region" = "ap-beijing",
+ "s3.endpoint" = "https://${s3Endpoint}";,
+ "s3.region" = "${s3Region}",
"s3.access_key"= "xx",
"s3.secret_key" = "yy"
);
@@ -89,8 +90,8 @@ suite("test_outfile_exception") {
into outfile "s3://ftw-datalake-test/test_outfile/exp_"
format as orc
properties(
- "s3.endpoint" = "https://cos.ap-beijing.myqcloud.com";,
- "s3.region" = "ap-beijing",
+ "s3.endpoint" = "https://${s3Endpoint}";,
+ "s3.region" = "${s3Region}",
"s3.access_key"= "xx",
"s3.secret_key" = "yy"
);
@@ -108,8 +109,8 @@ suite("test_outfile_exception") {
into outfile "s3://ftw-datalake-test/test_outfile/exp_"
format as csv
properties(
- "s3.endpoint" = "https://cos.ap-beijing.myqcloud.com";,
- "s3.region" = "ap-beijing",
+ "s3.endpoint" = "https://${s3Endpoint}";,
+ "s3.region" = "${s3Region}",
"s3.access_key"= "xx",
"s3.secret_key" = "yy"
);
@@ -127,8 +128,8 @@ suite("test_outfile_exception") {
into outfile "s3://ftw-datalake-test/test_outfile/exp_"
format as csv_with_names
properties(
- "s3.endpoint" = "https://cos.ap-beijing.myqcloud.com";,
- "s3.region" = "ap-beijing",
+ "s3.endpoint" = "https://${s3Endpoint}";,
+ "s3.region" = "${s3Region}",
"s3.access_key"= "xx",
"s3.secret_key" = "yy"
);
@@ -146,8 +147,8 @@ suite("test_outfile_exception") {
into outfile "s3://ftw-datalake-test/test_outfile/exp_"
format as csv_with_names_and_types
properties(
- "s3.endpoint" = "https://cos.ap-beijing.myqcloud.com";,
- "s3.region" = "ap-beijing",
+ "s3.endpoint" = "https://${s3Endpoint}";,
+ "s3.region" = "${s3Region}",
"s3.access_key"= "xx",
"s3.secret_key" = "yy"
);
diff --git
a/regression-test/suites/external_table_p2/mysql/test_external_catalog_mysql.groovy
b/regression-test/suites/external_table_p2/mysql/test_external_catalog_mysql.groovy
index c85641c141f..f38a7533aeb 100644
---
a/regression-test/suites/external_table_p2/mysql/test_external_catalog_mysql.groovy
+++
b/regression-test/suites/external_table_p2/mysql/test_external_catalog_mysql.groovy
@@ -42,7 +42,7 @@ suite("test_external_catalog_mysql", "p2") {
"user"="${extMysqlUser}",
"password"="${extMysqlPassword}",
"jdbc_url"="jdbc:mysql://${extMysqlHost}:${extMysqlPort}/ssb?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&serverTimezone=Asia/Shanghai&useSSL=false",
-
"driver_url"="https://doris-community-bj-1308700295.cos.ap-beijing.myqcloud.com/jdbc_drivers/mysql-connector-java-8.0.25.jar";,
+
"driver_url"="https://${getS3BucketName()}.${getS3Endpoint()}/jdbc_drivers/mysql-connector-java-8.0.25.jar",
"driver_class"="com.mysql.cj.jdbc.Driver"
);
"""
diff --git
a/regression-test/suites/external_table_p2/mysql/test_external_resource_mysql.groovy
b/regression-test/suites/external_table_p2/mysql/test_external_resource_mysql.groovy
index 4a31dab7008..1957046c14d 100644
---
a/regression-test/suites/external_table_p2/mysql/test_external_resource_mysql.groovy
+++
b/regression-test/suites/external_table_p2/mysql/test_external_resource_mysql.groovy
@@ -44,7 +44,7 @@ suite("test_external_resource_mysql", "p2") {
"user"="${extMysqlUser}",
"password"="${extMysqlPassword}",
"jdbc_url"="jdbc:mysql://${extMysqlHost}:${extMysqlPort}/ssb?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&serverTimezone=Asia/Shanghai&useSSL=false",
-
"driver_url"="https://doris-community-bj-1308700295.cos.ap-beijing.myqcloud.com/jdbc_drivers/mysql-connector-java-8.0.25.jar";,
+
"driver_url"="https://${getS3BucketName()}.${getS3Endpoint()}/jdbc_drivers/mysql-connector-java-8.0.25.jar",
"driver_class"="com.mysql.cj.jdbc.Driver"
);
"""
diff --git
a/regression-test/suites/external_table_p2/mysql/test_external_resource_mysql_nereids.groovy
b/regression-test/suites/external_table_p2/mysql/test_external_resource_mysql_nereids.groovy
index 68e7e56b859..5e92c8380c8 100644
---
a/regression-test/suites/external_table_p2/mysql/test_external_resource_mysql_nereids.groovy
+++
b/regression-test/suites/external_table_p2/mysql/test_external_resource_mysql_nereids.groovy
@@ -46,7 +46,7 @@ suite("test_external_resource_mysql_nereids", "p2") {
"user"="${extMysqlUser}",
"password"="${extMysqlPassword}",
"jdbc_url"="jdbc:mysql://${extMysqlHost}:${extMysqlPort}/ssb?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true&serverTimezone=Asia/Shanghai&useSSL=false",
-
"driver_url"="https://doris-community-bj-1308700295.cos.ap-beijing.myqcloud.com/jdbc_drivers/mysql-connector-java-8.0.25.jar";,
+
"driver_url"="https://${getS3BucketName()}.${getS3Endpoint()}/jdbc_drivers/mysql-connector-java-8.0.25.jar",
"driver_class"="com.mysql.cj.jdbc.Driver"
);
"""
diff --git
a/regression-test/suites/external_table_p2/pg/test_external_pg.groovy
b/regression-test/suites/external_table_p2/pg/test_external_pg.groovy
index b165f01805d..69ad996505a 100644
--- a/regression-test/suites/external_table_p2/pg/test_external_pg.groovy
+++ b/regression-test/suites/external_table_p2/pg/test_external_pg.groovy
@@ -42,7 +42,7 @@ suite("test_external_pg", "p2") {
"user"="${extPgUser}",
"password"="${extPgPassword}",
"jdbc_url"="jdbc:postgresql://${extPgHost}:${extPgPort}/ssb?currentSchema=ssb&useCursorFetch=true",
-
"driver_url"="https://doris-community-bj-1308700295.cos.ap-beijing.myqcloud.com/jdbc_drivers/postgresql-42.5.0.jar";,
+
"driver_url"="https://${getS3BucketName()}.${getS3Endpoint()}/jdbc_drivers/postgresql-42.5.0.jar",
"driver_class"="org.postgresql.Driver"
);
"""
diff --git
a/regression-test/suites/external_table_p2/pg/test_external_pg_nereids.groovy
b/regression-test/suites/external_table_p2/pg/test_external_pg_nereids.groovy
index 02eb12851fa..4812dce9ce8 100644
---
a/regression-test/suites/external_table_p2/pg/test_external_pg_nereids.groovy
+++
b/regression-test/suites/external_table_p2/pg/test_external_pg_nereids.groovy
@@ -44,7 +44,7 @@ suite("test_external_pg_nereids", "p2") {
"user"="${extPgUser}",
"password"="${extPgPassword}",
"jdbc_url"="jdbc:postgresql://${extPgHost}:${extPgPort}/ssb?currentSchema=ssb&useCursorFetch=true",
-
"driver_url"="https://doris-community-bj-1308700295.cos.ap-beijing.myqcloud.com/jdbc_drivers/postgresql-42.5.0.jar";,
+
"driver_url"="https://${getS3BucketName()}.${getS3Endpoint()}/jdbc_drivers/postgresql-42.5.0.jar",
"driver_class"="org.postgresql.Driver"
);
"""
diff --git a/regression-test/suites/github_events_p2/load.groovy
b/regression-test/suites/github_events_p2/load.groovy
index 105e98c8d44..f40d80189cb 100644
--- a/regression-test/suites/github_events_p2/load.groovy
+++ b/regression-test/suites/github_events_p2/load.groovy
@@ -31,7 +31,7 @@ suite("load") {
ak "${getS3AK()}"
sk "${getS3SK()}"
endpoint "http://${getS3Endpoint()}"
- region "ap-beijing"
+ region "${getS3Region()}"
repository "regression_test_github_events"
snapshot "github_events"
timestamp "2022-03-23-12-19-51"
diff --git
a/regression-test/suites/load_p0/broker_load/test_compress_type.groovy
b/regression-test/suites/load_p0/broker_load/test_compress_type.groovy
index 50d4e34cd99..1538b09bfc8 100644
--- a/regression-test/suites/load_p0/broker_load/test_compress_type.groovy
+++ b/regression-test/suites/load_p0/broker_load/test_compress_type.groovy
@@ -16,6 +16,7 @@
// under the License.
suite("test_compress_type", "load_p0") {
+ def s3BucketName = getS3BucketName()
def tableName = "basic_data"
// GZ/LZO/BZ2/LZ4FRAME/DEFLATE/LZOP
@@ -50,18 +51,18 @@ suite("test_compress_type", "load_p0") {
]
def paths = [
-
"s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz",
-
"s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2",
-
"s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4",
-
"s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz",
-
"s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2",
-
"s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4",
-
"s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz",
-
"s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2",
-
"s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4",
-
"s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz",
-
"s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2",
-
"s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4"
+ "s3://${s3BucketName}/regression/load/data/basic_data.csv.gz",
+ "s3://${s3BucketName}/regression/load/data/basic_data.csv.bz2",
+ "s3://${s3BucketName}/regression/load/data/basic_data.csv.lz4",
+ "s3://${s3BucketName}/regression/load/data/basic_data.csv.gz",
+ "s3://${s3BucketName}/regression/load/data/basic_data.csv.bz2",
+ "s3://${s3BucketName}/regression/load/data/basic_data.csv.lz4",
+ "s3://${s3BucketName}/regression/load/data/basic_data.csv.gz",
+ "s3://${s3BucketName}/regression/load/data/basic_data.csv.bz2",
+ "s3://${s3BucketName}/regression/load/data/basic_data.csv.lz4",
+ "s3://${s3BucketName}/regression/load/data/basic_data.csv.gz",
+ "s3://${s3BucketName}/regression/load/data/basic_data.csv.bz2",
+ "s3://${s3BucketName}/regression/load/data/basic_data.csv.lz4"
]
def labels = []
@@ -88,8 +89,8 @@ suite("test_compress_type", "load_p0") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
- "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
- "AWS_REGION" = "ap-beijing"
+ "AWS_ENDPOINT" = "${getS3Endpoint()}",
+ "AWS_REGION" = "${getS3Region()}"
)
properties(
"use_new_load_scan_node" = "true"
diff --git a/regression-test/suites/load_p0/broker_load/test_etl_failed.groovy
b/regression-test/suites/load_p0/broker_load/test_etl_failed.groovy
index 3b4eac874d5..499b6d262bd 100644
--- a/regression-test/suites/load_p0/broker_load/test_etl_failed.groovy
+++ b/regression-test/suites/load_p0/broker_load/test_etl_failed.groovy
@@ -33,7 +33,7 @@ suite("test_etl_failed", "load_p0") {
PROPERTIES ("replication_allocation" = "tag.location.default: 1");
"""
String label = "test_etl_failed"
- String path =
"s3://doris-build-1308700295/regression/load/data/etl_failure/etl-failure.csv"
+ String path =
"s3://${getS3BucketName()}/regression/load/data/etl_failure/etl-failure.csv"
String format = "CSV"
String ak = getS3AK()
String sk = getS3SK()
@@ -46,8 +46,8 @@ suite("test_etl_failed", "load_p0") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
- "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
- "AWS_REGION" = "ap-beijing"
+ "AWS_ENDPOINT" = "${getS3Endpoint()}",
+ "AWS_REGION" = "${getS3Region()}"
)
PROPERTIES(
"use_new_load_scan_node" = "true",
diff --git a/regression-test/suites/load_p0/broker_load/test_s3_load.groovy
b/regression-test/suites/load_p0/broker_load/test_s3_load.groovy
index 6312cc3feb7..5c96ea22da3 100644
--- a/regression-test/suites/load_p0/broker_load/test_s3_load.groovy
+++ b/regression-test/suites/load_p0/broker_load/test_s3_load.groovy
@@ -16,7 +16,9 @@
// under the License.
suite("test_s3_load", "load_p0") {
-
+ def s3BucketName = getS3BucketName()
+ def s3Endpoint = getS3Endpoint()
+ def s3Region = getS3Region()
def tables = [
"agg_tbl_basic",
"dup_tbl_array",
@@ -51,183 +53,183 @@ suite("test_s3_load", "load_p0") {
/* ========================================================== normal
========================================================== */
for (String table : basicTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", ""))
}
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\" ", "FORMAT AS \"csv\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", ""))
for (String table : arrayTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "", "", ""))
}
for (String table : basicTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(K00,K01,K02,K03,K04,K05,K06,K07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17,K18)",
"", "", "", "", ""))
}
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\" ", "FORMAT AS \"csv\"",
"(K00,K01,K02,K03,K04,K05,K06,k07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17,K18)",
"", "", "SET
(K19=to_bitmap(k04),K20=HLL_HASH(k04),K21=TO_QUANTILE_STATE(K04,1.0),Kd19=to_bitmap(K05),kd20=HLL_HASH(K05),KD21=TO_QUANTILE_STATE(K05,1.0))",
"", ""))
for (String table : arrayTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(K00,K01,K02,K03,K04,K05,K06,K07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17)",
"", "", "", "", ""))
}
// TODO: should be success ?
// for (String table : basicTables) {
-// attributesList.add(new
LoadAttributes("s3://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_data.csv",
+// attributesList.add(new
LoadAttributes("s3://${s3Endpoint}/${s3BucketName}/regression/load/data/basic_data.csv",
// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
// "", "", "", "", "").withPathStyle())
// }
//
-// attributesList.add(new
LoadAttributes("s3://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_data.csv",
+// attributesList.add(new
LoadAttributes("s3://${s3Endpoint}/${s3BucketName}/regression/load/data/basic_data.csv",
// "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\" ", "FORMAT AS \"csv\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
// "", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", "").withPathStyle())
//
// for (String table : arrayTables) {
-// attributesList.add(new
LoadAttributes("s3://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_array_data.csv",
+// attributesList.add(new
LoadAttributes("s3://${s3Endpoint}/${s3BucketName}/regression/load/data/basic_array_data.csv",
// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
// "", "", "", "", "").withPathStyle())
// }
//
// for (String table : basicTables) {
-// attributesList.add(new
LoadAttributes("s3://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_data.csv",
+// attributesList.add(new
LoadAttributes("s3://${s3Endpoint}/${s3BucketName}/regression/load/data/basic_data.csv",
// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
// "", "", "", "", "").withPathStyle())
// }
//
-// attributesList.add(new
LoadAttributes("s3://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_data.csv",
+// attributesList.add(new
LoadAttributes("s3://${s3Endpoint}/${s3BucketName}/regression/load/data/basic_data.csv",
// "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\" ", "FORMAT AS \"csv\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
// "", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", "").withPathStyle())
//
// for (String table : arrayTables) {
-// attributesList.add(new
LoadAttributes("s3://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_array_data.csv",
+// attributesList.add(new
LoadAttributes("s3://${s3Endpoint}/${s3BucketName}/regression/load/data/basic_array_data.csv",
// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
// "", "", "", "", "").withPathStyle())
// }
/* ========================================================== error
========================================================== */
for (String table : basicTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_with_errors.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", "", true))
}
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_with_errors.csv",
"agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\" ", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", "", true))
for (String table : arrayTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data_with_errors.csv",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data_with_errors.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"csv\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "", "", "", true))
}
// has problem, should be success
// for(String table: basicTables) {
-// attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+// attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_with_errors.csv",
// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
// "", "", "", "","").addProperties("max_filter_ratio", "0.5"))
// }
//
-// attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+// attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_with_errors.csv",
// "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\" ", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
// "", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", "").addProperties("max_filter_ratio", "0.5"))
//
// for(String table : arrayTables) {
-// attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data_with_errors.csv",
+// attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data_with_errors.csv",
// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
// "", "", "", "","").addProperties("max_filter_ratio", "0.5"))
// }
// for(String table: basicTables) {
-// attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+// attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_with_errors.csv",
// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
// "", "", "", "","", true).addProperties("max_filter_ratio",
"0.4"))
// }
//
-// attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+// attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_with_errors.csv",
// "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\" ", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
// "", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", "", true).addProperties("max_filter_ratio", "0.4"))
//
// for(String table : arrayTables) {
-// attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data_with_errors.csv",
+// attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data_with_errors.csv",
// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
// "", "", "", "","", true).addProperties("max_filter_ratio",
"0.4"))
// }
// skip lines
// for(String table: basicTables) {
-// attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+// attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_with_errors.csv",
// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
// "", "", "", "","").addProperties("skip_lines", "10"))
// }
//
-// attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+// attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_with_errors.csv",
// "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\" ", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
// "", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", "").addProperties("skip_lines", "10"))
//
// for(String table : arrayTables) {
-// attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data_with_errors.csv",
+// attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data_with_errors.csv",
// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
// "", "", "", "","").addProperties("skip_lines", "10"))
// }
/* ========================================================== wrong column
sep ========================================================== */
for (String table : basicTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \",\"", "FORMAT AS \"csv\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", "", true))
}
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \",\" ", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", "", true))
for (String table : arrayTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \",\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "", "", "", true))
}
/* ========================================================== wrong line
delim ========================================================== */
for (String table : basicTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"${table}", "LINES TERMINATED BY \"\t\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", "", true))
}
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"agg_tbl_basic", "LINES TERMINATED BY \"\t\"", "COLUMNS TERMINATED
BY \"|\" ", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", "", true))
for (String table : arrayTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.csv",
"${table}", "LINES TERMINATED BY \"\t\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "", "", "", true))
}
/* ========================================================== strict mode
========================================================== */
for(String table: basicTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_with_errors.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "","", true).addProperties("strict_mode", "true"))
}
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_with_errors.csv",
"agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\" ", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", "", true).addProperties("strict_mode","true"))
for(String table : arrayTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data_with_errors.csv",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data_with_errors.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "", "","", true).addProperties("strict_mode", "true"))
}
@@ -235,131 +237,131 @@ suite("test_s3_load", "load_p0") {
/* ========================================================== timezone
========================================================== */
for(String table: basicTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET (k00=unix_timestamp('2023-09-01 12:00:00'))",
"","").addProperties("timezone", "Asia/Shanghai"))
}
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\" ", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET (k00=unix_timestamp('2023-09-01
12:00:00'),k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", "").addProperties("timezone", "Asia/Shanghai"))
for(String table : arrayTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "SET (k00=unix_timestamp('2023-09-01 12:00:00'))",
"","").addProperties("timezone", "Asia/Shanghai"))
}
for(String table: basicTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET (k00=unix_timestamp('2023-09-01 12:00:00'))",
"","").addProperties("timezone", "America/Chicago"))
}
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\" ", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET (k00=unix_timestamp('2023-09-01
12:00:00'),k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", "").addProperties("timezone", "America/Chicago"))
for(String table : arrayTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "SET (k00=unix_timestamp('2023-09-01 12:00:00'))",
"","").addProperties("timezone", "America/Chicago"))
}
/* ========================================================== compress
type ========================================================== */
for (String table : basicTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.gz",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", ""))
}
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.gz",
"agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\" ", "FORMAT AS \"csv\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", ""))
for (String table : arrayTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv.gz",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.csv.gz",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "", "", ""))
}
for (String table : basicTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.bz2",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", ""))
}
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.bz2",
"agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\" ", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", ""))
for (String table : arrayTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv.bz2",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.csv.bz2",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "", "", ""))
}
for (String table : basicTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.lz4",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", ""))
}
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.lz4",
"agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\" ", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", ""))
for (String table : arrayTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv.lz4",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.csv.lz4",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "", "", ""))
}
for (String table : basicTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.gz",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", ""))
}
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.gz",
"agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\" ", "",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", ""))
for (String table : arrayTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv.gz",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.csv.gz",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "", "", ""))
}
for (String table : basicTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.bz2",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", ""))
}
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.bz2",
"agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\" ", "",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", ""))
for (String table : arrayTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv.bz2",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.csv.bz2",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "", "", ""))
}
for (String table : basicTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.lz4",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", ""))
}
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.lz4",
"agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\" ", "",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", ""))
for (String table : arrayTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv.lz4",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.csv.lz4",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "", "", ""))
}
@@ -367,7 +369,7 @@ suite("test_s3_load", "load_p0") {
/*========================================================== order by
==========================================================*/
for (String table : uniqTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", "ORDER BY k01"))
}
@@ -375,71 +377,71 @@ suite("test_s3_load", "load_p0") {
/*========================================================== json
==========================================================*/
for (String table : basicTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.json",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.json",
"${table}", "", "", "FORMAT AS \"json\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", "PROPERTIES(\"strip_outer_array\" = \"true\",
\"fuzzy_parse\" = \"true\")"))
}
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.json",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.json",
"agg_tbl_basic", "", "", "FORMAT AS \"json\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", "PROPERTIES(\"strip_outer_array\" = \"true\", \"fuzzy_parse\" =
\"true\")"))
for (String table : arrayTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.json",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.json",
"${table}", "", "", "FORMAT AS \"json\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "", "", "PROPERTIES(\"strip_outer_array\" = \"true\",
\"fuzzy_parse\" = \"true\")"))
}
for (String table : basicTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_by_line.json",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_by_line.json",
"${table}", "", "", "FORMAT AS \"JSON\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", "PROPERTIES(\"read_json_by_line\" =
\"true\")"))
}
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_by_line.json",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_by_line.json",
"agg_tbl_basic", "", "", "FORMAT AS \"JSON\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", "PROPERTIES(\"read_json_by_line\" = \"true\")"))
for (String table : arrayTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data_by_line.json",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data_by_line.json",
"${table}", "", "", "FORMAT AS \"JSON\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "", "", "PROPERTIES(\"read_json_by_line\" =
\"true\")"))
}
for (String table : basicTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.parq",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.parq",
"${table}", "", "", "FORMAT AS \"parquet\"",
"(K00,K01,K02,K03,K04,K05,K06,K07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17,K18)",
"", "", "", "", ""))
}
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.parq",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.parq",
"agg_tbl_basic", "", "", "FORMAT AS \"PARQUET\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", ""))
// for (String table : arrayTables) {
-// attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.parq",
+// attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.parq",
// "${table}", "", "", "FORMAT AS \"parquet\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
// "", "", "", "", ""))
// }
for (String table : basicTables) {
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.orc",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.orc",
"${table}", "", "", "FORMAT AS \"orc\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", ""))
}
- attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.orc",
+ attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.orc",
"agg_tbl_basic", "", "", "FORMAT AS \"ORC\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", ""))
// for (String table : arrayTables) {
-// attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.parq",
+// attributesList.add(new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.parq",
// "${table}", "", "", "FORMAT AS \"parquet\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
// "", "", "", "", ""))
// }
for(String table : uniqTables) {
- def attributes = new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_delete.csv",
+ def attributes = new
LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_delete.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,__DEL__)",
"", "", "", "", "DELETE ON __DEL__=true")
attributes.dataDesc.mergeType = "MERGE"
@@ -478,8 +480,8 @@ suite("test_s3_load", "load_p0") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
- "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
- "AWS_REGION" = "ap-beijing",
+ "AWS_ENDPOINT" = "${s3Endpoint}",
+ "AWS_REGION" = "${getS3Region()}",
"use_path_style" = "$attributes.usePathStyle"
)
${prop}
diff --git a/regression-test/suites/load_p0/broker_load/test_seq_load.groovy
b/regression-test/suites/load_p0/broker_load/test_seq_load.groovy
index a762d79e5c8..963e4ae8020 100644
--- a/regression-test/suites/load_p0/broker_load/test_seq_load.groovy
+++ b/regression-test/suites/load_p0/broker_load/test_seq_load.groovy
@@ -61,7 +61,6 @@ suite("test_seq_load", "load_p0") {
kd16 VARCHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
kd17 STRING NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
kd18 JSON NULL,
-
INDEX idx_bitmap_k104 (`k02`) USING BITMAP,
INDEX idx_bitmap_k110 (`kd01`) USING BITMAP,
INDEX idx_bitmap_k113 (`k13`) USING BITMAP,
@@ -83,7 +82,7 @@ suite("test_seq_load", "load_p0") {
"""
def label = UUID.randomUUID().toString().replace("-", "0")
- def path =
"s3://doris-build-1308700295/regression/load/data/basic_data.csv"
+ def path = "s3://${getS3BucketName()}/regression/load/data/basic_data.csv"
def format_str = "CSV"
def ak = getS3AK()
def sk = getS3SK()
@@ -101,8 +100,8 @@ suite("test_seq_load", "load_p0") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
- "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
- "AWS_REGION" = "ap-beijing"
+ "AWS_ENDPOINT" = "${getS3Endpoint()}",
+ "AWS_REGION" = "${getS3Region()}"
)
properties(
"use_new_load_scan_node" = "true"
diff --git a/regression-test/suites/load_p0/tvf/test_s3_tvf.groovy
b/regression-test/suites/load_p0/tvf/test_s3_tvf.groovy
index 391b76d37eb..d32210e0997 100644
--- a/regression-test/suites/load_p0/tvf/test_s3_tvf.groovy
+++ b/regression-test/suites/load_p0/tvf/test_s3_tvf.groovy
@@ -16,7 +16,9 @@
// under the License.
suite("test_s3_tvf", "load_p0") {
-
+ def s3BucketName = getS3BucketName()
+ def s3Endpoint = getS3Endpoint()
+ def s3Region = getS3Region()
def tables = [
"agg_tbl_basic_tvf",
"dup_tbl_array_tvf",
@@ -54,67 +56,67 @@ suite("test_s3_tvf", "load_p0") {
/* normal */
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03",
"K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14",
"K15", "K16", "K17", "K18"], "", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|"))
}
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|"))
for(String table : arrayTables) {
- attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03",
"K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14",
"K15", "K16", "K17"], "", "").addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv")
+ attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03",
"K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14",
"K15", "K16", "K17"], "", "").addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.csv")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|"))
}
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|"))
}
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17"], "", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|"))
}
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "", "")
- .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv";)
+ .addProperty("uri",
"https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv";)
.addProperty("format", "csv")
.addProperty("column_separator", "|"))
}
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
- .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv";)
+ .addProperty("uri",
"https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv";)
.addProperty("format", "csv")
.addProperty("column_separator", "|"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17"], "", "")
- .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv";)
+ .addProperty("uri",
"https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.csv";)
.addProperty("format", "csv")
.addProperty("column_separator", "|"))
}
// for(String table : basicTables) {
// attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02",
"k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13",
"k14", "k15", "k16", "k17", "k18"], "", "")
-// .addProperty("uri",
"https://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_data.csv";)
+// .addProperty("uri",
"https://${s3Endpoint}/${s3BucketName}/regression/load/data/basic_data.csv";)
// .addProperty("format", "csv")
// .addProperty("column_separator", "|")
// .addProperty("use_path_style", "true"))
@@ -122,14 +124,14 @@ suite("test_s3_tvf", "load_p0") {
//
// attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
//
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
-// .addProperty("uri",
"https://cos.ap-beijing.myqcloud.com/doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv";)
+// .addProperty("uri",
"https://${s3Endpoint}/${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv";)
// .addProperty("format", "csv")
// .addProperty("column_separator", "|")
// .addProperty("use_path_style", "true"))
//
// for(String table : arrayTables) {
// attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02",
"k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13",
"k14", "k15", "k16", "k17"], "", "")
-// .addProperty("uri",
"https://cos.ap-beijing.myqcloud.com/doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv";)
+// .addProperty("uri",
"https://${s3Endpoint}/${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.csv";)
// .addProperty("format", "csv")
// .addProperty("column_separator", "|")
// .addProperty("use_path_style", "true"))
@@ -138,20 +140,20 @@ suite("test_s3_tvf", "load_p0") {
/* error */
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "", "", true)
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_with_errors.csv")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_with_errors.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|"))
}
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "", true)
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_with_errors.csv")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_with_errors.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "kd01", "kd02", "kd03", "kd04", "kd05", "kd06", "kd07",
"kd08", "kd09", "kd10", "kd11", "kd12", "kd13", "kd14", "kd15", "kd16"], "",
"", true)
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_with_errors.csv")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data_with_errors.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|"))
}
@@ -159,7 +161,7 @@ suite("test_s3_tvf", "load_p0") {
/* skip lines */
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_with_errors.csv")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_with_errors.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("skip_lines", "10"))
@@ -167,14 +169,14 @@ suite("test_s3_tvf", "load_p0") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_with_errors.csv")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_with_errors.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("skip_lines", "10"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17"], "", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_with_errors.csv")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data_with_errors.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("skip_lines", "10"))
@@ -183,21 +185,21 @@ suite("test_s3_tvf", "load_p0") {
/* compress type */
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "", "")
- .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.gz";)
+ .addProperty("uri",
"https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv.gz";)
.addProperty("format", "csv")
.addProperty("column_separator",
"|").addProperty("compress_type", "GZ"))
}
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
- .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.gz";)
+ .addProperty("uri",
"https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv.gz";)
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("compress_type", "GZ"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17"], "", "")
- .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv.gz";)
+ .addProperty("uri",
"https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.csv.gz";)
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("compress_type", "GZ"))
@@ -205,21 +207,21 @@ suite("test_s3_tvf", "load_p0") {
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "", "")
- .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.bz2";)
+ .addProperty("uri",
"https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv.bz2";)
.addProperty("format", "csv")
.addProperty("column_separator",
"|").addProperty("compress_type", "BZ2"))
}
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
- .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.bz2";)
+ .addProperty("uri",
"https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv.bz2";)
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("compress_type", "BZ2"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17"], "", "")
- .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv.bz2";)
+ .addProperty("uri",
"https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.csv.bz2";)
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("compress_type", "BZ2"))
@@ -227,21 +229,21 @@ suite("test_s3_tvf", "load_p0") {
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "", "")
- .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.lz4";)
+ .addProperty("uri",
"https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv.lz4";)
.addProperty("format", "csv")
.addProperty("column_separator",
"|").addProperty("compress_type", "LZ4FRAME"))
}
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
- .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.lz4";)
+ .addProperty("uri",
"https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv.lz4";)
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("compress_type", "LZ4FRAME"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17"], "", "")
- .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv.lz4";)
+ .addProperty("uri",
"https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.csv.lz4";)
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("compress_type", "LZ4FRAME"))
@@ -249,74 +251,74 @@ suite("test_s3_tvf", "load_p0") {
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "WHERE c1 > 50", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|"))
}
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"WHERE c1 > 50", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17"], "WHERE c1 > 50", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|"))
}
for(String table : uniqTable) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "", "ORDER BY c1")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|"))
}
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18",
"", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.parq")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.parq")
.addProperty("format", "parquet")
.addProperty("column_separator", "|"))
}
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,
to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as
k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0)
as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.parq")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.parq")
.addProperty("format", "parquet")
.addProperty("column_separator", "|"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "",
"")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.parq")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.parq")
.addProperty("format", "parquet")
.addProperty("column_separator", "|"))
}
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18",
"", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.orc")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.orc")
.addProperty("format", "orc")
.addProperty("column_separator", "|"))
}
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,
to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as
k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0)
as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.orc")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.orc")
.addProperty("format", "orc")
.addProperty("column_separator", "|"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "",
"")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.orc")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.orc")
.addProperty("format", "orc")
.addProperty("column_separator", "|"))
}
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18",
"", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.json")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "false")
.addProperty("strip_outer_array", "true")
@@ -325,7 +327,7 @@ suite("test_s3_tvf", "load_p0") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,
to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as
k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0)
as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.json")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "false")
.addProperty("strip_outer_array", "true")
@@ -333,7 +335,7 @@ suite("test_s3_tvf", "load_p0") {
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "",
"")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.json")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "false")
.addProperty("strip_outer_array", "true")
@@ -342,7 +344,7 @@ suite("test_s3_tvf", "load_p0") {
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18",
"", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_by_line.json")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_by_line.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "true")
.addProperty("strip_outer_array", "false")
@@ -351,7 +353,7 @@ suite("test_s3_tvf", "load_p0") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,
to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as
k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0)
as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_by_line.json")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_by_line.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "true")
.addProperty("strip_outer_array", "false")
@@ -359,7 +361,7 @@ suite("test_s3_tvf", "load_p0") {
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "",
"")
- .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_by_line.json")
+ .addProperty("uri",
"s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data_by_line.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "true")
.addProperty("strip_outer_array", "false")
@@ -387,7 +389,7 @@ suite("test_s3_tvf", "load_p0") {
FROM S3 (
"s3.access_key" = "$ak",
"s3.secret_key" = "$sk",
- "s3.region" = "ap-beijing",
+ "s3.region" = "${getS3Region()}",
${prop}
) ${attribute.whereClause}
${attribute.orderByClause}
diff --git a/regression-test/suites/load_p2/broker_load/test_broker_load.groovy
b/regression-test/suites/load_p2/broker_load/test_broker_load.groovy
index e51b7e2a7e5..60949e20d37 100644
--- a/regression-test/suites/load_p2/broker_load/test_broker_load.groovy
+++ b/regression-test/suites/load_p2/broker_load/test_broker_load.groovy
@@ -16,7 +16,9 @@
// under the License.
suite("test_broker_load_p2", "p2") {
-
+ def s3BucketName = getS3BucketName()
+ def s3Endpoint = getS3Endpoint()
+ def s3Region = getS3Region()
def tables = ["part",
"upper_case",
"reverse",
@@ -49,37 +51,37 @@ suite("test_broker_load_p2", "p2") {
"orc_s3_case9", // table column uppercase * load column
lowercase * orc file uppercase
"csv_s3_case_line_delimiter" // csv format table with
special line delimiter
]
- def paths = ["s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
-
"s3://doris-build-1308700295/regression/load/data/path/*/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
-
"s3://doris-build-1308700295/regression/load/data/orc/hits_100k_rows.orc",
-
"s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_lowercase.orc",
-
"s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_lowercase.orc",
-
"s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_uppercase.orc",
-
"s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_uppercase.orc",
-
"s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_lowercase.orc",
-
"s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_lowercase.orc",
-
"s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_uppercase.orc",
-
"s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_uppercase.orc",
-
"s3://doris-build-1308700295/regression/line_delimiter/lineitem_0x7.csv.gz"
+ def paths = ["s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/path/*/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+
"s3://${s3BucketName}/regression/load/data/orc/hits_100k_rows.orc",
+
"s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc",
+
"s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc",
+
"s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc",
+
"s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc",
+
"s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc",
+
"s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc",
+
"s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc",
+
"s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc",
+
"s3://${s3BucketName}/regression/line_delimiter/lineitem_0x7.csv.gz"
]
def columns_list = ["""p_partkey, p_name, p_mfgr, p_brand, p_type, p_size,
p_container, p_retailprice, p_comment""",
"""p_partkey, p_name, p_mfgr, p_brand, p_type, p_size,
p_container, p_retailprice, p_comment""",
@@ -194,37 +196,37 @@ suite("test_broker_load_p2", "p2") {
"\\N"
]
- def task_info = ["cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0"
+ def task_info = ["cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0"
]
def error_msg = ["",
@@ -291,8 +293,8 @@ suite("test_broker_load_p2", "p2") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
- "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
- "AWS_REGION" = "ap-beijing"
+ "AWS_ENDPOINT" = "${s3Endpoint}",
+ "AWS_REGION" = "${getS3Region()}"
)
properties(
"use_new_load_scan_node" = "true"
@@ -383,11 +385,11 @@ suite("test_broker_load_p2", "p2") {
def label_22666 = "part_" +
UUID.randomUUID().toString().replace("-", "0")
sql """
LOAD LABEL ${label_22666} (
- DATA
INFILE("s3://doris-build-1308700295/regression/load/data/part0.parquet")
+ DATA
INFILE("s3://${s3BucketName}/regression/load/data/part0.parquet")
INTO TABLE ${tbl_22666}
FORMAT AS "PARQUET"
(p_partkey, p_name, p_mfgr),
- DATA
INFILE("s3://doris-build-1308700295/regression/load/data/part1.parquet")
+ DATA
INFILE("s3://${s3BucketName}/regression/load/data/part1.parquet")
INTO TABLE ${tbl_22666}
FORMAT AS "PARQUET"
(p_partkey, p_brand, p_type)
@@ -395,8 +397,8 @@ suite("test_broker_load_p2", "p2") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
- "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
- "AWS_REGION" = "ap-beijing"
+ "AWS_ENDPOINT" = "${s3Endpoint}",
+ "AWS_REGION" = "${getS3Region()}"
);
"""
diff --git
a/regression-test/suites/load_p2/broker_load/test_parquet_large_metadata_load.groovy
b/regression-test/suites/load_p2/broker_load/test_parquet_large_metadata_load.groovy
index 70e0681d0cb..fad42a0d3b1 100644
---
a/regression-test/suites/load_p2/broker_load/test_parquet_large_metadata_load.groovy
+++
b/regression-test/suites/load_p2/broker_load/test_parquet_large_metadata_load.groovy
@@ -16,10 +16,12 @@
// under the License.
suite("test_parquet_large_metadata_load_p2", "p2") {
-
+ def s3BucketName = getS3BucketName()
+ def s3Endpoint = getS3Endpoint()
+ def s3Region = getS3Region()
def tables = ["parquet_large_metadata_100mb" // metadata size more than
100MB
]
- def paths =
["s3://doris-build-1308700295/regression/load/metadata/parquet_large_metadata_100mb.parquet"
+ def paths =
["s3://${s3BucketName}/regression/load/metadata/parquet_large_metadata_100mb.parquet"
]
String ak = getS3AK()
String sk = getS3SK()
@@ -27,10 +29,10 @@ suite("test_parquet_large_metadata_load_p2", "p2") {
def expect_tvf_result = """[[2, 8], [2, 8], [2, 8], [2, 8], [2, 8]]"""
String[][] tvf_result = sql """select `1`,`2` from s3(
- "uri" =
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/metadata/parquet_large_metadata_100mb.parquet";,
+ "uri" =
"https://${s3BucketName}.${s3Endpoint}/regression/load/metadata/parquet_large_metadata_100mb.parquet";,
"s3.access_key" = "$ak",
"s3.secret_key" = "$sk",
- "s3.region" = "ap-beijing",
+ "s3.region" = "${s3Region}",
"format" = "parquet"
) order by `1`,`2` limit 5;
"""
@@ -47,8 +49,8 @@ suite("test_parquet_large_metadata_load_p2", "p2") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
- "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
- "AWS_REGION" = "ap-beijing"
+ "AWS_ENDPOINT" = "${s3Endpoint}",
+ "AWS_REGION" = "${s3Region}"
)
PROPERTIES
(
@@ -59,7 +61,7 @@ suite("test_parquet_large_metadata_load_p2", "p2") {
}
def etl_info = ["unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=45000"]
- def task_info = ["cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0"]
+ def task_info = ["cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0"]
def error_msg = [""]
// test unified load
if (enabled != null && enabled.equalsIgnoreCase("true")) {
diff --git
a/regression-test/suites/load_p2/broker_load/test_tvf_based_broker_load.groovy
b/regression-test/suites/load_p2/broker_load/test_tvf_based_broker_load.groovy
index bba1fb239e6..c67179cbee6 100644
---
a/regression-test/suites/load_p2/broker_load/test_tvf_based_broker_load.groovy
+++
b/regression-test/suites/load_p2/broker_load/test_tvf_based_broker_load.groovy
@@ -16,7 +16,9 @@
// under the License.
suite("test_tvf_based_broker_load_p2", "p2") {
-
+ def s3BucketName = getS3BucketName()
+ def s3Endpoint = getS3Endpoint()
+ def s3Region = getS3Region()
def tables = ["part",
"upper_case",
"reverse",
@@ -50,38 +52,38 @@ suite("test_tvf_based_broker_load_p2", "p2") {
"orc_s3_case9", // table column uppercase * load column
lowercase * orc file uppercase
"csv_s3_case_line_delimiter" // csv format table with
special line delimiter
]
- def paths = ["s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
-
"s3://doris-build-1308700295/regression/load/data/path/*/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
-
"s3://doris-build-1308700295/regression/load/data/random_all_types/part*",
-
"s3://doris-build-1308700295/regression/load/data/orc/hits_100k_rows.orc",
-
"s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_lowercase.orc",
-
"s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_lowercase.orc",
-
"s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_uppercase.orc",
-
"s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_uppercase.orc",
-
"s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_lowercase.orc",
-
"s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_lowercase.orc",
-
"s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_uppercase.orc",
-
"s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_uppercase.orc",
-
"s3://doris-build-1308700295/regression/line_delimiter/lineitem_0x7.csv.gz"
+ def paths = ["s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/path/*/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+
"s3://${s3BucketName}/regression/load/data/random_all_types/part*",
+
"s3://${s3BucketName}/regression/load/data/orc/hits_100k_rows.orc",
+
"s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc",
+
"s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc",
+
"s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc",
+
"s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc",
+
"s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc",
+
"s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc",
+
"s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc",
+
"s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc",
+
"s3://${s3BucketName}/regression/line_delimiter/lineitem_0x7.csv.gz"
]
def columns_list = ["""p_partkey, p_name, p_mfgr, p_brand, p_type, p_size,
p_container, p_retailprice, p_comment""",
"""p_partkey, p_name, p_mfgr, p_brand, p_type, p_size,
p_container, p_retailprice, p_comment""",
@@ -196,37 +198,37 @@ suite("test_tvf_based_broker_load_p2", "p2") {
"unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=10000",
]
- def task_info = ["cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400;
max_filter_ratio:0.0",
+ def task_info = ["cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400;
max_filter_ratio:0.0",
]
def error_msg = ["",
@@ -291,8 +293,8 @@ suite("test_tvf_based_broker_load_p2", "p2") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
- "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
- "AWS_REGION" = "ap-beijing"
+ "AWS_ENDPOINT" = "${s3Endpoint}",
+ "AWS_REGION" = "${s3Region}"
)
"""
logger.info("Submit load with lable: $uuid, table: $table, path:
$path")
diff --git
a/regression-test/suites/nereids_p0/outfile/test_outfile_exception.groovy
b/regression-test/suites/nereids_p0/outfile/test_outfile_exception.groovy
index fdba0916b6d..daeb04319b1 100644
--- a/regression-test/suites/nereids_p0/outfile/test_outfile_exception.groovy
+++ b/regression-test/suites/nereids_p0/outfile/test_outfile_exception.groovy
@@ -22,6 +22,9 @@ import java.nio.file.Files
import java.nio.file.Paths
suite("test_outfile_exception") {
+ def s3BucketName = getS3BucketName()
+ def s3Endpoint = getS3Endpoint()
+ def s3Region = getS3Region()
sql 'set enable_nereids_planner=true'
sql 'set enable_fallback_to_original_planner=false'
@@ -72,8 +75,8 @@ suite("test_outfile_exception") {
into outfile "s3://ftw-datalake-test/test_outfile/exp_"
format as parquet
properties(
- "s3.endpoint" = "https://cos.ap-beijing.myqcloud.com";,
- "s3.region" = "ap-beijing",
+ "s3.endpoint" = "https://${s3Endpoint}";,
+ "s3.region" = "${s3Region}",
"s3.access_key"= "xx",
"s3.secret_key" = "yy"
);
@@ -91,8 +94,8 @@ suite("test_outfile_exception") {
into outfile "s3://ftw-datalake-test/test_outfile/exp_"
format as orc
properties(
- "s3.endpoint" = "https://cos.ap-beijing.myqcloud.com";,
- "s3.region" = "ap-beijing",
+ "s3.endpoint" = "https://${s3Endpoint}";,
+ "s3.region" = "${s3Region}",
"s3.access_key"= "xx",
"s3.secret_key" = "yy"
);
@@ -110,8 +113,8 @@ suite("test_outfile_exception") {
into outfile "s3://ftw-datalake-test/test_outfile/exp_"
format as csv
properties(
- "s3.endpoint" = "https://cos.ap-beijing.myqcloud.com";,
- "s3.region" = "ap-beijing",
+ "s3.endpoint" = "https://${s3Endpoint}";,
+ "s3.region" = "${s3Region}",
"s3.access_key"= "xx",
"s3.secret_key" = "yy"
);
@@ -129,8 +132,8 @@ suite("test_outfile_exception") {
into outfile "s3://ftw-datalake-test/test_outfile/exp_"
format as csv_with_names
properties(
- "s3.endpoint" = "https://cos.ap-beijing.myqcloud.com";,
- "s3.region" = "ap-beijing",
+ "s3.endpoint" = "https://${s3Endpoint}";,
+ "s3.region" = "${s3Region}",
"s3.access_key"= "xx",
"s3.secret_key" = "yy"
);
@@ -148,8 +151,8 @@ suite("test_outfile_exception") {
into outfile "s3://ftw-datalake-test/test_outfile/exp_"
format as csv_with_names_and_types
properties(
- "s3.endpoint" = "https://cos.ap-beijing.myqcloud.com";,
- "s3.region" = "ap-beijing",
+ "s3.endpoint" = "https://${s3Endpoint}";,
+ "s3.region" = "${s3Region}",
"s3.access_key"= "xx",
"s3.secret_key" = "yy"
);
diff --git a/regression-test/suites/tpcds_sf1000_p2/load.groovy
b/regression-test/suites/tpcds_sf1000_p2/load.groovy
index aaf4fd54d71..b4dfee6274d 100644
--- a/regression-test/suites/tpcds_sf1000_p2/load.groovy
+++ b/regression-test/suites/tpcds_sf1000_p2/load.groovy
@@ -26,7 +26,7 @@ suite("load") {
ak "${getS3AK()}"
sk "${getS3SK()}"
endpoint "http://${getS3Endpoint()}"
- region "ap-beijing"
+ region "${s3Region}"
repository "tpcds_backup"
snapshot "tpcds_customer"
timestamp "2022-03-31-10-16-46"
@@ -40,7 +40,7 @@ suite("load") {
ak "${getS3AK()}"
sk "${getS3SK()}"
endpoint "http://${getS3Endpoint()}"
- region "ap-beijing"
+ region "${s3Region}"
repository "tpcds_backup"
snapshot "tpcds"
timestamp "2022-03-30-12-22-31"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
