This is an automated email from the ASF dual-hosted git repository.
hellostephen pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 50057f916c6 [regression](s3) add default conf for s3 releated cases
(#37952) (#38473)
50057f916c6 is described below
commit 50057f916c663d8a54de8a350c624b2baf9e3085
Author: Dongyang Li <hello_step...@qq.com>
AuthorDate: Mon Jul 29 19:40:36 2024 +0800
[regression](s3) add default conf for s3 releated cases (#37952) (#38473)
replace COS with OSS in the TeamCity pipeline to improve stability
pick from master #37952
## Proposed changes
Issue Number: close #xxx
<!--Describe your changes.-->
---
regression-test/conf/regression-conf.groovy | 16 ++--
.../org/apache/doris/regression/Config.groovy | 92 +++++++++++++++++++++-
.../pipeline/external/conf/regression-conf.groovy | 4 +-
.../pipeline/p0/conf/regression-conf.groovy | 4 +-
.../pipeline/p1/conf/regression-conf.groovy | 4 +-
.../spark_connector/spark_connector.groovy | 3 +-
6 files changed, 106 insertions(+), 17 deletions(-)
diff --git a/regression-test/conf/regression-conf.groovy
b/regression-test/conf/regression-conf.groovy
index 220b7c0b255..3f5257e240f 100644
--- a/regression-test/conf/regression-conf.groovy
+++ b/regression-test/conf/regression-conf.groovy
@@ -81,8 +81,16 @@ brokerName = "broker_name"
// broker load test config
enableBrokerLoad=true
-ak=""
-sk=""
+
+// for s3 releated cases, "aliyun" or "aliyun-internal" or "tencent" or
"huawei" or "azure" or "gcp"
+// if s3Source is set, s3Endpoint s3BucketName s3Region s3Provider will be
filled with default value if not set
+s3Source="aliyun"
+// s3Endpoint = ""
+// s3BucketName = ""
+// s3Region = ""
+// s3Provider = ""
+ak="***********"
+sk="***********"
// jdbc connector test config
// To enable jdbc test, you need first start mysql/pg container.
@@ -161,10 +169,6 @@ aliYunSk="***********"
hwYunAk="***********"
hwYunSk="***********"
-s3Endpoint = "cos.ap-hongkong.myqcloud.com"
-s3BucketName = "doris-build-hk-1308700295"
-s3Region = "ap-hongkong"
-
// iceberg rest catalog config
iceberg_rest_uri_port=18181
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
index 2b5ecc130ce..90638778284 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/Config.groovy
@@ -105,14 +105,17 @@ class Config {
public boolean withOutLoadData
public String caseNamePrefix
+ public String s3Source
+
Config() {}
- Config(String caseNamePrefix, String defaultDb, String jdbcUrl, String
jdbcUser, String jdbcPassword,
+ Config(String s3Source, String caseNamePrefix, String defaultDb, String
jdbcUrl, String jdbcUser, String jdbcPassword,
String feSourceThriftAddress, String feTargetThriftAddress, String
feSyncerUser, String feSyncerPassword,
String syncerPassword, String feHttpAddress, String feHttpUser,
String feHttpPassword, String metaServiceHttpAddress,
String suitePath, String dataPath, String realDataPath, String
cacheDataPath, Boolean enableCacheData,
String testGroups, String excludeGroups, String testSuites, String
excludeSuites,
String testDirectories, String excludeDirectories, String
pluginPath, String sslCertificatePath) {
+ this.s3Source = s3Source
this.caseNamePrefix = caseNamePrefix
this.defaultDb = defaultDb
this.jdbcUrl = jdbcUrl
@@ -280,6 +283,16 @@ class Config {
log.info("withOutLoadData is ${config.withOutLoadData}".toString())
log.info("caseNamePrefix is ${config.caseNamePrefix}".toString())
log.info("dryRun is ${config.dryRun}".toString())
+ def s3SourceList = ["aliyun", "aliyun-internal", "tencent", "huawei",
"azure", "gcp"]
+ if (s3SourceList.contains(config.s3Source)) {
+ log.info("s3Source is ${config.s3Source}".toString())
+ log.info("s3Provider is
${config.otherConfigs.get("s3Provider")}".toString())
+ log.info("s3BucketName is
${config.otherConfigs.get("s3BucketName")}".toString())
+ log.info("s3Region is
${config.otherConfigs.get("s3Region")}".toString())
+ log.info("s3Endpoint is
${config.otherConfigs.get("s3Endpoint")}".toString())
+ } else {
+ throw new Exception("The s3Source '${config.s3Source}' is invalid,
optional values ${s3SourceList}")
+ }
Properties props = cmd.getOptionProperties("conf")
config.otherConfigs.putAll(props)
@@ -292,6 +305,7 @@ class Config {
static Config fromConfigObject(ConfigObject obj) {
def config = new Config(
+ configToString(obj.s3Source),
configToString(obj.caseNamePrefix),
configToString(obj.defaultDb),
configToString(obj.jdbcUrl),
@@ -344,6 +358,82 @@ class Config {
}
static void fillDefaultConfig(Config config) {
+ if (config.s3Source == null) {
+ config.s3Source = "aliyun"
+ log.info("Set s3Source to 'aliyun' because not
specify.".toString())
+ }
+
+ if (config.otherConfigs.get("s3Provider") == null) {
+ def s3Provider = "OSS"
+ if (config.s3Source == "aliyun" || config.s3Source ==
"aliyun-internal") {
+ s3Provider = "OSS"
+ } else if (config.s3Source == "tencent") {
+ s3Provider = "COS"
+ } else if (config.s3Source == "huawei") {
+ s3Provider = "OBS"
+ } else if (config.s3Source == "azure") {
+ s3Provider = "AZURE"
+ } else if (config.s3Source == "gcp") {
+ s3Provider = "GCP"
+ }
+ config.otherConfigs.put("s3Provider", "${s3Provider}")
+ log.info("Set s3Provider to '${s3Provider}' because not
specify.".toString())
+ }
+ if (config.otherConfigs.get("s3BucketName") == null) {
+ def s3BucketName = "doris-regression-hk"
+ if (config.s3Source == "aliyun") {
+ s3BucketName = "doris-regression-hk"
+ } else if (config.s3Source == "aliyun-internal") {
+ s3BucketName = "doris-regression"
+ } else if (config.s3Source == "tencent") {
+ s3BucketName = "doris-build-1308700295"
+ } else if (config.s3Source == "huawei") {
+ s3BucketName = "doris-build"
+ } else if (config.s3Source == "azure") {
+ s3BucketName = "qa-build"
+ } else if (config.s3Source == "gcp") {
+ s3BucketName = "doris-regression"
+ }
+ config.otherConfigs.put("s3BucketName", "${s3BucketName}")
+ log.info("Set s3BucketName to '${s3BucketName}' because not
specify.".toString())
+ }
+ if (config.otherConfigs.get("s3Region") == null) {
+ def s3Region = "oss-cn-hongkong"
+ if (config.s3Source == "aliyun") {
+ s3Region = "oss-cn-hongkong"
+ } else if (config.s3Source == "aliyun-internal") {
+ s3Region = "oss-cn-beijing"
+ } else if (config.s3Source == "tencent") {
+ s3Region = "ap-beijing"
+ } else if (config.s3Source == "huawei") {
+ s3Region = "cn-north-4"
+ } else if (config.s3Source == "azure") {
+ s3Region = "azure-region"
+ } else if (config.s3Source == "gcp") {
+ s3Region = "us-central1"
+ }
+ config.otherConfigs.put("s3Region", "${s3Region}")
+ log.info("Set s3Region to '${s3Region}' because not
specify.".toString())
+ }
+ if (config.otherConfigs.get("s3Endpoint") == null) {
+ def s3Endpoint = "oss-cn-hongkong.aliyuncs.com"
+ if (config.s3Source == "aliyun") {
+ s3Endpoint = "oss-cn-hongkong.aliyuncs.com"
+ } else if (config.s3Source == "aliyun-internal") {
+ s3Endpoint = "oss-cn-beijing-internal.aliyuncs.com"
+ } else if (config.s3Source == "tencent") {
+ s3Endpoint = "cos.ap-beijing.myqcloud.com"
+ } else if (config.s3Source == "huawei") {
+ s3Endpoint = "obs.cn-north-4.myhuaweicloud.com"
+ } else if (config.s3Source == "azure") {
+ s3Endpoint = "azure-endpoint"
+ } else if (config.s3Source == "gcp") {
+ s3Endpoint = "storage.googleapis.com"
+ }
+ config.otherConfigs.put("s3Endpoint", "${s3Endpoint}")
+ log.info("Set s3Endpoint to '${s3Endpoint}' because not
specify.".toString())
+ }
+
if (config.caseNamePrefix == null) {
config.caseNamePrefix = ""
log.info("set caseNamePrefix to '' because not
specify.".toString())
diff --git a/regression-test/pipeline/external/conf/regression-conf.groovy
b/regression-test/pipeline/external/conf/regression-conf.groovy
index 5f7ac8f563d..e35b81d6844 100644
--- a/regression-test/pipeline/external/conf/regression-conf.groovy
+++ b/regression-test/pipeline/external/conf/regression-conf.groovy
@@ -119,9 +119,7 @@ es_8_port=39200
cacheDataPath = "/data/regression/"
-s3Endpoint = "cos.ap-hongkong.myqcloud.com"
-s3BucketName = "doris-build-hk-1308700295"
-s3Region = "ap-hongkong"
+s3Source="aliyun"
max_failure_num=50
diff --git a/regression-test/pipeline/p0/conf/regression-conf.groovy
b/regression-test/pipeline/p0/conf/regression-conf.groovy
index 57257a07bf2..11486e2dc09 100644
--- a/regression-test/pipeline/p0/conf/regression-conf.groovy
+++ b/regression-test/pipeline/p0/conf/regression-conf.groovy
@@ -118,9 +118,7 @@ es_8_port=39200
cacheDataPath = "/data/regression/"
-s3Endpoint = "cos.ap-hongkong.myqcloud.com"
-s3BucketName = "doris-build-hk-1308700295"
-s3Region = "ap-hongkong"
+s3Source="aliyun"
max_failure_num=50
diff --git a/regression-test/pipeline/p1/conf/regression-conf.groovy
b/regression-test/pipeline/p1/conf/regression-conf.groovy
index 98f0d9173b2..9df3489871a 100644
--- a/regression-test/pipeline/p1/conf/regression-conf.groovy
+++ b/regression-test/pipeline/p1/conf/regression-conf.groovy
@@ -57,9 +57,7 @@ excludeSuites =
"test_profile,test_broker_load_p2,test_spark_load,test_refresh_m
excludeDirectories = "workload_manager_p1"
cacheDataPath="/data/regression/"
-s3Endpoint = "cos.ap-hongkong.myqcloud.com"
-s3BucketName = "doris-build-hk-1308700295"
-s3Region = "ap-hongkong"
+s3Source="aliyun"
max_failure_num=0
diff --git
a/regression-test/suites/connector_p0/spark_connector/spark_connector.groovy
b/regression-test/suites/connector_p0/spark_connector/spark_connector.groovy
index ecd4e6dfc14..55c2aaad9eb 100644
--- a/regression-test/suites/connector_p0/spark_connector/spark_connector.groovy
+++ b/regression-test/suites/connector_p0/spark_connector/spark_connector.groovy
@@ -25,7 +25,8 @@ suite("spark_connector", "connector") {
logger.info("start download spark doris demo ...")
logger.info("getS3Url ==== ${getS3Url()}")
def download_spark_jar = "/usr/bin/curl
${getS3Url()}/regression/spark-doris-connector-demo-jar-with-dependencies.jar
--output spark-doris-demo.jar".execute().getText()
- logger.info("finish download spark doris demo ...")
+ def out = "/usr/bin/ls -al spark-doris-demo.jar".execute().getText()
+ logger.info("finish download spark doris demo, out: ${out}")
def run_cmd = "java -jar spark-doris-demo.jar
$context.config.feHttpAddress $context.config.feHttpUser
regression_test_connector_p0_spark_connector.$tableName"
logger.info("run_cmd : $run_cmd")
def run_spark_jar = run_cmd.execute().getText()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
