This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 3043d95d3ed [regression-test](framework) support Non concurrent mode
#26487 (#26574)
3043d95d3ed is described below
commit 3043d95d3edb3b0d6622f5c018d4ec8119df0422
Author: shuke <37901441+shuke...@users.noreply.github.com>
AuthorDate: Thu Nov 9 18:44:28 2023 +0800
[regression-test](framework) support Non concurrent mode #26487 (#26574)
---
.../apache/doris/regression/RegressionTest.groovy | 59 +++++++++++++++++++---
.../pipeline/p0/conf/regression-conf.groovy | 2 +-
.../sql_block_rule_p0/test_sql_block_rule.groovy | 2 +-
3 files changed, 55 insertions(+), 8 deletions(-)
diff --git
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy
index 4e7a5793aa5..6cdb918a7f4 100644
---
a/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy
+++
b/regression-test/framework/src/main/groovy/org/apache/doris/regression/RegressionTest.groovy
@@ -50,9 +50,11 @@ class RegressionTest {
static GroovyShell shell
static ExecutorService scriptExecutors
static ExecutorService suiteExecutors
+ static ExecutorService singleSuiteExecutors
static ExecutorService actionExecutors
static ThreadLocal<Integer> threadLoadedClassNum = new ThreadLocal<>()
static final int cleanLoadedClassesThreshold = 20
+ static String nonConcurrentTestGroup = "nonConcurrent"
static void main(String[] args) {
CommandLine cmd = ConfigOptions.initCommands(args)
@@ -70,6 +72,7 @@ class RegressionTest {
}
actionExecutors.shutdown()
suiteExecutors.shutdown()
+ singleSuiteExecutors.shutdown()
scriptExecutors.shutdown()
log.info("Test finished")
if (!success) {
@@ -96,6 +99,12 @@ class RegressionTest {
.build();
suiteExecutors = Executors.newFixedThreadPool(config.suiteParallel,
suiteFactory)
+ BasicThreadFactory singleSuiteFactory = new
BasicThreadFactory.Builder()
+ .namingPattern("non-concurrent-thread-%d")
+ .priority(Thread.MAX_PRIORITY)
+ .build();
+ singleSuiteExecutors = Executors.newFixedThreadPool(1,
singleSuiteFactory)
+
BasicThreadFactory actionFactory = new BasicThreadFactory.Builder()
.namingPattern("action-thread-%d")
.priority(Thread.MAX_PRIORITY)
@@ -131,9 +140,9 @@ class RegressionTest {
return sources
}
- static void runScript(Config config, ScriptSource source, Recorder
recorder) {
+ static void runScript(Config config, ScriptSource source, Recorder
recorder, boolean isSingleThreadScript) {
def suiteFilter = { String suiteName, String groupName ->
- canRun(config, suiteName, groupName)
+ canRun(config, suiteName, groupName, isSingleThreadScript)
}
def file = source.getFile()
int failureLimit =
Integer.valueOf(config.otherConfigs.getOrDefault("max_failure_num",
"-1").toString());
@@ -144,7 +153,14 @@ class RegressionTest {
return;
}
def eventListeners = getEventListeners(config, recorder)
- new ScriptContext(file, suiteExecutors, actionExecutors,
+ ExecutorService executors = null
+ if (isSingleThreadScript) {
+ executors = singleSuiteExecutors
+ } else {
+ executors = suiteExecutors
+ }
+
+ new ScriptContext(file, executors, actionExecutors,
config, eventListeners, suiteFilter).start { scriptContext ->
try {
SuiteScript suiteScript = source.toScript(scriptContext, shell)
@@ -168,7 +184,26 @@ class RegressionTest {
scriptSources.eachWithIndex { source, i ->
// log.info("Prepare scripts [${i + 1}/${totalFile}]".toString())
def future = scriptExecutors.submit {
- runScript(config, source, recorder)
+ runScript(config, source, recorder, false)
+ }
+ futures.add(future)
+ }
+
+ // wait all scripts
+ for (Future future : futures) {
+ try {
+ future.get()
+ } catch (Throwable t) {
+ // do nothing, because already save to Recorder
+ }
+ }
+
+ log.info('Start to run single scripts')
+ futures.clear()
+ scriptSources.eachWithIndex { source, i ->
+// log.info("Prepare scripts [${i + 1}/${totalFile}]".toString())
+ def future = scriptExecutors.submit {
+ runScript(config, source, recorder, true)
}
futures.add(future)
}
@@ -192,8 +227,9 @@ class RegressionTest {
{ fileName -> fileName.substring(0,
fileName.lastIndexOf(".")) == "load" })
}
log.info('Start to run scripts')
- runScripts(config, recorder, directoryFilter,
+ runScripts(config, recorder, directoryFilter,
{ fileName -> fileName.substring(0, fileName.lastIndexOf("."))
!= "load" })
+
return recorder
}
@@ -228,7 +264,18 @@ class RegressionTest {
return true
}
- static boolean canRun(Config config, String suiteName, String group) {
+ static boolean canRun(Config config, String suiteName, String group,
boolean isSingleThreadScript) {
+ Set<String> suiteGroups = group.split(',').collect { g -> g.trim()
}.toSet();
+ if (isSingleThreadScript) {
+ if (!suiteGroups.contains(nonConcurrentTestGroup)) {
+ return false
+ }
+ } else {
+ if (suiteGroups.contains(nonConcurrentTestGroup)) {
+ return false
+ }
+ }
+
return filterGroups(config, group) && filterSuites(config, suiteName)
}
diff --git a/regression-test/pipeline/p0/conf/regression-conf.groovy
b/regression-test/pipeline/p0/conf/regression-conf.groovy
index 7551bc086b3..445161bcd2c 100644
--- a/regression-test/pipeline/p0/conf/regression-conf.groovy
+++ b/regression-test/pipeline/p0/conf/regression-conf.groovy
@@ -54,7 +54,7 @@ testDirectories = ""
// this groups will not be executed
excludeGroups = ""
// this suites will not be executed
-excludeSuites =
"test_sql_block_rule,test_outfile_exception,test_digest,test_aggregate_all_functions2,test_cast_with_scale_type,test_hive_read_orc_complex_type,test_with_and_two_phase_agg,explode,test_cast_function,test_profile,test_broker_load_p2,test_spark_load,test_analyze_stats_p1,test_refresh_mtmv,test_bitmap_filter,test_export_parquet,test_doris_jdbc_catalog"
+excludeSuites =
"test_outfile_exception,test_digest,test_aggregate_all_functions2,test_cast_with_scale_type,test_hive_read_orc_complex_type,test_with_and_two_phase_agg,explode,test_cast_function,test_profile,test_broker_load_p2,test_spark_load,test_analyze_stats_p1,test_refresh_mtmv,test_bitmap_filter,test_export_parquet,test_doris_jdbc_catalog"
// this directories will not be executed
excludeDirectories =
"nereids_tpcds_shape_sf100_p0,nereids_tpch_shape_sf1000_p0,nereids_tpch_shape_sf500_p0,workload_manager_p1,fault_injection_p0"
diff --git
a/regression-test/suites/sql_block_rule_p0/test_sql_block_rule.groovy
b/regression-test/suites/sql_block_rule_p0/test_sql_block_rule.groovy
index 978486d8afd..6308376fcbb 100644
--- a/regression-test/suites/sql_block_rule_p0/test_sql_block_rule.groovy
+++ b/regression-test/suites/sql_block_rule_p0/test_sql_block_rule.groovy
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-suite("test_sql_block_rule") {
+suite("test_sql_block_rule", "nonConcurrent") {
sql """
DROP SQL_BLOCK_RULE if exists test_rule_partition
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
