This is an automated email from the ASF dual-hosted git repository.
zhangstar333 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ca5bb62817f [chore](ci) remove some unused code (#30253)
ca5bb62817f is described below
commit ca5bb62817f7dd7b25aef8051fd1303bec5403e1
Author: Dongyang Li <hello_step...@qq.com>
AuthorDate: Tue Jan 23 15:18:31 2024 +0800
[chore](ci) remove some unused code (#30253)
Co-authored-by: stephen <hello-step...@qq.com>
---
.github/workflows/comment-to-trigger-teamcity.yml | 21 +---
regression-test/pipeline/common/doris-utils.sh | 12 +-
regression-test/pipeline/common/github-utils.sh | 25 ----
regression-test/pipeline/performance/run-tpcds.sh | 1 +
regression-test/pipeline/performance/run-tpch.sh | 1 +
regression-test/pipeline/tpch/tpch-sf100/clean.sh | 40 -------
.../pipeline/tpch/tpch-sf100/conf/be.conf | 85 -------------
.../pipeline/tpch/tpch-sf100/conf/external.json | 26 ----
.../pipeline/tpch/tpch-sf100/conf/fe.conf | 78 ------------
.../pipeline/tpch/tpch-sf100/conf/odbcinst.ini | 43 -------
regression-test/pipeline/tpch/tpch-sf100/deploy.sh | 119 -------------------
.../pipeline/tpch/tpch-sf100/prepare.sh | 69 -----------
regression-test/pipeline/tpch/tpch-sf100/run.sh | 131 ---------------------
13 files changed, 9 insertions(+), 642 deletions(-)
diff --git a/.github/workflows/comment-to-trigger-teamcity.yml
b/.github/workflows/comment-to-trigger-teamcity.yml
index 0fdd80ba28f..8ae0832fc9e 100644
--- a/.github/workflows/comment-to-trigger-teamcity.yml
+++ b/.github/workflows/comment-to-trigger-teamcity.yml
@@ -45,7 +45,6 @@ jobs:
"${COMMENT_BODY}" == *'run p1'* ||
"${COMMENT_BODY}" == *'run external'* ||
"${COMMENT_BODY}" == *'run pipelinex_p0'* ||
- "${COMMENT_BODY}" == *'run clickbench'* ||
"${COMMENT_BODY}" == *'run arm'* ||
"${COMMENT_BODY}" == *'run performance'* ]]; then
echo "comment_trigger=true" | tee -a "$GITHUB_OUTPUT"
@@ -63,7 +62,7 @@ jobs:
echo "TARGET_BRANCH='${TARGET_BRANCH}'" | tee -a "$GITHUB_OUTPUT"
echo "COMMENT_BODY='${COMMENT_BODY}'" | tee -a "$GITHUB_OUTPUT"
- reg="run
(buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|performance)(
[1-9]*[0-9]+)*"
+ reg="run
(buildall|compile|p0|p1|feut|beut|external|pipelinex_p0|arm|performance)(
[1-9]*[0-9]+)*"
COMMENT_TRIGGER_TYPE="$(echo -e "${COMMENT_BODY}" | xargs | grep -E
"${reg}" | awk -F' ' '{print $2}' | sed -n 1p | sed 's/\r//g')"
COMMENT_REPEAT_TIMES="$(echo -e "${COMMENT_BODY}" | xargs | grep -E
"${reg}" | awk -F' ' '{print $3}' | sed -n 1p | sed 's/\r//g')"
echo "COMMENT_TRIGGER_TYPE=${COMMENT_TRIGGER_TYPE}" | tee -a
"$GITHUB_OUTPUT"
@@ -90,11 +89,6 @@ jobs:
else
echo "changed_be_ut=false" | tee -a "$GITHUB_OUTPUT"
fi
- if file_changed_ckb; then
- echo "changed_ckb=true" | tee -a "$GITHUB_OUTPUT"
- else
- echo "changed_ckb=false" | tee -a "$GITHUB_OUTPUT"
- fi
if file_changed_regression_p0; then
echo "changed_p0=true" | tee -a "$GITHUB_OUTPUT"
echo "changed_external=true" | tee -a "$GITHUB_OUTPUT"
@@ -120,7 +114,6 @@ jobs:
echo "INFO: failed to _get_pr_changed_files, default trigger all"
echo "changed_fe_ut=true" | tee -a "$GITHUB_OUTPUT"
echo "changed_be_ut=true" | tee -a "$GITHUB_OUTPUT"
- echo "changed_ckb=true" | tee -a "$GITHUB_OUTPUT"
echo "changed_p0=true" | tee -a "$GITHUB_OUTPUT"
echo "changed_external=true" | tee -a "$GITHUB_OUTPUT"
echo "changed_pipelinex_p0=true" | tee -a "$GITHUB_OUTPUT"
@@ -243,18 +236,6 @@ jobs:
"arm" \
"${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
- - name: "Trigger or Skip clickbench"
- if: ${{ fromJSON(steps.parse.outputs.comment_trigger) &&
contains(fromJSON('["clickbench", "buildall"]'),
steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
- run: |
- source ./regression-test/pipeline/common/teamcity-utils.sh
- set -x
- trigger_or_skip_build \
- "${{ steps.changes.outputs.changed_ckb }}" \
- "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
- "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
- "clickbench" \
- "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
-
- name: "Trigger or Skip performance"
if: ${{ fromJSON(steps.parse.outputs.comment_trigger) &&
contains(fromJSON('["performance", "buildall"]'),
steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
run: |
diff --git a/regression-test/pipeline/common/doris-utils.sh
b/regression-test/pipeline/common/doris-utils.sh
index 47bcedad351..327f00beac1 100644
--- a/regression-test/pipeline/common/doris-utils.sh
+++ b/regression-test/pipeline/common/doris-utils.sh
@@ -394,18 +394,18 @@ archive_doris_logs() {
print_doris_fe_log() {
if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
- echo "WARNING: --------------------tail -n 100
${DORIS_HOME}/fe/log/fe.out--------------------"
+ echo -e "\n\n\n\nWARNING: --------------------tail -n 100
${DORIS_HOME}/fe/log/fe.out--------------------"
tail -n 100 "${DORIS_HOME}"/fe/log/fe.out
- echo "WARNING: --------------------tail -n 100
${DORIS_HOME}/fe/log/fe.log--------------------"
+ echo -e "\n\n\n\nWARNING: --------------------tail -n 100
${DORIS_HOME}/fe/log/fe.log--------------------"
tail -n 100 "${DORIS_HOME}"/fe/log/fe.log
- echo "WARNING: ----------------------------------------"
+ echo -e "WARNING: ----------------------------------------\n\n\n\n"
}
print_doris_be_log() {
if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
- echo "WARNING: --------------------tail -n 100
${DORIS_HOME}/be/log/be.out--------------------"
+ echo -e "\n\n\n\nWARNING: --------------------tail -n 100
${DORIS_HOME}/be/log/be.out--------------------"
tail -n 100 "${DORIS_HOME}"/be/log/be.out
- echo "WARNING: --------------------tail -n 100
${DORIS_HOME}/be/log/be.INFO--------------------"
+ echo -e "\n\n\n\nWARNING: --------------------tail -n 100
${DORIS_HOME}/be/log/be.INFO--------------------"
tail -n 100 "${DORIS_HOME}"/be/log/be.INFO
- echo "WARNING: ----------------------------------------"
+ echo -e "WARNING: ----------------------------------------\n\n\n\n"
}
diff --git a/regression-test/pipeline/common/github-utils.sh
b/regression-test/pipeline/common/github-utils.sh
index 317667d15b2..5a9d2ebb7b3 100644
--- a/regression-test/pipeline/common/github-utils.sh
+++ b/regression-test/pipeline/common/github-utils.sh
@@ -304,31 +304,6 @@ file_changed_regression_p1() {
file_changed_regression_p0
}
-file_changed_ckb() {
- local all_files
- all_files=$(cat all_files)
- if _only_modified_regression_conf; then echo "return no need" && return 1;
fi
- if [[ -z ${all_files} ]]; then echo "return need" && return 0; fi
- for af in ${all_files}; do
- if [[ "${af}" == 'be'* ]] ||
- [[ "${af}" == 'bin'* ]] ||
- [[ "${af}" == 'conf'* ]] ||
- [[ "${af}" == 'fe'* ]] ||
- [[ "${af}" == 'gensrc'* ]] ||
- [[ "${af}" == 'thirdparty'* ]] ||
- [[ "${af}" == 'build.sh' ]] ||
- [[ "${af}" == 'env.sh' ]] ||
- [[ "${af}" == 'regression-test/pipeline/common/github-utils.sh' ]]
||
- [[ "${af}" == 'regression-test/pipeline/common/doris-utils.sh' ]]
||
- [[ "${af}" == 'regression-test/pipeline/common/oss-utils.sh' ]] ||
- [[ "${af}" == 'tools/tpch-tools/bin/run-tpch-queries.sh' ]] ||
- [[ "${af}" == 'regression-test/pipeline/tpch/tpch-sf100/'* ]]; then
- echo "clickbench performance related file changed, return need" &&
return 0
- fi
- done
- echo "return no need" && return 1
-}
-
file_changed_performance() {
local all_files
all_files=$(cat all_files)
diff --git a/regression-test/pipeline/performance/run-tpcds.sh
b/regression-test/pipeline/performance/run-tpcds.sh
index d054b85ee4d..0ca31c57e74 100644
--- a/regression-test/pipeline/performance/run-tpcds.sh
+++ b/regression-test/pipeline/performance/run-tpcds.sh
@@ -125,6 +125,7 @@ exit_flag=0
echo "#### 3. run tpcds-sf${SF} query"
set_session_variable runtime_filter_mode global
bash
"${teamcity_build_checkoutDir}"/tools/tpcds-tools/bin/run-tpcds-queries.sh -s
"${SF}" | tee "${teamcity_build_checkoutDir}"/run-tpcds-queries.log
+ echo
cold_run_time_threshold=${cold_run_time_threshold_master:-315000} # ms
hot_run_time_threshold=${hot_run_time_threshold_master:-190000} # ms
if [[ "${target_branch}" == "branch-2.0" ]]; then
diff --git a/regression-test/pipeline/performance/run-tpch.sh
b/regression-test/pipeline/performance/run-tpch.sh
index 1620c5f2922..f1e4c8df2a2 100644
--- a/regression-test/pipeline/performance/run-tpch.sh
+++ b/regression-test/pipeline/performance/run-tpch.sh
@@ -117,6 +117,7 @@ exit_flag=0
echo "#### 3. run tpch-sf${SF} query"
set_session_variable runtime_filter_mode global
bash
"${teamcity_build_checkoutDir}"/tools/tpch-tools/bin/run-tpch-queries.sh -s
"${SF}" | tee "${teamcity_build_checkoutDir}"/run-tpch-queries.log
+ echo
cold_run_time_threshold=${cold_run_time_threshold_master:-120000} # ms
hot_run_time_threshold=${hot_run_time_threshold_master:-42000} # ms
if [[ "${target_branch}" == "branch-2.0" ]]; then
diff --git a/regression-test/pipeline/tpch/tpch-sf100/clean.sh
b/regression-test/pipeline/tpch/tpch-sf100/clean.sh
deleted file mode 100644
index 16fa490ef9c..00000000000
--- a/regression-test/pipeline/tpch/tpch-sf100/clean.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Build Step: Command Line
-: <<EOF
-#!/bin/bash
-
-# Execute step even if some of the previous steps failed
-teamcity_build_checkoutDir="%teamcity.build.checkoutDir%"
-if [[ -f
"${teamcity_build_checkoutDir:-}"/regression-test/pipeline/tpch/tpch-sf100/clean.sh
]]; then
- cd
"${teamcity_build_checkoutDir}"/regression-test/pipeline/tpch/tpch-sf100/
- bash -x clean.sh
-else
- echo "Build Step file missing:
regression-test/pipeline/tpch/tpch-sf100/clean.sh" && exit 1
-fi
-EOF
-
-## clean.sh content ##
-
-# stop_doris
-source ../../common/doris-utils.sh
-
-DORIS_HOME="${teamcity_build_checkoutDir}/output"
-export DORIS_HOME
-stop_doris
diff --git a/regression-test/pipeline/tpch/tpch-sf100/conf/be.conf
b/regression-test/pipeline/tpch/tpch-sf100/conf/be.conf
deleted file mode 100644
index cc23ae153bf..00000000000
--- a/regression-test/pipeline/tpch/tpch-sf100/conf/be.conf
+++ /dev/null
@@ -1,85 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-CUR_DATE=`date +%Y%m%d-%H%M%S`
-
-PPROF_TMPDIR="$DORIS_HOME/log/"
-
-JAVA_OPTS="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log
-Xloggc:$DORIS_HOME/log/be.gc.log.$CUR_DATE
-Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true
-Dsun.java.command=DorisBE -XX:-CriticalJNINatives -DJDBC_MIN_POOL=1
-DJDBC_MAX_POOL=100 -DJDBC_MAX_IDLE_TIME=300000 -DJDBC_MAX_WAIT_TIME=5000"
-
-# For jdk 9+, this JAVA_OPTS will be used as default JVM options
-JAVA_OPTS_FOR_JDK_9="-Xmx1024m -DlogPath=$DORIS_HOME/log/jni.log
-Xlog:gc:$DORIS_HOME/log/be.gc.log.$CUR_DATE
-Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true
-Dsun.java.command=DorisBE -XX:-CriticalJNINatives -DJDBC_MIN_POOL=1
-DJDBC_MAX_POOL=100 -DJDBC_MAX_IDLE_TIME=300000 -DJDBC_MAX_WAIT_TIME=5000"
-
-# since 1.2, the JAVA_HOME need to be set to run BE process.
-# JAVA_HOME=/path/to/jdk/
-
-#
https://github.com/apache/doris/blob/master/docs/zh-CN/community/developer-guide/debug-tool.md#jemalloc-heap-profile
-# https://jemalloc.net/jemalloc.3.html
-JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:15000,dirty_decay_ms:15000,oversize_threshold:0,lg_tcache_max:20,prof:false,lg_prof_interval:32,lg_prof_sample:19,prof_gdump:false,prof_accum:false,prof_leak:false,prof_final:false"
-JEMALLOC_PROF_PRFIX=""
-
-# INFO, WARNING, ERROR, FATAL
-sys_log_level = INFO
-
-# ports for admin, web, heartbeat service
-be_port = 9060
-webserver_port = 8040
-heartbeat_service_port = 9050
-brpc_port = 8060
-arrow_flight_sql_port = -1
-
-# HTTPS configures
-enable_https = false
-# path of certificate in PEM format.
-ssl_certificate_path = "$DORIS_HOME/conf/cert.pem"
-# path of private key in PEM format.
-ssl_private_key_path = "$DORIS_HOME/conf/key.pem"
-
-
-# Choose one if there are more than one ip except loopback address.
-# Note that there should at most one ip match this list.
-# If no ip match this rule, will choose one randomly.
-# use CIDR format, e.g. 10.10.10.0/24 or IP format, e.g. 10.10.10.1
-# Default value is empty.
-# priority_networks = 10.10.10.0/24;192.168.0.0/16
-
-# data root path, separate by ';'
-# You can specify the storage type for each root path, HDD (cold data) or SSD
(hot data)
-# eg:
-# storage_root_path = /home/disk1/doris;/home/disk2/doris;/home/disk2/doris
-# storage_root_path =
/home/disk1/doris,medium:SSD;/home/disk2/doris,medium:SSD;/home/disk2/doris,medium:HDD
-# /home/disk2/doris,medium:HDD(default)
-#
-# you also can specify the properties by setting '<property>:<value>',
separate by ','
-# property 'medium' has a higher priority than the extension of path
-#
-# Default value is ${DORIS_HOME}/storage, you should create it by hand.
-# storage_root_path = ${DORIS_HOME}/storage
-
-# Default dirs to put jdbc drivers,default value is ${DORIS_HOME}/jdbc_drivers
-# jdbc_drivers_dir = ${DORIS_HOME}/jdbc_drivers
-
-# Advanced configurations
-# sys_log_dir = ${DORIS_HOME}/log
-# sys_log_roll_mode = SIZE-MB-1024
-# sys_log_roll_num = 10
-# sys_log_verbose_modules = *
-# log_buffer_level = -1
-# palo_cgroups
-
-priority_networks=172.16.0.0/24
-storage_root_path=/mnt/datadisk0/doris-storage
diff --git a/regression-test/pipeline/tpch/tpch-sf100/conf/external.json
b/regression-test/pipeline/tpch/tpch-sf100/conf/external.json
deleted file mode 100644
index 9461d836e67..00000000000
--- a/regression-test/pipeline/tpch/tpch-sf100/conf/external.json
+++ /dev/null
@@ -1,26 +0,0 @@
-[
- {
- "file": "docker/thirdparties/docker-compose/mysql/mysql-5.7.env",
- "replacements": {
- "DOCKER_MYSQL_57_EXTERNAL_PORT": 7111
- }
- },
- {
- "file": "docker/thirdparties/docker-compose/postgresql/postgresql-14.env",
- "replacements": {
- "DOCKER_PG_14_EXTERNAL_PORT": 7121
- }
- },
- {
- "file": "docker/thirdparties/docker-compose/hive/gen_env.sh",
- "replacements": {
- "FS_PORT": 7131,
- "HMS_PORT": 7141
- }
- }, {
- "file": "docker/thirdparties/start-thirdparties-docker.sh",
- "replacements": {
- "CONTAINER_UID": "doris-regression-fakeid-fakecommit"
- }
- }
-]
diff --git a/regression-test/pipeline/tpch/tpch-sf100/conf/fe.conf
b/regression-test/pipeline/tpch/tpch-sf100/conf/fe.conf
deleted file mode 100644
index 7c02d3898dc..00000000000
--- a/regression-test/pipeline/tpch/tpch-sf100/conf/fe.conf
+++ /dev/null
@@ -1,78 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#####################################################################
-## The uppercase properties are read and exported by bin/start_fe.sh.
-## To see all Frontend configurations,
-## see fe/src/org/apache/doris/common/Config.java
-#####################################################################
-
-CUR_DATE=`date +%Y%m%d-%H%M%S`
-
-# the output dir of stderr and stdout
-LOG_DIR = ${DORIS_HOME}/log
-
-JAVA_OPTS="-Dsun.security.krb5.debug=true
-Djavax.security.auth.useSubjectCredsOnly=false -Xss4m -Xmx8192m -XX:+UseMembar
-XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps
-XX:+PrintGCDetails -XX:+UseConcMarkSweepGC -XX:+UseParNewGC
-XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled
-XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0
-Xloggc:$DORIS_HOME/log/fe.gc.log.$CUR_DATE"
-
-# For jdk 9+, this JAVA_OPTS will be used as default JVM options
-JAVA_OPTS_FOR_JDK_9="-Dsun.security.krb5.debug=true
-Djavax.security.auth.useSubjectCredsOnly=false -Xss4m -Xmx8192m
-XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+CMSClassUnloadingEnabled
-XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80
-XX:SoftRefLRUPolicyMSPerMB=0
-Xlog:gc*:$DORIS_HOME/log/fe.gc.log.$CUR_DATE:time"
-
-##
-## the lowercase properties are read by main program.
-##
-
-# INFO, WARN, ERROR, FATAL
-sys_log_level = INFO
-
-# NORMAL, BRIEF, ASYNC
-sys_log_mode = NORMAL
-
-# store metadata, must be created before start FE.
-# Default value is ${DORIS_HOME}/doris-meta
-# meta_dir = ${DORIS_HOME}/doris-meta
-
-# Default dirs to put jdbc drivers,default value is ${DORIS_HOME}/jdbc_drivers
-# jdbc_drivers_dir = ${DORIS_HOME}/jdbc_drivers
-
-http_port = 8030
-rpc_port = 9020
-query_port = 9030
-edit_log_port = 9010
-arrow_flight_sql_port = -1
-
-# Choose one if there are more than one ip except loopback address.
-# Note that there should at most one ip match this list.
-# If no ip match this rule, will choose one randomly.
-# use CIDR format, e.g. 10.10.10.0/24 or IP format, e.g. 10.10.10.1
-# Default value is empty.
-# priority_networks = 10.10.10.0/24;192.168.0.0/16
-
-# Advanced configurations
-# log_roll_size_mb = 1024
-# sys_log_dir = ${DORIS_HOME}/log
-# sys_log_roll_num = 10
-# sys_log_verbose_modules = org.apache.doris
-# audit_log_dir = ${DORIS_HOME}/log
-# audit_log_modules = slow_query, query
-# audit_log_roll_num = 10
-# meta_delay_toleration_second = 10
-# qe_max_connection = 1024
-# qe_query_timeout_second = 300
-# qe_slow_log_ms = 5000
-
-priority_networks=172.16.0.0/24
-meta_dir=/mnt/datadisk0/doris-meta
diff --git a/regression-test/pipeline/tpch/tpch-sf100/conf/odbcinst.ini
b/regression-test/pipeline/tpch/tpch-sf100/conf/odbcinst.ini
deleted file mode 100644
index 41e21f92277..00000000000
--- a/regression-test/pipeline/tpch/tpch-sf100/conf/odbcinst.ini
+++ /dev/null
@@ -1,43 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Example driver definitions, you should not use the example odbc driver
-# before you prepare env in your server
-
-# Driver from the postgresql-odbc package
-# Setup from the unixODBC package
-[PostgreSQL]
-Description = ODBC for PostgreSQL
-Driver = /usr/lib/psqlodbc.so
-Setup = /usr/lib/libodbcpsqlS.so
-FileUsage = 1
-
-
-# Driver from the mysql-connector-odbc package
-# Setup from the unixODBC package
-[MySQL ODBC 8.0 Unicode Driver]
-Description = ODBC for MySQL
-Driver = /usr/lib64/libmyodbc8w.so
-FileUsage = 1
-
-# Driver from the oracle-connector-odbc package
-# Setup from the unixODBC package
-[Oracle 19 ODBC driver]
-Description=Oracle ODBC driver for Oracle 19
-Driver=/usr/lib/libsqora.so.19.1
-
-
diff --git a/regression-test/pipeline/tpch/tpch-sf100/deploy.sh
b/regression-test/pipeline/tpch/tpch-sf100/deploy.sh
deleted file mode 100644
index 64d816f0aca..00000000000
--- a/regression-test/pipeline/tpch/tpch-sf100/deploy.sh
+++ /dev/null
@@ -1,119 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Build Step: Command Line
-: <<EOF
-#!/bin/bash
-
-teamcity_build_checkoutDir="%teamcity.build.checkoutDir%"
-if [[ -f
"${teamcity_build_checkoutDir:-}"/regression-test/pipeline/tpch/tpch-sf100/deploy.sh
]]; then
- cd
"${teamcity_build_checkoutDir}"/regression-test/pipeline/tpch/tpch-sf100/
- bash -x deploy.sh
-else
- echo "Build Step file missing:
regression-test/pipeline/tpch/tpch-sf100/deploy.sh" && exit 1
-fi
-EOF
-
-## deploy.sh content ##
-
-# download_oss_file
-source ../../common/oss-utils.sh
-# start_doris_fe, get_doris_conf_value, start_doris_be, stop_doris,
-# print_doris_fe_log, print_doris_be_log, archive_doris_logs
-source ../../common/doris-utils.sh
-
-echo "#### Check env"
-if [[ -z "${teamcity_build_checkoutDir}" ||
- -z "${pull_request_id}" ||
- -z "${commit_id}" ]]; then
- echo "ERROR: env teamcity_build_checkoutDir or pull_request_id or
commit_id not set"
- exit 1
-fi
-if ${DEBUG:-false}; then
- pull_request_id="26465"
- commit_id="a532f7113f463e144e83918a37288f2649448482"
-fi
-
-echo "#### Deploy Doris ####"
-DORIS_HOME="${teamcity_build_checkoutDir}/output"
-export DORIS_HOME
-exit_flag=0
-need_backup_doris_logs=false
-
-echo "#### 1. try to kill old doris process and remove old doris binary"
-stop_doris && rm -rf output
-
-echo "#### 2. download doris binary tar ball"
-cd "${teamcity_build_checkoutDir}" || exit 1
-if download_oss_file "${pull_request_id:-}_${commit_id:-}.tar.gz"; then
- if ! command -v pigz >/dev/null; then sudo apt install -y pigz; fi
- tar -I pigz -xf "${pull_request_id:-}_${commit_id:-}.tar.gz"
- if [[ -d output && -d output/fe && -d output/be ]]; then
- echo "INFO: be version: $(./output/be/lib/doris_be --version)"
- rm -rf "${pull_request_id}_${commit_id}.tar.gz"
- fi
-else
- echo "ERROR: download compiled binary failed" && exit 1
-fi
-
-echo "#### 3. copy conf from regression-test/pipeline/tpch/tpch-sf100/conf/"
-rm -f "${DORIS_HOME}"/fe/conf/fe_custom.conf
"${DORIS_HOME}"/be/conf/be_custom.conf
-if [[ -f
"${teamcity_build_checkoutDir}"/regression-test/pipeline/tpch/tpch-sf100/conf/fe.conf
&&
- -f
"${teamcity_build_checkoutDir}"/regression-test/pipeline/tpch/tpch-sf100/conf/be.conf
]]; then
- cp -f
"${teamcity_build_checkoutDir}"/regression-test/pipeline/tpch/tpch-sf100/conf/fe.conf
"${DORIS_HOME}"/fe/conf/
- cp -f
"${teamcity_build_checkoutDir}"/regression-test/pipeline/tpch/tpch-sf100/conf/be.conf
"${DORIS_HOME}"/be/conf/
-else
- echo "ERROR: doris conf file missing in
${teamcity_build_checkoutDir}/regression-test/pipeline/tpch/tpch-sf100/conf/"
- exit 1
-fi
-
-echo "#### 4. start Doris"
-meta_dir=$(get_doris_conf_value "${DORIS_HOME}"/fe/conf/fe.conf meta_dir)
-storage_root_path=$(get_doris_conf_value "${DORIS_HOME}"/be/conf/be.conf
storage_root_path)
-mkdir -p "${meta_dir}"
-mkdir -p "${storage_root_path}"
-if ! start_doris_fe; then
- echo "ERROR: Start doris fe failed."
- print_doris_fe_log
- need_backup_doris_logs=true
- exit_flag=1
-fi
-if ! start_doris_be; then
- echo "ERROR: Start doris be failed."
- print_doris_be_log
- need_backup_doris_logs=true
- exit_flag=1
-fi
-
-# wait 10s for doris totally started, otherwize may encounter the error below,
-# ERROR 1105 (HY000) at line 102: errCode = 2, detailMessage = Failed to find
enough backend, please check the replication num,replication tag and storage
medium.
-sleep 10s
-
-echo "#### 5. set session variables"
-echo "TODO"
-
-echo "#### 6. check if need backup doris logs"
-if ${need_backup_doris_logs}; then
- print_doris_fe_log
- print_doris_be_log
- if file_name=$(archive_doris_logs
"${pull_request_id}_${commit_id}_doris_logs.tar.gz"); then
- upload_doris_log_to_oss "${file_name}"
- fi
-fi
-
-exit "${exit_flag}"
diff --git a/regression-test/pipeline/tpch/tpch-sf100/prepare.sh
b/regression-test/pipeline/tpch/tpch-sf100/prepare.sh
deleted file mode 100644
index 07959a03d6f..00000000000
--- a/regression-test/pipeline/tpch/tpch-sf100/prepare.sh
+++ /dev/null
@@ -1,69 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Build Step: Command Line
-: <<EOF
-#!/bin/bash
-
-teamcity_build_checkoutDir="%teamcity.build.checkoutDir%"
-if [[ -f
"${teamcity_build_checkoutDir:-}"/regression-test/pipeline/tpch/tpch-sf100/prepare.sh
]]; then
- cd
"${teamcity_build_checkoutDir}"/regression-test/pipeline/tpch/tpch-sf100/
- bash -x prepare.sh
-else
- echo "Build Step file missing:
regression-test/pipeline/tpch/tpch-sf100/prepare.sh" && exit 1
-fi
-EOF
-
-## run.sh content ##
-
-echo "#### Check env"
-if [[ -z "${commit_id_from_trigger}" || -z ${commit_id:-} || -z
${pull_request_id:-} ]]; then
- echo "ERROR: env commit_id_from_trigger or commit_id or pull_request_id
not set" && exit 1
-else
- commit_id_from_checkout=${commit_id}
-fi
-if ${DEBUG:-false}; then commit_id_from_trigger=${commit_id}; fi
-
-echo "Prepare to run tpch sf100 test"
-
-echo "#### 1. check if need run"
-if [[ "${commit_id_from_trigger}" != "${commit_id_from_checkout}" ]]; then
- echo -e "目前是在 clickbench 流水线 compile 完后触发本 tpch 流水线的，
-有可能 pr 在 clickbench 流水线还在跑的时候新提交了commit，
-这时候 tpch 流水线 checkout 出来的 commit 就不是触发时的传过来的 commit了，
-这种情况不需要跑"
- echo -e "ERROR: PR(${pull_request_id}),
- the lastest commit id
- ${commit_id_from_checkout}
- not equail to the commit_id_from_trigger
- ${commit_id_from_trigger}
- commit_id_from_trigger is outdate"
- exit 1
-fi
-
-echo "#### 2. check if depending files exist"
-if ! [[ -f
"${teamcity_build_checkoutDir}"/regression-test/pipeline/tpch/tpch-sf100/conf/fe.conf
&&
- -f
"${teamcity_build_checkoutDir}"/regression-test/pipeline/tpch/tpch-sf100/conf/be.conf
&&
- -f
"${teamcity_build_checkoutDir}"/regression-test/pipeline/tpch/tpch-sf100/deploy.sh
&&
- -f
"${teamcity_build_checkoutDir}"/regression-test/pipeline/tpch/tpch-sf100/run.sh
&&
- -f
"${teamcity_build_checkoutDir}"/regression-test/pipeline/common/oss-utils.sh &&
- -f
"${teamcity_build_checkoutDir}"/regression-test/pipeline/common/doris-utils.sh
&&
- -f
"${teamcity_build_checkoutDir}"/regression-test/pipeline/common/github-utils.sh
&&
- -f
"${teamcity_build_checkoutDir}"/tools/tpch-tools/bin/run-tpch-queries.sh ]];
then
- echo "ERROR: depending files missing" && exit 1
-fi
diff --git a/regression-test/pipeline/tpch/tpch-sf100/run.sh
b/regression-test/pipeline/tpch/tpch-sf100/run.sh
deleted file mode 100644
index d9252f742c1..00000000000
--- a/regression-test/pipeline/tpch/tpch-sf100/run.sh
+++ /dev/null
@@ -1,131 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Build Step: Command Line
-: <<EOF
-#!/bin/bash
-
-teamcity_build_checkoutDir="%teamcity.build.checkoutDir%"
-if [[ -f
"${teamcity_build_checkoutDir:-}"/regression-test/pipeline/tpch/tpch-sf100/run.sh
]]; then
- cd
"${teamcity_build_checkoutDir}"/regression-test/pipeline/tpch/tpch-sf100/
- bash -x run.sh
-else
- echo "Build Step file missing:
regression-test/pipeline/tpch/tpch-sf100/run.sh" && exit 1
-fi
-EOF
-
-## run.sh content ##
-
-# check_tpch_table_rows, stop_doris, set_session_variable
-source ../../common/doris-utils.sh
-# create_an_issue_comment
-source ../../common/github-utils.sh
-# upload_doris_log_to_oss
-source ../../common/oss-utils.sh
-
-echo "#### Check env"
-if [[ -z "${teamcity_build_checkoutDir}" ||
- -z "${pull_request_id}" ||
- -z "${commit_id}" ]]; then
- echo "ERROR: env teamcity_build_checkoutDir or pull_request_id or
commit_id not set"
- exit 1
-fi
-
-echo "#### Run tpch-sf100 test on Doris ####"
-DORIS_HOME="${teamcity_build_checkoutDir}/output"
-exit_flag=0
-
-check_tpch_result() {
- log_file="$1"
- if ! grep '^Total cold run time' "${log_file}" || ! grep '^Total hot run
time' "${log_file}"; then
- echo "ERROR: can not find 'Total hot run time' in '${log_file}'"
- return 1
- else
- cold_run_time=$(grep '^Total cold run time' "${log_file}" | awk
'{print $5}')
- hot_run_time=$(grep '^Total hot run time' "${log_file}" | awk '{print
$5}')
- fi
- # 单位是毫秒
- cold_run_time_threshold=${cold_run_time_threshold:-50000}
- hot_run_time_threshold=${hot_run_time_threshold:-42000}
- if [[ ${cold_run_time} -gt 50000 || ${hot_run_time} -gt 42000 ]]; then
- echo "ERROR:
- cold_run_time ${cold_run_time} is great than the threshold
${cold_run_time_threshold},
- or, hot_run_time ${hot_run_time} is great than the threshold
${hot_run_time_threshold}"
- return 1
- else
- echo "INFO:
- cold_run_time ${cold_run_time} is less than the threshold
${cold_run_time_threshold},
- or, hot_run_time ${hot_run_time} is less than the threshold
${hot_run_time_threshold}"
- fi
-}
-
-(
- set -e
- shopt -s inherit_errexit
-
- echo "#### 1. check if need to load data"
- SF="100" # SCALE FACTOR
- if ${DEBUG:-false}; then SF="1"; fi
- db_name="tpch_sf${SF}"
- sed -i "s|^export DB=.*$|export DB='${db_name}'|g" \
-
"${teamcity_build_checkoutDir}"/tools/tpch-tools/conf/doris-cluster.conf
- if ! check_tpch_table_rows "${db_name}" "${SF}"; then
- echo "ERROR: check_tpch_table_rows failed." && exit 1
- fi
-
- echo "#### 2. run tpch-sf${SF} query"
- set_session_variable runtime_filter_mode global
- sed -i "s|^SCALE_FACTOR=[0-9]\+$|SCALE_FACTOR=${SF}|g"
"${teamcity_build_checkoutDir}"/tools/tpch-tools/bin/run-tpch-queries.sh
- bash
"${teamcity_build_checkoutDir}"/tools/tpch-tools/bin/run-tpch-queries.sh | tee
"${teamcity_build_checkoutDir}"/run-tpch-queries.log
- if ! check_tpch_result
"${teamcity_build_checkoutDir}"/run-tpch-queries.log; then exit 1; fi
- line_end=$(sed -n '/^Total hot run time/='
"${teamcity_build_checkoutDir}"/run-tpch-queries.log)
- line_begin=$((line_end - 23))
- comment_body="Tpch sf${SF} test result on commit ${commit_id:-}, data
reload: ${data_reload:-"false"}
-
-run tpch-sf${SF} query with default conf and session variables
-$(sed -n "${line_begin},${line_end}p"
"${teamcity_build_checkoutDir}"/run-tpch-queries.log)"
-
- echo "#### 3. run tpch-sf${SF} query with runtime_filter_mode=off"
- set_session_variable runtime_filter_mode off
- bash
"${teamcity_build_checkoutDir}"/tools/tpch-tools/bin/run-tpch-queries.sh | tee
"${teamcity_build_checkoutDir}"/run-tpch-queries.log
- if ! grep '^Total hot run time'
"${teamcity_build_checkoutDir}"/run-tpch-queries.log >/dev/null; then exit 1; fi
- line_end=$(sed -n '/^Total hot run time/='
"${teamcity_build_checkoutDir}"/run-tpch-queries.log)
- line_begin=$((line_end - 23))
- comment_body="${comment_body}
-
-run tpch-sf${SF} query with default conf and set session variable
runtime_filter_mode=off
-$(sed -n "${line_begin},${line_end}p"
"${teamcity_build_checkoutDir}"/run-tpch-queries.log)"
-
- echo "#### 4. comment result on tpch"
- comment_body=$(echo "${comment_body}" | sed -e
':a;N;$!ba;s/\t/\\t/g;s/\n/\\n/g') # 将所有的 Tab字符替换为\t 换行符替换为\n
- create_an_issue_comment_tpch "${pull_request_id:-}" "${comment_body}"
-
- stop_doris
-)
-exit_flag="$?"
-
-echo "#### 5. check if need backup doris logs"
-if [[ ${exit_flag} != "0" ]]; then
- print_doris_fe_log
- print_doris_be_log
- if file_name=$(archive_doris_logs
"${pull_request_id}_${commit_id}_doris_logs.tar.gz"); then
- upload_doris_log_to_oss "${file_name}"
- fi
-fi
-
-exit "${exit_flag}"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
