This is an automated email from the ASF dual-hosted git repository.
xxyu pushed a commit to branch kylin5-tmp
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 07dee81769169a45e70a9364f85076b0ff7af256
Author: XiaoxiangYu <x...@apache.org>
AuthorDate: Tue Sep 12 18:34:59 2023 +0800
Update release pipeline
---
.../release-machine/Dockerfile_2 | 2 +-
.../release-machine/README.md | 15 +-
.../release-machine/conf/settings.xml | 2 +
.../release-machine/create-release-machine.sh | 17 +-
.../release-machine/release-publish.sh | 309 +++++++++++----------
pom.xml | 6 +
6 files changed, 192 insertions(+), 159 deletions(-)
diff --git a/build/release/release-pipeline-docker/release-machine/Dockerfile_2
b/build/release/release-pipeline-docker/release-machine/Dockerfile_2
index abdb44f632..da75a7c459 100644
--- a/build/release/release-pipeline-docker/release-machine/Dockerfile_2
+++ b/build/release/release-pipeline-docker/release-machine/Dockerfile_2
@@ -21,7 +21,7 @@
# 1. https://www.apache.org/legal/release-policy.html
# 2. https://infra.apache.org/release-publishing.html
-FROM release-machine:5.0-base
+FROM apachekylin/release-machine:5.0-base
COPY conf/settings.xml /root/.m2/settings.xml
COPY release-publish.sh /root/release-publish.sh
RUN chmod u+x /root/release-publish.sh
\ No newline at end of file
diff --git a/build/release/release-pipeline-docker/release-machine/README.md
b/build/release/release-pipeline-docker/release-machine/README.md
index 0fc50ac49f..290f9fbd9f 100644
--- a/build/release/release-pipeline-docker/release-machine/README.md
+++ b/build/release/release-pipeline-docker/release-machine/README.md
@@ -27,9 +27,20 @@ It also provided a way to publish documentation for Kylin 5.
-[ ] Update `CURRENT_KYLIN_VERSION` in `KylinVersion.java` .
```bash
-docker start release-machine-1 \
+docker start release-machine \
+ --volume ~/.gnupg/:/root/.gnupg \
+ --env GIT_USERNAME='???' \
+ --env GPG_KEY=??? \
+ --env GPG_PASSPHRASE=??? \
+ --env ASF_USERNAME=??? \
+ --env ASF_PASSWORD='???' \
+ --env GIT_BRANCH=??? \
+ --env RELEASE_VERSION=??? \
+ --env NEXT_RELEASE_VERSION=??? \
+ --env RC_NUMBER=??? \
-p 4040:4040 \
- bash
+ -t release-machine:latest \
+ bash
```
| Name | Comment
|
diff --git
a/build/release/release-pipeline-docker/release-machine/conf/settings.xml
b/build/release/release-pipeline-docker/release-machine/conf/settings.xml
index 9241afee93..f9b38fa969 100644
--- a/build/release/release-pipeline-docker/release-machine/conf/settings.xml
+++ b/build/release/release-pipeline-docker/release-machine/conf/settings.xml
@@ -69,6 +69,8 @@
</servers>
<profiles>
+ <!-- This is default proxy for China mainland,
+ remove this profile if you are not in China-->
<profile>
<repositories>
<repository>
diff --git
a/build/release/release-pipeline-docker/release-machine/create-release-machine.sh
b/build/release/release-pipeline-docker/release-machine/create-release-machine.sh
index 497944cc6b..8d588bce28 100644
---
a/build/release/release-pipeline-docker/release-machine/create-release-machine.sh
+++
b/build/release/release-pipeline-docker/release-machine/create-release-machine.sh
@@ -21,13 +21,16 @@
#
docker image rm release-machine:latest
-build_status='1'
-while [ "$build_status" != "0" ]
-do
- echo "Build release-machine from $(date)"
- docker build -f Dockerfile_1 -t release-machine:5.0-base .
- build_status="$?"
-done
+
+# if you do not want to build from "apachekylin/release-machine:5.0-base",
+# please remove comments of following code
+#build_status='1'
+#while [ "$build_status" != "0" ]
+#do
+# echo "Build release-machine from $(date)"
+# docker build -f Dockerfile_1 -t release-machine:5.0-base .
+# build_status="$?"
+#done
docker build -f Dockerfile_2 -t release-machine:latest .
diff --git
a/build/release/release-pipeline-docker/release-machine/release-publish.sh
b/build/release/release-pipeline-docker/release-machine/release-publish.sh
index 615e0891ef..5c904997dd 100644
--- a/build/release/release-pipeline-docker/release-machine/release-publish.sh
+++ b/build/release/release-pipeline-docker/release-machine/release-publish.sh
@@ -23,6 +23,7 @@
#
https://stackoverflow.com/questions/57591432/gpg-signing-failed-inappropriate-ioctl-for-device-on-macos-with-maven
GPG_TTY=$(tty)
export GPG_TTY
+CUR_DATE=$(date "+%Y-%m-%d")
export LC_ALL=C.UTF-8
export LANG=C.UTF-8
set -e
@@ -80,18 +81,19 @@ fi
function read_config {
if [ "$MODE" = "batch" ] ;then
- return 0
- fi
- local PROMPT="$1"
- local DEFAULT="$2"
- local REPLY=
-
- read -p "$PROMPT [default is $DEFAULT]: " REPLY
- local RETVAL="${REPLY:-$DEFAULT}"
- if [ -z "$RETVAL" ]; then
- error "$PROMPT is must be provided."
+ echo "$2"
+ elif [ "$MODE" = "" ]; then
+ local PROMPT="$1"
+ local DEFAULT="$2"
+ local REPLY=
+
+ read -p "$PROMPT [default is $DEFAULT]: " REPLY
+ local RETVAL="${REPLY:-$DEFAULT}"
+ if [ -z "$RETVAL" ]; then
+ error "$PROMPT is must be provided."
+ fi
+ echo "$RETVAL"
fi
- echo "$RETVAL"
}
function run_command {
@@ -100,10 +102,14 @@ function run_command {
echo "========================"
echo "| $BANNER"
- echo "Command: $@"
-
- "$@" 2>&1
+ echo "| $(date) Command: $@"
+ if [ "$LOG_OPTION" = "log" ] ;then
+ echo "Redirect to ${LOG} ..."
+ "$@">> ${LOG} 2>&1
+ else
+ "$@" 2>&1
+ fi
local EC=$?
if [ $EC != 0 ]; then
echo "Command FAILED : $@, please check!!!"
@@ -122,6 +128,8 @@ function switch_node_for_website {
node -v # 16.14.2
}
+DRY_RUN=false
+
ASF_USERNAME=$(read_config "Your apache id?" "$ASF_USERNAME")
GIT_USERNAME=$(read_config "Your full name(used as author of git commit)?"
"$GIT_USERNAME")
ASF_PASSWORD=$(read_config "Your apache password?" "$ASF_PASSWORD")
@@ -134,13 +142,15 @@ RELEASE_VERSION=$(read_config "Which version are you
going to release?" "$RELEAS
NEXT_RELEASE_VERSION=$(read_config "Which version is the next development
version?" "$NEXT_RELEASE_VERSION")
RC_NUMBER="rc"$(read_config "Number for release candidate?" "$RC_NUMBER")
-export working_dir=/root/release-folder
-source_code_folder=$working_dir/source/kylin
+export working_dir=/root/kylin-folder
svn_folder=$working_dir/svn
+source_code_folder=$working_dir/source/kylin
+
packaging_folder=$source_code_folder/target/checkout
svn_stage_folder=$svn_folder/dev
rc_name=apache-kylin-"${RELEASE_VERSION}"-${RC_NUMBER}
release_candidate_folder=$svn_stage_folder/$rc_name
+final_release_folder=$svn_folder/release
branch_doc_1=document
branch_doc_2=doc5.0
@@ -150,7 +160,8 @@ document_folder_elder=$document_folder_src/$branch_doc_1
document_folder_newer=$document_folder_src/$branch_doc_2
document_folder_svn=$svn_folder/site
-LOG=$working_dir/build.log
+LOG=$working_dir/$rc_name-$CUR_DATE.log
+rm -rf "$LOG"
ASF_KYLIN_REPO="gitbox.apache.org/repos/asf/kylin.git"
# GITHUB_REPO_URL=${GIT_REPO_URL:-https://github.com/apache/kylin.git}
@@ -181,106 +192,106 @@ for env in ASF_USERNAME GPG_PASSPHRASE RC_NUMBER
GPG_KEY; do
done
function reset_release {
- info "Reset release folder"
- cd ${source_code_folder}
- git reset --hard HEAD~5
- git pull -r origin "$GIT_BRANCH"
- mvn clean
- mvn release:clean
- return 0
+ info "Reset release folder"
+ cd ${source_code_folder}
+ git reset --hard HEAD~5
+ git pull -r origin "$GIT_BRANCH"
+ mvn clean
+ mvn release:clean
+ # Update current script
+ mv /root/release-publish.sh .release-publish.sh.bak
+ cp
$source_code_folder/build/release/release-pipeline-docker/release-machine/release-publish.sh
/root/
+ rm release-publish.sh
}
function prepare_release {
- info "Configuration and Clone Code"
- git config --global user.name "${GIT_USERNAME}"
- git config --global user.email ${GIT_EMAIL}
- git config --global user.password ${ASF_PASSWORD}
- mkdir -p $working_dir
- cd $working_dir
-
- if [ ! -d "${source_code_folder}" ]
- then
- mkdir -p ${source_code_folder}
- info "Clone source code to ${source_code_folder} ."
- run_command "Clone Gitbox" git clone
"https://$ASF_USERNAME:$ASF_PASSWORD@$ASF_KYLIN_REPO"; -b "$GIT_BRANCH"
${source_code_folder}
- fi
+ info "Configuration and Clone Code"
+ git config --global user.name "${GIT_USERNAME}"
+ git config --global user.email ${GIT_EMAIL}
+ git config --global user.password ${ASF_PASSWORD}
+ mkdir -p $working_dir
+ cd $working_dir
+
+ if [ ! -d "${source_code_folder}" ]
+ then
+ mkdir -p ${source_code_folder}
+ info "Clone source code to ${source_code_folder} ."
+ run_command "Clone Gitbox" git clone
"https://$ASF_USERNAME:$ASF_PASSWORD@$ASF_KYLIN_REPO"; --single-branch --branch
"$GIT_BRANCH" ${source_code_folder}
+ fi
- if [ ! -d "${release_candidate_folder}" ]
- then
- mkdir -p ${release_candidate_folder}
- info "Clone svn working dir to $svn_folder ."
- run_command "Clone ASF SVN" svn co $RELEASE_STAGING_LOCATION
$svn_stage_folder
- fi
- switch_node_for_packaging
- return 0
+ if [ ! -d "${release_candidate_folder}" ]
+ then
+ mkdir -p "${release_candidate_folder}"
+ info "Clone svn working dir to $svn_folder ."
+ run_command "Clone ASF SVN" svn co $RELEASE_STAGING_LOCATION
$svn_stage_folder
+ fi
+ switch_node_for_packaging
}
function publish_snapshot_source {
- info "Publish source code, maven artifact, git tag for release candidate"
- cd ${source_code_folder}
-
- tag_exist=`git tag --list | grep kylin-"${RELEASE_VERSION}" | wc -l`
- if [[ $tag_exist != 0 ]]; then
- echo "Delete local and remote tag"
- git tag --delete kylin-"${RELEASE_VERSION}"
- git push --delete origin kylin-"${RELEASE_VERSION}"
- fi
+ info "Publish source code, maven artifact, git tag for release candidate"
+ cd ${source_code_folder}
+
+ tag_exist=`git tag --list | grep kylin-"${RELEASE_VERSION}" | wc -l`
+ if [[ $tag_exist != 0 ]]; then
+ echo "Delete local and remote tag"
+ git tag --delete kylin-"${RELEASE_VERSION}"
+ git push --delete origin kylin-"${RELEASE_VERSION}"
+ fi
- ## Prepare tag & source tarball & upload maven artifact
- # Use release-plugin to check license & build source package & build and
upload maven artifact
- #
https://maven.apache.org/maven-release/maven-release-plugin/examples/prepare-release.html
- # https://infra.apache.org/publishing-maven-artifacts.html
- # Use `mvn release:clean` if you want to prepare again
- maven_options="-DskipTests \
- -DreleaseVersion=${RELEASE_VERSION} \
- -DdevelopmentVersion=${NEXT_RELEASE_VERSION}-SNAPSHOT \
- -Papache-release,nexus \
- -DdryRun=${DRY_RUN} \
- -Dmaven.javadoc.skip=true \
- -Dgpg.passphrase=${GPG_PASSPHRASE} \
- -DgpgArguments=--no-tty --batch --pinentry-mode loopback \
- -Dkeyname=$GPG_KEY"
- run_command "Maven Release Prepare" mvn "${maven_options}" release:prepare
- run_command "Maven Release Perform" mvn "${maven_options}" release:perform
-
- # Create a directory for this release candidate
- mkdir -p ${release_candidate_folder}
- rm -rf target/apache-kylin-*ource-release.zip.asc.sha256
-
- # Move source code and signture of source code to release candidate
directory
- cp target/apache-kylin-*source-release.zip* "${release_candidate_folder}"
- return 0
+ ## Prepare tag & source tarball & upload maven artifact
+ # Use release-plugin to check license & build source package & build and
upload maven artifact
+ #
https://maven.apache.org/maven-release/maven-release-plugin/examples/prepare-release.html
+ # https://infra.apache.org/publishing-maven-artifacts.html
+ # Use `mvn release:clean` if you want to prepare again
+ maven_options="-DskipTests \
+ -DreleaseVersion=${RELEASE_VERSION} \
+ -DdevelopmentVersion=${NEXT_RELEASE_VERSION}-SNAPSHOT \
+ -Papache-release,nexus \
+ -DdryRun=${DRY_RUN} \
+ -Dmaven.javadoc.skip=true \
+ -Dgpg.passphrase=${GPG_PASSPHRASE} \
+ -DgpgArguments=--no-tty --batch --pinentry-mode loopback \
+ -Dkeyname=$GPG_KEY"
+ run_command "Maven Release Prepare" mvn "${maven_options}" release:prepare
+ run_command "Maven Release Perform" mvn "${maven_options}" release:perform
+
+ # Create a directory for this release candidate
+ mkdir -p ${release_candidate_folder}
+ rm -rf target/apache-kylin-*ource-release.zip.asc.sha256
+
+ # Move source code and signature of source code to release candidate
directory
+ cp target/apache-kylin-*source-release.zip* "${release_candidate_folder}"
}
function publish_snapshot_package {
- info "Building kylin binary for $RELEASE_VERSION"
- cd ${packaging_folder}
+ info "Building kylin binary for $RELEASE_VERSION"
+ cd ${packaging_folder}
- export release_version=$RELEASE_VERSION
- run_command "Build binary" bash build/release/release.sh -official -noSpark
+ export release_version=$RELEASE_VERSION
+ run_command "Build binary" bash build/release/release.sh -official -noSpark
- cp dist/apache-kylin-*.tar.gz "${release_candidate_folder}"
+ cp dist/apache-kylin-*.tar.gz "${release_candidate_folder}"
- ## Sign binary
- echo "publish-release-candidate binary"
- cd "${release_candidate_folder}"
- run_command "Sign binary" echo $GPG_PASSPHRASE | $GPG_COMMAND
--passphrase-fd 0 --armour --output
apache-kylin-"${RELEASE_VERSION}"-bin.tar.gz.asc --detach-sig
apache-kylin-${RELEASE_VERSION}-bin.tar.gz
- shasum -a 512 apache-kylin-"${RELEASE_VERSION}"-bin.tar.gz >
apache-kylin-${RELEASE_VERSION}-bin.tar.gz.sha512
+ ## Sign binary
+ echo "publish-release-candidate binary"
+ cd "${release_candidate_folder}"
+ run_command "Sign binary" echo $GPG_PASSPHRASE | $GPG_COMMAND
--passphrase-fd 0 --armour --output
apache-kylin-"${RELEASE_VERSION}"-bin.tar.gz.asc --detach-sig
apache-kylin-${RELEASE_VERSION}-bin.tar.gz
+ shasum -a 512 apache-kylin-"${RELEASE_VERSION}"-bin.tar.gz >
apache-kylin-${RELEASE_VERSION}-bin.tar.gz.sha512
- ## Upload to svn repository
- ask_confirm "You are going to upload rc, are you sure you have the
permissions?"
- cd ${svn_stage_folder}
- svn add ${rc_name}
- run_command "Publish release candidate dir" svn commit --password
${ASF_PASSWORD} -m 'Check in release artifacts for '${rc_name}
- echo "Please check $RELEASE_STAGING_LOCATION"
- return 0
+ ## Upload to svn repository
+ ask_confirm "You are going to upload release candidate, are you sure you
have the right permission?"
+ cd ${svn_stage_folder}
+ svn add ${rc_name}
+ run_command "Publish release candidate dir" svn commit --password
${ASF_PASSWORD} -m 'Check in release artifacts for '${rc_name}
+ echo "Please check $RELEASE_STAGING_LOCATION"
+ return 0
}
function publish_release {
- info "Publish release candidate after vote succeed."
- # TODO
- return 0
+ info "Publish release candidate after vote succeed."
+ svn co $RELEASE_LOCATION $final_release_folder
}
@@ -289,67 +300,67 @@ function publish_release {
# Following is for website publish
function preview_site() {
- info "Prepare website"
- if [ ! -d "${document_folder_src}" ]; then
- mkdir -p $document_folder_src
- run_command "Install nodejs for docusaurus" switch_node_for_website
- fi
- cd $document_folder_src
- if [ ! -d "${document_folder_elder}" ]; then
- run_command "Clone website for kylin4" git clone --branch
$branch_doc_1 "https://$ASF_USERNAME:$ASF_PASSWORD@$ASF_KYLIN_REPO";
$branch_doc_1
- else
- cd ${document_folder_elder}
- git reset --hard HEAD~4
- git pull -r origin $branch_doc_1
- fi
+ info "Prepare website"
+ if [ ! -d "${document_folder_src}" ]; then
+ mkdir -p $document_folder_src
+ run_command "Install nodejs for docusaurus" switch_node_for_website
+ fi
+ cd $document_folder_src
+ if [ ! -d "${document_folder_elder}" ]; then
+ run_command "Clone website for kylin4" git clone --single-branch
--branch $branch_doc_1 "https://$ASF_USERNAME:$ASF_PASSWORD@$ASF_KYLIN_REPO";
$branch_doc_1
+ else
+ cd ${document_folder_elder}
+ git reset --hard HEAD~4
+ git pull -r origin $branch_doc_1
+ fi
- if [ ! -d "${document_folder_newer}" ]; then
- run_command "Clone website for kylin5" git clone --branch
$branch_doc_2 "https://$ASF_USERNAME:$ASF_PASSWORD@$ASF_KYLIN_REPO";
$branch_doc_2
- else
- cd ${document_folder_newer}
- git reset --hard HEAD~4
- git pull -r origin $branch_doc_2
- fi
+ if [ ! -d "${document_folder_newer}" ]; then
+ run_command "Clone website for kylin5" git clone --single-branch
--branch $branch_doc_2 "https://$ASF_USERNAME:$ASF_PASSWORD@$ASF_KYLIN_REPO";
$branch_doc_2
+ else
+ cd ${document_folder_newer}
+ git reset --hard HEAD~4
+ git pull -r origin $branch_doc_2
+ fi
- if [ ! -d "${document_folder_svn}" ]; then
- mkdir ${document_folder_svn}
- run_command "Checkout website files from svn" svn co $WEBSITE_SVN
${document_folder_svn}
- fi
+ if [ ! -d "${document_folder_svn}" ]; then
+ mkdir ${document_folder_svn}
+ run_command "Checkout website files from svn" svn co $WEBSITE_SVN
${document_folder_svn}
+ fi
- info "Build website"
+ info "Build website"
- # Build inner website
- cd ${document_folder_newer}/website
- # nvm use 16.14
- run_command "Install node modules" npm install
- run_command "Build inner website" npm run build
- document_folder_newer_build=${document_folder_newer}/website/build
+ # Build inner website
+ cd ${document_folder_newer}/website
+ # nvm use 16.14
+ run_command "Install node modules" npm install
+ run_command "Build inner website" npm run build
+ document_folder_newer_build=${document_folder_newer}/website/build
- # Build outer website
- cd ${document_folder_elder}/website
- run_command "Build outer website" jekyll build >>$LOG 2>&1
- document_folder_elder_build=${document_folder_elder}/_site
+ # Build outer website
+ cd ${document_folder_elder}/website
+ run_command "Build outer website" jekyll build >>$LOG 2>&1
+ document_folder_elder_build=${document_folder_elder}/_site
- # Merge two websites
- rm -rf ${document_folder_elder_build}/5.0
- mv ${document_folder_newer_build} ${document_folder_elder_build}/5.0
- info "Build website should be done, and stored in
${document_folder_elder_build} ."
+ # Merge two websites
+ rm -rf ${document_folder_elder_build}/5.0
+ mv ${document_folder_newer_build} ${document_folder_elder_build}/5.0
+ info "Build website should be done, and stored in
${document_folder_elder_build} ."
- run_command "Preview merged website" jekyll s -P 7070
- info "Website could be previewed at localhost:7070"
+ run_command "Preview merged website" jekyll s -P 7070
+ info "Website could be previewed at localhost:7070"
}
function publish_site() {
- info "Publish website"
- svn update ${document_folder_svn}
- svn add --force ${document_folder_svn}/* --auto-props --parents --depth
infinity -q
- svn status ${document_folder_svn}
- if [ `svn status ${document_folder_svn} | wc -l ` != 1 ];
- then MSG=`git log --format=oneline | head -1`
- svn commit --password ${ASF_PASSWORD} ${document_folder_svn} -m
"${MSG:41}"
- else
- echo "No need to refresh website.";
- fi
+ info "Publish website"
+# svn update ${document_folder_svn}
+# svn add --force ${document_folder_svn}/* --auto-props --parents --depth
infinity -q
+# svn status ${document_folder_svn}
+# if [ `svn status ${document_folder_svn} | wc -l ` != 1 ];
+# then MSG=`git log --format=oneline | head -1`
+# svn commit --password ${ASF_PASSWORD} ${document_folder_svn} -m
"${MSG:41}"
+# else
+# echo "No need to refresh website.";
+# fi
}
diff --git a/pom.xml b/pom.xml
index 8881e3c625..6406221725 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2961,6 +2961,11 @@
<build>
<pluginManagement>
<plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-gpg-plugin</artifactId>
+ <version>3.1.0</version>
+ </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
@@ -3593,6 +3598,7 @@
<artifactId>maven-gpg-plugin</artifactId>
<configuration>
<skip>false</skip>
+ <interactive>true</interactive>
</configuration>
</plugin>
<!-- Override the parent assembly execution to customize
the assembly
