This is an automated email from the ASF dual-hosted git repository. szetszwo pushed a commit to branch release-3.1.3_review in repository https://gitbox.apache.org/repos/asf/ratis.git
commit 6faeb7522748880f3b2eabbb3ce031dd78141dbf Author: Doroszlai, Attila <[email protected]> AuthorDate: Tue Dec 24 19:22:44 2024 +0100 RATIS-2210. Reduce duplication in CI workflow (#1195) --- .github/workflows/check.yml | 203 +++++++++++++++++++++++++++++ .github/workflows/ci.yml | 141 ++++++++++++++++++++ .github/workflows/post-commit.yml | 261 ++------------------------------------ dev-support/checks/compile.sh | 31 +++++ 4 files changed, 385 insertions(+), 251 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml new file mode 100644 index 000000000..b4498927a --- /dev/null +++ b/.github/workflows/check.yml @@ -0,0 +1,203 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This reusable workflow executes a single check from `dev-support/checks/`. +# Before and after the check, it performs various steps based on workflow inputs. + +name: ci-check + +on: + workflow_call: + inputs: + # REQUIRED + script: + type: string + description: "Test script to run from dev-support/checks, without .sh extension" + required: true + + # OPTIONAL (ordered alphabetically) + java-version: + type: string + description: "Java version to set up (default: 8)" + default: '8' + required: false + + needs-binary-tarball: + type: boolean + description: "Whether to download Ratis binary tarball created by build (default: no)" + default: false + required: false + + needs-maven-repo: + type: boolean + description: "Whether to download Ratis jars created by build (default: no)" + default: false + required: false + + needs-source-tarball: + type: boolean + description: "Whether to download Ratis source tarball created by build (default: no)" + default: false + required: false + + runner: + type: string + description: "GitHub Actions runner to use" + default: 'ubuntu-20.04' + required: false + + script-args: + type: string + description: "Arguments for the test script" + default: '' + required: false + + split: + type: string + description: "Name of split for matrix jobs, only used in display name" + default: '' + required: false + + timeout-minutes: + type: number + description: "Job timeout in minutes (default: 30)" + default: 30 + required: false + +env: + MAVEN_ARGS: --batch-mode --show-version + MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + WITH_COVERAGE: true + +jobs: + check: + name: ${{ (inputs.split && format('{0} ({1})', inputs.script, inputs.split)) || inputs.script }} + runs-on: ${{ inputs.runner }} + timeout-minutes: ${{ inputs.timeout-minutes }} + steps: + - name: Checkout project + if: ${{ !inputs.needs-source-tarball }} + uses: actions/checkout@v4 + + - name: Download source tarball + if: ${{ inputs.needs-source-tarball }} + uses: actions/download-artifact@v4 + with: + name: ratis-src + + - name: Extract source tarball + if: ${{ inputs.needs-source-tarball }} + run: | + tar --strip-components 1 -xzvf ratis*-src.tar.gz + + - name: Create cache for Maven dependencies + if: ${{ inputs.script == 'build' }} + uses: actions/cache@v4 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/ratis + key: maven-repo-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-repo- + + - name: Restore cache for Maven dependencies + if: ${{ inputs.script != 'build' }} + uses: actions/cache/restore@v4 + with: + path: | + ~/.m2/repository/*/*/* + !~/.m2/repository/org/apache/ratis + key: maven-repo-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-repo- + + - name: Download Maven repo + id: download-maven-repo + if: ${{ inputs.needs-maven-repo }} + uses: actions/download-artifact@v4 + with: + name: maven-repo + path: | + ~/.m2/repository/org/apache/ratis + + - name: Download binary tarball + if: ${{ inputs.needs-binary-tarball }} + uses: actions/download-artifact@v4 + with: + name: ratis-bin + + - name: Extract binary tarball + if: ${{ inputs.needs-binary-tarball }} + run: | + mkdir -p ratis-assembly/target + tar xzvf ratis-*-bin.tar.gz -C ratis-assembly/target + + - name: Setup java ${{ inputs.java-version }} + if: ${{ inputs.java-version }} + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: ${{ inputs.java-version }} + + - name: Execute tests + run: | + dev-support/checks/${{ inputs.script }}.sh ${{ inputs.script-args }} + env: + WITH_COVERAGE: ${{ inputs.with-coverage }} + + - name: Summary of failures + if: ${{ failure() }} + run: | + if [[ -s "target/${{ inputs.script }}/summary.txt" ]]; then + cat target/${{ inputs.script }}/summary.txt + fi + + - name: Archive build results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: ${{ (inputs.split && format('{0}-{1}', inputs.script, inputs.split)) || inputs.script }} + path: target/${{ inputs.script }} + continue-on-error: true + + # The following steps are hard-coded to be run only for 'build' check, + # to avoid the need for 3 more inputs. + - name: Store binaries for tests + if: ${{ inputs.script == 'build' && !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: ratis-bin + path: | + ratis-assembly/target/ratis-assembly-*-bin.tar.gz + retention-days: 1 + + - name: Store source tarball for compilation + if: ${{ inputs.script == 'build' && !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: ratis-src + path: | + ratis-assembly/target/ratis-assembly-*-src.tar.gz + retention-days: 1 + + - name: Store Maven repo for tests + if: ${{ inputs.script == 'build' && !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: maven-repo + path: | + ~/.m2/repository/org/apache/ratis + retention-days: 1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..11fb1bd51 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,141 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: CI + +on: + workflow_call: + inputs: + ref: + type: string + description: Ratis git ref (branch, tag or commit hash) + default: '' + required: false + +jobs: + build: + uses: ./.github/workflows/check.yml + with: + script: build + script-args: -Prelease + timeout-minutes: 30 + secrets: inherit + + compile: + needs: + - build + strategy: + matrix: + java: [ 11, 17, 21 ] + fail-fast: false + uses: ./.github/workflows/check.yml + with: + java-version: ${{ matrix.java }} + needs-source-tarball: true + script: compile + script-args: -Djavac.version=${{ matrix.java }} + split: ${{ matrix.java }} + timeout-minutes: 30 + secrets: inherit + + repro: + needs: + - build + uses: ./.github/workflows/check.yml + with: + needs-maven-repo: true + script: repro + script-args: -Prelease + timeout-minutes: 30 + secrets: inherit + + basic: + strategy: + matrix: + check: + - author + - checkstyle + - findbugs + - rat + fail-fast: false + uses: ./.github/workflows/check.yml + with: + script: ${{ matrix.check }} + timeout-minutes: 30 + secrets: inherit + + unit: + strategy: + matrix: + profile: + - grpc + - server + - misc + fail-fast: false + uses: ./.github/workflows/check.yml + with: + script: unit + script-args: -P${{ matrix.profile }}-tests + split: ${{ matrix.profile }} + timeout-minutes: 60 + secrets: inherit + + coverage: + needs: + - build + - unit + runs-on: ubuntu-20.04 + timeout-minutes: 30 + if: (github.repository == 'apache/ratis' || github.repository == 'apache/incubator-ratis') && github.event_name != 'pull_request' + steps: + - name: Checkout project + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Cache for maven dependencies + uses: actions/cache/restore@v4 + with: + path: | + ~/.m2/repository + !~/.m2/repository/org/apache/ratis + key: maven-repo-${{ hashFiles('**/pom.xml') }} + restore-keys: | + maven-repo- + - name: Setup java 17 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + path: target/artifacts + - name: Untar binaries + run: | + mkdir -p ratis-assembly/target + tar xzvf target/artifacts/ratis-bin/ratis-assembly-*.tar.gz -C ratis-assembly/target + - name: Calculate combined coverage + run: ./dev-support/checks/coverage.sh + - name: Upload coverage to Sonar + run: ./dev-support/checks/sonar.sh + env: + SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Archive build results + uses: actions/upload-artifact@v4 + if: always() + with: + name: ${{ github.job }} + path: target/${{ github.job }} diff --git a/.github/workflows/post-commit.yml b/.github/workflows/post-commit.yml index 5fee9462a..9b0df4ce4 100644 --- a/.github/workflows/post-commit.yml +++ b/.github/workflows/post-commit.yml @@ -12,7 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + name: build-branch + on: push: branches-ignore: @@ -20,255 +22,12 @@ on: tags: - '**' pull_request: -env: - WITH_COVERAGE: true + +concurrency: + group: ci-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: - build: - runs-on: ubuntu-20.04 - timeout-minutes: 30 - steps: - - name: Checkout project - uses: actions/checkout@v4 - - name: Cache for maven dependencies - uses: actions/cache@v4 - with: - path: | - ~/.m2/repository - !~/.m2/repository/org/apache/ratis - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: | - maven-repo- - - name: Setup java - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: 8 - - name: Run a full build - run: ./dev-support/checks/build.sh -Prelease - - name: Store binaries for tests - uses: actions/upload-artifact@v4 - with: - name: ratis-bin - path: | - ratis-assembly/target/ratis-assembly-*.tar.gz - !ratis-assembly/target/ratis-assembly-*-src.tar.gz - retention-days: 1 - - name: Store source tarball for compilation - uses: actions/upload-artifact@v4 - with: - name: ratis-src - path: ratis-assembly/target/ratis-assembly-*-src.tar.gz - retention-days: 1 - compile: - needs: - - build - runs-on: ubuntu-20.04 - timeout-minutes: 30 - strategy: - matrix: - java: [ 11, 17, 21 ] - fail-fast: false - steps: - - name: Download source tarball - uses: actions/download-artifact@v4 - with: - name: ratis-src - - name: Untar sources - run: | - tar --strip-components 1 -xzvf ratis-assembly-*-src.tar.gz - - name: Cache for maven dependencies - uses: actions/cache/restore@v4 - with: - path: | - ~/.m2/repository - !~/.m2/repository/org/apache/ratis - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: | - maven-repo- - - name: Setup java - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: ${{ matrix.java }} - - name: Run a full build - run: ./dev-support/checks/build.sh -Djavac.version=${{ matrix.java }} - - name: Test reproducibility - run: ./dev-support/checks/repro.sh -Djavac.version=${{ matrix.java }} - rat: - name: rat - runs-on: ubuntu-20.04 - timeout-minutes: 15 - steps: - - name: Checkout project - uses: actions/checkout@v4 - - name: Cache for maven dependencies - uses: actions/cache/restore@v4 - with: - path: | - ~/.m2/repository - !~/.m2/repository/org/apache/ratis - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: | - maven-repo- - - name: Run tests - run: ./dev-support/checks/rat.sh - - name: Upload results - uses: actions/upload-artifact@v4 - if: always() - with: - name: rat - path: target/rat - author: - name: author - runs-on: ubuntu-20.04 - timeout-minutes: 15 - steps: - - name: Checkout project - uses: actions/checkout@v4 - - name: Run tests - run: ./dev-support/checks/author.sh - - name: Upload results - uses: actions/upload-artifact@v4 - if: always() - with: - name: author - path: target/author - unit: - name: unit - runs-on: ubuntu-20.04 - timeout-minutes: 60 - strategy: - matrix: - profile: - - grpc - - server - - misc - fail-fast: false - steps: - - name: Checkout project - uses: actions/checkout@v4 - - name: Cache for maven dependencies - uses: actions/cache/restore@v4 - with: - path: | - ~/.m2/repository - !~/.m2/repository/org/apache/ratis - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: | - maven-repo- - - name: Setup java - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: 8 - - name: Run tests - run: ./dev-support/checks/unit.sh -P${{ matrix.profile }}-tests - - name: Summary of failures - run: cat target/${{ github.job }}/summary.txt - if: ${{ !cancelled() }} - - name: Upload results - uses: actions/upload-artifact@v4 - if: ${{ !cancelled() }} - with: - name: unit-${{ matrix.profile }} - path: target/unit - checkstyle: - name: checkstyle - runs-on: ubuntu-20.04 - timeout-minutes: 15 - steps: - - name: Checkout project - uses: actions/checkout@v4 - - name: Cache for maven dependencies - uses: actions/cache/restore@v4 - with: - path: | - ~/.m2/repository - !~/.m2/repository/org/apache/ratis - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: | - maven-repo- - - name: Run tests - run: ./dev-support/checks/checkstyle.sh - - name: Upload results - uses: actions/upload-artifact@v4 - if: always() - with: - name: checkstyle - path: target/checkstyle - findbugs: - name: findbugs - runs-on: ubuntu-20.04 - timeout-minutes: 30 - steps: - - name: Setup java - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: 8 - - name: Checkout project - uses: actions/checkout@v4 - - name: Cache for maven dependencies - uses: actions/cache/restore@v4 - with: - path: | - ~/.m2/repository - !~/.m2/repository/org/apache/ratis - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: | - maven-repo- - - name: Run tests - run: ./dev-support/checks/findbugs.sh - - name: Upload results - uses: actions/upload-artifact@v4 - if: always() - with: - name: findbugs - path: target/findbugs - coverage: - needs: - - build - - unit - runs-on: ubuntu-20.04 - timeout-minutes: 30 - if: (github.repository == 'apache/ratis' || github.repository == 'apache/incubator-ratis') && github.event_name != 'pull_request' - steps: - - name: Checkout project - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Cache for maven dependencies - uses: actions/cache/restore@v4 - with: - path: | - ~/.m2/repository - !~/.m2/repository/org/apache/ratis - key: maven-repo-${{ hashFiles('**/pom.xml') }} - restore-keys: | - maven-repo- - - name: Setup java 17 - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: 17 - - name: Download artifacts - uses: actions/download-artifact@v4 - with: - path: target/artifacts - - name: Untar binaries - run: | - mkdir -p ratis-assembly/target - tar xzvf target/artifacts/ratis-bin/ratis-assembly-*.tar.gz -C ratis-assembly/target - - name: Calculate combined coverage - run: ./dev-support/checks/coverage.sh - - name: Upload coverage to Sonar - run: ./dev-support/checks/sonar.sh - env: - SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Archive build results - uses: actions/upload-artifact@v4 - if: always() - with: - name: ${{ github.job }} - path: target/${{ github.job }} + CI: + uses: ./.github/workflows/ci.yml + secrets: inherit diff --git a/dev-support/checks/compile.sh b/dev-support/checks/compile.sh new file mode 100755 index 000000000..fba4394de --- /dev/null +++ b/dev-support/checks/compile.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "$DIR/../.." || exit 1 + +source "${DIR}/../find_maven.sh" + +: ${WITH_COVERAGE:="false"} + +MAVEN_OPTIONS='-V -B -Dmaven.javadoc.skip=true -DskipTests' + +if [[ "${WITH_COVERAGE}" != "true" ]]; then + MAVEN_OPTIONS="${MAVEN_OPTIONS} -Djacoco.skip" +fi + +export MAVEN_OPTS="-Xmx4096m" +${MVN} ${MAVEN_OPTIONS} clean verify "$@" +exit $?
