This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 3eed26faf1c Add retry wrapper for Maven steps in workflows, for only
fast (<20m) failures (#17316)
3eed26faf1c is described below
commit 3eed26faf1c02c5b9dbe5ca4eeef9a962016a01e
Author: Xiang Fu <[email protected]>
AuthorDate: Tue Dec 9 14:07:34 2025 -0800
Add retry wrapper for Maven steps in workflows, for only fast (<20m)
failures (#17316)
---
.github/workflows/pinot_compatibility_tests.yml | 11 +-
...ulti_stage_query_engine_compatibility_tests.yml | 11 +-
.github/workflows/pinot_tests.yml | 175 ++++++++++++++++++---
.github/workflows/scripts/retry_on_fast_fail.sh | 42 +++++
4 files changed, 212 insertions(+), 27 deletions(-)
diff --git a/.github/workflows/pinot_compatibility_tests.yml
b/.github/workflows/pinot_compatibility_tests.yml
index ecbc2300e01..a2a2bf9dd16 100644
--- a/.github/workflows/pinot_compatibility_tests.yml
+++ b/.github/workflows/pinot_compatibility_tests.yml
@@ -69,7 +69,16 @@ jobs:
--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
- run: .github/workflows/scripts/.pinot_compatibility_verifier.sh
+ uses: nick-fields/retry@v3
+ with:
+ max_attempts: 3
+ retry_wait_seconds: 20
+ timeout_minutes: 120
+ retry_on_exit_code: 75
+ command: |
+ cat <<'EOF' | .github/workflows/scripts/retry_on_fast_fail.sh
+ .github/workflows/scripts/.pinot_compatibility_verifier.sh
+ EOF
- name: Archive artifacts into zip
if: always()
run: |
diff --git
a/.github/workflows/pinot_multi_stage_query_engine_compatibility_tests.yml
b/.github/workflows/pinot_multi_stage_query_engine_compatibility_tests.yml
index 52b7773033d..fc5c337ca05 100644
--- a/.github/workflows/pinot_multi_stage_query_engine_compatibility_tests.yml
+++ b/.github/workflows/pinot_multi_stage_query_engine_compatibility_tests.yml
@@ -69,7 +69,16 @@ jobs:
--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
- run: .github/workflows/scripts/.pinot_compatibility_verifier.sh
+ uses: nick-fields/retry@v3
+ with:
+ max_attempts: 3
+ retry_wait_seconds: 20
+ timeout_minutes: 120
+ retry_on_exit_code: 75
+ command: |
+ cat <<'EOF' | .github/workflows/scripts/retry_on_fast_fail.sh
+ .github/workflows/scripts/.pinot_compatibility_verifier.sh
+ EOF
- name: Archive artifacts into zip
if: always()
run: |
diff --git a/.github/workflows/pinot_tests.yml
b/.github/workflows/pinot_tests.yml
index 23949e33c74..ec64fd65915 100644
--- a/.github/workflows/pinot_tests.yml
+++ b/.github/workflows/pinot_tests.yml
@@ -57,11 +57,19 @@ jobs:
distribution: 'temurin'
cache: 'maven'
- name: Install pinot-dependency-verifier into repo
- run: |
- mvn clean install \
- -pl pinot-dependency-verifier \
- -am \
- -DskipTests
+ uses: nick-fields/retry@v3
+ with:
+ max_attempts: 3
+ retry_wait_seconds: 20
+ timeout_minutes: 120
+ retry_on_exit_code: 75
+ command: |
+ cat <<'EOF' | .github/workflows/scripts/retry_on_fast_fail.sh
+ mvn clean install \
+ -pl pinot-dependency-verifier \
+ -am \
+ -DskipTests
+ EOF
- name: Linter Test
env:
DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
@@ -75,7 +83,16 @@ jobs:
--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
- run: .github/workflows/scripts/.pinot_linter.sh
+ uses: nick-fields/retry@v3
+ with:
+ max_attempts: 3
+ retry_wait_seconds: 20
+ timeout_minutes: 120
+ retry_on_exit_code: 75
+ command: |
+ cat <<'EOF' | .github/workflows/scripts/retry_on_fast_fail.sh
+ .github/workflows/scripts/.pinot_linter.sh
+ EOF
binary-compat-check:
if: github.repository == 'apache/pinot'
@@ -100,10 +117,18 @@ jobs:
${{ runner.os }}-maven-
# Build the PR branch's module to get a new JAR to compare
- name: Build SPI on PR branch
- run: |
- for mod in pinot-spi pinot-segment-spi; do
- mvn clean package -T1C -pl $mod -am -DskipTests
-Dproject.build.finalName="$mod"
- done
+ uses: nick-fields/retry@v3
+ with:
+ max_attempts: 3
+ retry_wait_seconds: 20
+ timeout_minutes: 120
+ retry_on_exit_code: 75
+ command: |
+ cat <<'EOF' | .github/workflows/scripts/retry_on_fast_fail.sh
+ for mod in pinot-spi pinot-segment-spi; do
+ mvn clean package -T1C -pl $mod -am -DskipTests
-Dproject.build.finalName="$mod"
+ done
+ EOF
- name: Checkout master into a subfolder
uses: actions/checkout@v4
with:
@@ -111,11 +136,19 @@ jobs:
path: master-head
# Build the same module on master to produce the baseline JAR
- name: Build SPI on master
- working-directory: master-head
- run: |
- for mod in pinot-spi pinot-segment-spi; do
- mvn clean package -T1C -pl $mod -am -DskipTests
-Dproject.build.finalName="$mod"
- done
+ uses: nick-fields/retry@v3
+ with:
+ max_attempts: 3
+ retry_wait_seconds: 20
+ timeout_minutes: 120
+ retry_on_exit_code: 75
+ command: |
+ cat <<'EOF' | .github/workflows/scripts/retry_on_fast_fail.sh
+ cd master-head || exit 1
+ for mod in pinot-spi pinot-segment-spi; do
+ mvn clean package -T1C -pl $mod -am -DskipTests
-Dproject.build.finalName="$mod"
+ done
+ EOF
# Download the japicmp standalone JAR which will be used to compare APIs
- name: Download japicmp CLI
run: |
@@ -188,7 +221,16 @@ jobs:
--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED
-Dio.netty.tryReflectionSetAccessible=true
-Dio.grpc.netty.shaded.io.netty.tryReflectionSetAccessible=true
- run: .github/workflows/scripts/pr-tests/.pinot_tests_build.sh
+ uses: nick-fields/retry@v3
+ with:
+ max_attempts: 3
+ retry_wait_seconds: 20
+ timeout_minutes: 120
+ retry_on_exit_code: 75
+ command: |
+ cat <<'EOF' | .github/workflows/scripts/retry_on_fast_fail.sh
+ .github/workflows/scripts/pr-tests/.pinot_tests_build.sh
+ EOF
- name: Unit Test
env:
RUN_INTEGRATION_TESTS: false
@@ -212,7 +254,16 @@ jobs:
--add-opens=java.base/jdk.internal.misc=ALL-UNNAMED
-Dio.netty.tryReflectionSetAccessible=true
-Dio.grpc.netty.shaded.io.netty.tryReflectionSetAccessible=true
- run: .github/workflows/scripts/pr-tests/.pinot_tests_unit.sh
+ uses: nick-fields/retry@v3
+ with:
+ max_attempts: 3
+ retry_wait_seconds: 20
+ timeout_minutes: 120
+ retry_on_exit_code: 75
+ command: |
+ cat <<'EOF' | .github/workflows/scripts/retry_on_fast_fail.sh
+ .github/workflows/scripts/pr-tests/.pinot_tests_unit.sh
+ EOF
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v5
continue-on-error: true
@@ -235,7 +286,17 @@ jobs:
verbose: true
- name: Generate Surefire Report
if: ${{ !cancelled() && github.ref == 'refs/heads/master' }}
- run: mvn surefire-report:report-only -P
github-actions,codecoverage,no-integration-tests || echo "Surefire report
generation failed, but continuing..."
+ continue-on-error: true
+ uses: nick-fields/retry@v3
+ with:
+ max_attempts: 3
+ retry_wait_seconds: 20
+ timeout_minutes: 120
+ retry_on_exit_code: 75
+ command: |
+ cat <<'EOF' | .github/workflows/scripts/retry_on_fast_fail.sh
+ mvn surefire-report:report-only -P
github-actions,codecoverage,no-integration-tests
+ EOF
- name: Upload Surefire Reports
if: ${{ !cancelled() && github.ref == 'refs/heads/master' }}
uses: actions/upload-artifact@v4
@@ -288,7 +349,16 @@ jobs:
--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
- run: .github/workflows/scripts/pr-tests/.pinot_tests_build.sh
+ uses: nick-fields/retry@v3
+ with:
+ max_attempts: 3
+ retry_wait_seconds: 20
+ timeout_minutes: 120
+ retry_on_exit_code: 75
+ command: |
+ cat <<'EOF' | .github/workflows/scripts/retry_on_fast_fail.sh
+ .github/workflows/scripts/pr-tests/.pinot_tests_build.sh
+ EOF
- name: Integration Test
env:
RUN_INTEGRATION_TESTS: true
@@ -304,7 +374,16 @@ jobs:
--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
- run: .github/workflows/scripts/pr-tests/.pinot_tests_integration.sh
+ uses: nick-fields/retry@v3
+ with:
+ max_attempts: 3
+ retry_wait_seconds: 20
+ timeout_minutes: 120
+ retry_on_exit_code: 75
+ command: |
+ cat <<'EOF' | .github/workflows/scripts/retry_on_fast_fail.sh
+ .github/workflows/scripts/pr-tests/.pinot_tests_integration.sh
+ EOF
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v5
continue-on-error: true
@@ -342,7 +421,16 @@ jobs:
--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
- run:
.github/workflows/scripts/pr-tests/.pinot_tests_custom_integration.sh
+ uses: nick-fields/retry@v3
+ with:
+ max_attempts: 3
+ retry_wait_seconds: 20
+ timeout_minutes: 120
+ retry_on_exit_code: 75
+ command: |
+ cat <<'EOF' | .github/workflows/scripts/retry_on_fast_fail.sh
+
.github/workflows/scripts/pr-tests/.pinot_tests_custom_integration.sh
+ EOF
- name: Upload coverage to Codecov
if : ${{ matrix.testset == 1 }}
uses: codecov/codecov-action@v5
@@ -366,7 +454,17 @@ jobs:
verbose: true
- name: Generate Surefire Report
if: ${{ !cancelled() && github.ref == 'refs/heads/master' }}
- run: mvn surefire-report:report-only -P github-actions,codecoverage ||
echo "Surefire report generation failed, but continuing..."
+ continue-on-error: true
+ uses: nick-fields/retry@v3
+ with:
+ max_attempts: 3
+ retry_wait_seconds: 20
+ timeout_minutes: 120
+ retry_on_exit_code: 75
+ command: |
+ cat <<'EOF' | .github/workflows/scripts/retry_on_fast_fail.sh
+ mvn surefire-report:report-only -P github-actions,codecoverage
+ EOF
- name: Upload Surefire Reports
if: ${{ !cancelled() && github.ref == 'refs/heads/master' }}
uses: actions/upload-artifact@v4
@@ -431,7 +529,16 @@ jobs:
--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
- run: .github/workflows/scripts/.pinot_compatibility_verifier.sh
+ uses: nick-fields/retry@v3
+ with:
+ max_attempts: 3
+ retry_wait_seconds: 20
+ timeout_minutes: 120
+ retry_on_exit_code: 75
+ command: |
+ cat <<'EOF' | .github/workflows/scripts/retry_on_fast_fail.sh
+ .github/workflows/scripts/.pinot_compatibility_verifier.sh
+ EOF
multi-stage-compatibility-verifier:
if: github.repository == 'apache/pinot'
@@ -487,7 +594,16 @@ jobs:
--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
- run: .github/workflows/scripts/.pinot_compatibility_verifier.sh
+ uses: nick-fields/retry@v3
+ with:
+ max_attempts: 3
+ retry_wait_seconds: 20
+ timeout_minutes: 120
+ retry_on_exit_code: 75
+ command: |
+ cat <<'EOF' | .github/workflows/scripts/retry_on_fast_fail.sh
+ .github/workflows/scripts/.pinot_compatibility_verifier.sh
+ EOF
quickstarts:
if: github.repository == 'apache/pinot'
@@ -518,4 +634,13 @@ jobs:
restore-keys: |
${{ runner.os }}-maven-
- name: Quickstart on JDK ${{ matrix.java }}
- run: .github/workflows/scripts/.pinot_quickstart.sh
+ uses: nick-fields/retry@v3
+ with:
+ max_attempts: 3
+ retry_wait_seconds: 20
+ timeout_minutes: 120
+ retry_on_exit_code: 75
+ command: |
+ cat <<'EOF' | .github/workflows/scripts/retry_on_fast_fail.sh
+ .github/workflows/scripts/.pinot_quickstart.sh
+ EOF
diff --git a/.github/workflows/scripts/retry_on_fast_fail.sh
b/.github/workflows/scripts/retry_on_fast_fail.sh
new file mode 100755
index 00000000000..848b55d7672
--- /dev/null
+++ b/.github/workflows/scripts/retry_on_fast_fail.sh
@@ -0,0 +1,42 @@
+#!/bin/bash -x
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Run a command piped on stdin and trigger a retry exit code only when the
+# command fails within a configurable threshold.
+
+set -euo pipefail
+
+RETRY_EXIT_CODE=${RETRY_EXIT_CODE:-75}
+RETRY_FAIL_MAX_SECONDS=${RETRY_FAIL_MAX_SECONDS:-1200}
+
+tmp_script=$(mktemp)
+cat > "$tmp_script"
+
+start=$(date +%s)
+set +e
+bash "$tmp_script"
+rc=$?
+duration=$(( $(date +%s) - start ))
+
+if [ "$rc" -ne 0 ] && [ "$duration" -le "$RETRY_FAIL_MAX_SECONDS" ]; then
+ echo "Command failed after ${duration}s (<=${RETRY_FAIL_MAX_SECONDS}s);
signalling retry with exit code ${RETRY_EXIT_CODE}"
+ exit "$RETRY_EXIT_CODE"
+fi
+
+exit "$rc"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
