This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new 396290adabd [improve][ci] Cleanup tune-runner-vm and clean-disk
actions (#25444)
396290adabd is described below
commit 396290adabdf85f905add66b03f406b1c14cd20b
Author: Lari Hotari <[email protected]>
AuthorDate: Tue Mar 31 23:20:40 2026 +0300
[improve][ci] Cleanup tune-runner-vm and clean-disk actions (#25444)
---
.github/actions/clean-disk/action.yml | 116 ++++++++++++++++++++++--------
.github/actions/tune-runner-vm/action.yml | 50 +++++--------
2 files changed, 102 insertions(+), 64 deletions(-)
diff --git a/.github/actions/clean-disk/action.yml
b/.github/actions/clean-disk/action.yml
index e67ce59a08d..fe5fa9e7cc9 100644
--- a/.github/actions/clean-disk/action.yml
+++ b/.github/actions/clean-disk/action.yml
@@ -23,44 +23,98 @@ inputs:
mode:
description: "Use 'full' to clean as much as possible"
required: false
+ clean-threshold-gb:
+ description: "Skip cleaning if available disk space exceeds this threshold
(in GB)"
+ required: false
+ default: '20'
+ full-clean-threshold-gb:
+ description: "In 'full' mode, skip additional cleaning steps if available
disk space exceeds this threshold (in GB)"
+ required: false
+ default: '40'
runs:
using: composite
steps:
- - run: |
+ - shell: bash
+ env:
+ CLEAN_MODE: ${{ inputs.mode }}
+ CLEAN_THRESHOLD_GB: ${{ inputs.clean-threshold-gb }}
+ FULL_CLEAN_THRESHOLD_GB: ${{ inputs.full-clean-threshold-gb }}
+ run: |
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
- directories=(/usr/local/lib/android /opt/ghc)
- if [[ "${{ inputs.mode }}" == "full" ]]; then
- # remove these directories only when mode is 'full'
- directories+=(/usr/share/dotnet /opt/hostedtoolcache/CodeQL)
+
+ available_gb() {
+ df --output=avail -BG / | tail -1 | tr -d ' G'
+ }
+
+ show_disk() {
+ if mountpoint -q /mnt; then df -BM / /mnt; else df -BM /; fi
+ }
+
+ remove_dir() {
+ local directory=$1
+ if [ -d "$directory" ]; then
+ echo "::group::Removing $directory"
+ local emptydir=/tmp/empty$$/
+ mkdir -p "$emptydir"
+ # fast way to delete a lot of files on linux
+ time sudo eatmydata rsync -a --delete "$emptydir" "${directory}/"
+ time sudo eatmydata rm -rf "${directory}"
+ rmdir "$emptydir" 2>/dev/null || true
+ time show_disk
+ echo "::endgroup::"
+ fi
+ }
+
+ # Remove directories one at a time, stopping when threshold is met
+ clean_dirs() {
+ local threshold=$1
+ shift
+ for directory in "$@"; do
+ if (( $(available_gb) >= threshold )); then
+ return
+ fi
+ remove_dir "$directory"
+ done
+ }
+
+ if [[ "$CLEAN_MODE" == "full" ]]; then
+ threshold=$FULL_CLEAN_THRESHOLD_GB
+ else
+ threshold=$CLEAN_THRESHOLD_GB
fi
- emptydir=/tmp/empty$$/
- mkdir $emptydir
- echo "::group::Available diskspace"
- time df -BM / /mnt
+
+ echo "::group::Available diskspace (threshold: ${threshold}GB)"
+ time show_disk
echo "::endgroup::"
- for directory in "${directories[@]}"; do
- echo "::group::Removing $directory"
- # fast way to delete a lot of files on linux
- time sudo eatmydata rsync -a --delete $emptydir ${directory}/
- time sudo eatmydata rm -rf ${directory}
- time df -BM / /mnt
- echo "::endgroup::"
- done
- if [[ "${{ inputs.mode }}" == "full" ]]; then
- echo "::group::Moving /var/lib/docker to /mnt/docker"
- sudo systemctl stop docker
- echo '{"data-root": "/mnt/docker"}' | sudo tee
/etc/docker/daemon.json
- sudo mv /var/lib/docker /mnt/docker
- sudo systemctl start docker
- time df -BM / /mnt
+
+ if (( $(available_gb) >= threshold )); then
+ echo "Available disk space meets threshold, skipping cleanup."
+ exit 0
+ fi
+
+ clean_dirs "$threshold" /usr/local/lib/android /opt/ghc
+
+ if [[ "$CLEAN_MODE" == "full" ]]; then
+ clean_dirs "$threshold" /usr/share/dotnet
/opt/hostedtoolcache/CodeQL
+ if (( $(available_gb) < threshold )) && mountpoint -q /mnt; then
+ echo "::group::Moving /var/lib/docker to /mnt/docker"
+ sudo systemctl stop docker
+ echo '{"data-root": "/mnt/docker"}' | sudo tee
/etc/docker/daemon.json
+ sudo mv /var/lib/docker /mnt/docker
+ sudo systemctl start docker
+ time show_disk
+ echo "::endgroup::"
+ fi
+ fi
+
+ if (( $(available_gb) < threshold )); then
+ echo "::group::Cleaning apt state"
+ time sudo bash -c "apt-get clean; apt-get autoclean; apt-get -y
--purge autoremove"
+ time show_disk
echo "::endgroup::"
fi
- echo "::group::Cleaning apt state"
- time sudo bash -c "apt-get clean; apt-get autoclean; apt-get -y
--purge autoremove"
- time df -BM / /mnt
+
+ echo "::group::Available diskspace"
+ time show_disk
echo "::endgroup::"
fi
- echo "::group::Available diskspace"
- time df -BM / /mnt
- echo "::endgroup::"
- shell: bash
diff --git a/.github/actions/tune-runner-vm/action.yml
b/.github/actions/tune-runner-vm/action.yml
index ab0f65767a6..d071f534dbb 100644
--- a/.github/actions/tune-runner-vm/action.yml
+++ b/.github/actions/tune-runner-vm/action.yml
@@ -22,7 +22,8 @@ description: tunes the GitHub Runner VM operation system
runs:
using: composite
steps:
- - run: |
+ - shell: bash
+ run: |
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
echo "::group::Configure and tune OS"
# Ensure that reverse lookups for current hostname are handled
properly
@@ -33,49 +34,38 @@ runs:
# consumption is high.
# Set vm.swappiness=1 to avoid swapping and allow high RAM usage
echo 1 | sudo tee /proc/sys/vm/swappiness
- (
- shopt -s nullglob
- # Set swappiness to 1 for all cgroups and sub-groups
- for swappiness_file in /sys/fs/cgroup/memory/*/memory.swappiness
/sys/fs/cgroup/memory/*/*/memory.swappiness; do
- echo 1 | sudo tee $swappiness_file > /dev/null
- done
- ) || true
# use "madvise" Linux Transparent HugePages (THP) setting
#
https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html
# "madvise" is generally a better option than the default "always"
setting
- # Based on Azul instructions from
https://docs.azul.com/prime/Enable-Huge-Pages#transparent-huge-pages-thp
+ # recommendation from
https://netflixtechblog.com/bending-pause-times-to-your-will-with-generational-zgc-256629c9386b
echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled
echo advise | sudo tee
/sys/kernel/mm/transparent_hugepage/shmem_enabled
- echo defer+madvise | sudo tee
/sys/kernel/mm/transparent_hugepage/defrag
+ echo defer | sudo tee /sys/kernel/mm/transparent_hugepage/defrag
echo 1 | sudo tee
/sys/kernel/mm/transparent_hugepage/khugepaged/defrag
# tune filesystem mount options,
https://www.kernel.org/doc/Documentation/filesystems/ext4.txt
# commit=999999, effectively disables automatic syncing to disk
(default is every 5 seconds)
# nobarrier/barrier=0, loosen data consistency on system crash (no
negative impact to empheral CI nodes)
sudo mount -o remount,nodiscard,commit=999999,barrier=0 / || true
- sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt ||
true
+ if mountpoint -q /mnt; then
+ sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt ||
true
+ fi
# disable discard/trim at device level since remount with
nodiscard doesn't seem to be effective
# https://www.spinics.net/lists/linux-ide/msg52562.html
for i in /sys/block/sd*/queue/discard_max_bytes; do
echo 0 | sudo tee $i
done
- # disable any background jobs that run SSD discard/trim
- sudo systemctl disable fstrim.timer || true
- sudo systemctl stop fstrim.timer || true
- sudo systemctl disable fstrim.service || true
- sudo systemctl stop fstrim.service || true
+ # disable unnecessary timers
+ sudo systemctl stop fstrim.timer fstrim.service \
+ podman-auto-update.timer sysstat-collect.timer
sysstat-summary.timer \
+ phpsessionclean.timer man-db.timer motd-news.timer \
+ dpkg-db-backup.timer e2scrub_all.timer \
+ update-notifier-download.timer update-notifier-motd.timer || true
- # stop php-fpm
- sudo systemctl stop php8.0-fpm.service || true
- sudo systemctl stop php7.4-fpm.service || true
- # stop mono-xsp4
- sudo systemctl disable mono-xsp4.service || true
- sudo systemctl stop mono-xsp4.service || true
- sudo killall mono || true
-
- # stop Azure Linux agent to save RAM
- sudo systemctl stop walinuxagent.service || true
+ # stop unnecessary services
+ sudo systemctl stop php8.3-fpm.service ModemManager.service \
+ multipathd.service udisks2.service walinuxagent.service || true
echo '::endgroup::'
@@ -87,10 +77,4 @@ runs:
echo "::group::Available diskspace"
df -BM
echo "::endgroup::"
- # show cggroup
- echo "::group::Cgroup settings for current cgroup $CURRENT_CGGROUP"
- CURRENT_CGGROUP=$(cat /proc/self/cgroup | grep '0::' | awk -F: '{
print $3 }')
- sudo cgget -a $CURRENT_CGGROUP || true
- echo '::endgroup::'
- fi
- shell: bash
+ fi
\ No newline at end of file
