David Caro has uploaded a new change for review.
Change subject: Added slave cleanup
......................................................................
Added slave cleanup
That will make all the jobs that have yaml, to cleanup as much as possible
before starting, to avoid the last disk space issues.
Change-Id: I7575d38367008f9d9e1eaeaedc7f639d95e5b723
Signed-off-by: David Caro <dcaro...@redhat.com>
---
A jobs/confs/shell-scripts/cleanup_slave.sh
A jobs/confs/yaml/builders/cleanup.yaml
M jobs/confs/yaml/templates/create-rpms_mock.yaml
M jobs/confs/yaml/templates/engine-upgrade-db.yaml
M jobs/confs/yaml/templates/install-rpm-sanity.yaml
M jobs/confs/yaml/templates/ovirt-node-job-templates.yaml
M jobs/confs/yaml/templates/standard-stage.yaml
7 files changed, 190 insertions(+), 5 deletions(-)
git pull ssh://gerrit.ovirt.org:29418/jenkins refs/changes/77/37977/1
diff --git a/jobs/confs/shell-scripts/cleanup_slave.sh
b/jobs/confs/shell-scripts/cleanup_slave.sh
new file mode 100644
index 0000000..c2a8d71
--- /dev/null
+++ b/jobs/confs/shell-scripts/cleanup_slave.sh
@@ -0,0 +1,178 @@
+#!/bin/bash -e
+echo "shell-scripts/cleanup_slave.sh"
+
+WORKSPACE="${WORKSPACE?}"
+
+
+umount_everyhting_inside() {
+ local mydir="${1?}"
+ local res=0
+ local inner_dirs=(
+ $(mount | awk '{ print $3 }' | egrep "^$mydir" | sort -r)
+ )
+ local inner_dir
+ for inner_dir in "${inner_dirs[@]}"; do
+ sudo umount "$inner_dir" \
+ && echo " Umounted $inner_dir" \
+ || {
+ res=1
+ echo " FAILED to umount $inner_dir"
+ }
+ done
+ return $res
+}
+
+
+safe_remove() {
+ local dir="${1?}"
+ [[ -e "$dir" ]] || return 0
+ if [[ -d "$dir" ]]; then
+ umount_everyhting_inside "$dir" \
+ || return 1
+ sudo rm -Rf "$dir"
+ else
+ sudo rm -f "$dir"
+ fi
+ return 0
+}
+
+
+cleanup_home() {
+ local dir
+ for dir in /home/*; do
+ if ! grep -q "$dir" /etc/passwd; then
+ echo " Cleaning up non-user home dir $dir"
+ safe_remove "$dir"
+ fi
+ done
+}
+
+
+cleanup_var() {
+ local res=0
+ local dir
+ echo "Cleaning up /var/tmp"
+ for dir in /var/tmp/*; do
+ safe_remove "$dir" \
+ || {
+ echo " Error cleaning up $dir, skipping"
+ res=1
+ continue
+ }
+ done
+ echo "done"
+ return $res
+}
+
+
+cleanup_postgres() {
+ local db dbs
+ echo "Cleaning up postgres databases"
+ if ! [[ -e /var/lib/pgsql ]]; then
+ echo " Postgres installation not found, skipping"
+ return 0
+ fi
+ pushd /tmp
+ psql="sudo -u postgres psql"
+ dbs=(
+ $($psql -c "\l" \
+ | grep engine \
+ | cut -d '|' -f1)
+ )
+ for db in "${dbs[@]}"; do
+ echo " $db"
+ $psql -c "DROP DATABASE $db;"
+ done
+ popd
+ echo "done"
+ return 0
+}
+
+
+cleanup_logs() {
+ echo "Emptying some common logs"
+ local log empty_logfiles remove_logfiles
+ empty_logfiles=(
+ /var/log/wtmp
+ )
+ remove_logfiles=(
+ /var/log/messages-*
+ /var/log/secure-*
+ )
+ for log in "${empty_logfiles[@]}"; do
+ [[ -f "$log" ]] \
+ && sudo bash -c "> $log" \
+ && echo " $log"
+ done
+ for log in "${remove_logfiles[@]}"; do
+ [[ -f "$log" ]] \
+ && safe_remove "$log" \
+ && echo " $log"
+ done
+ sudo dmesg -C
+ echo "Done"
+ return 0
+}
+
+cleanup_journal() {
+ local dir
+ echo "Cleaning up journal logs (if any)"
+ if ! sudo service systemd-journald status &>/dev/null; then
+ echo " journald not running, skipping"
+ return 0
+ fi
+ # Flush logs
+ sudo systemctl kill --kill-who=main --signal=SIGUSR1
systemd-journald.service
+ for dir in /var/log/journal/*; do
+ safe_remove "$dir"
+ done
+ # force log reattach
+ sudo service systemd-journald restart
+}
+
+
+cleanup_workspaces() {
+ local cur_workspace="${1?}"
+ local res=0
+ local base_workspace workspace
+ base_workspace=~jenkins/workspace
+ for workspace in $base_workspace/*; do
+ [[ -d "$workspace" ]] || continue
+ [[ "$cur_workspace" =~ ^$workspace ]] && continue
+ echo " $workspace"
+ safe_remove "$workspace" \
+ || {
+ echo " Failed to remove $workspace, skipping"
+ res=1
+ continue
+ }
+ done
+ return $res
+}
+
+main() {
+ local workspace="${1?}"
+ echo "###################################################################"
+ echo "# Cleaning up slave #"
+ echo "###################################################################"
+ sudo df -h || :
+ echo "---------------------------------------------------------------"
+ sudo du -hs /* 2>/dev/null || :
+ echo "---------------------------------------------------------------"
+ cleanup_postgres || :
+ cleanup_journal || :
+ cleanup_var || :
+ cleanup_logs || :
+ cleanup_workspaces "$workspace" || :
+ cleanup_home || :
+ echo "---------------------------------------------------------------"
+ sudo df -h || :
+ echo "---------------------------------------------------------------"
+ sudo du -hs /* 2>/dev/null || :
+ echo "###################################################################"
+ echo "# Slave cleanup done #"
+ echo "###################################################################"
+}
+
+
+main "$WORKSPACE"
diff --git a/jobs/confs/yaml/builders/cleanup.yaml
b/jobs/confs/yaml/builders/cleanup.yaml
new file mode 100644
index 0000000..ed25bfb
--- /dev/null
+++ b/jobs/confs/yaml/builders/cleanup.yaml
@@ -0,0 +1,4 @@
+- builder:
+ name: cleanup-slave
+ builders:
+ - shell: !include-raw shell-scripts/cleanup_slave.sh
diff --git a/jobs/confs/yaml/templates/create-rpms_mock.yaml
b/jobs/confs/yaml/templates/create-rpms_mock.yaml
index b448212..389474c 100644
--- a/jobs/confs/yaml/templates/create-rpms_mock.yaml
+++ b/jobs/confs/yaml/templates/create-rpms_mock.yaml
@@ -13,6 +13,7 @@
project: '{project}'
branch: '{branch}'
builders:
+ - cleanup-slave
- mock-{mock-build-type}:
project: '{project}'
distro: '{distro}'
@@ -46,6 +47,7 @@
branch: '{branch}'
files: 'specs/{subproject}/.*'
builders:
+ - cleanup-slave
- mock-{mock-build-type}-releng:
project: '{project}'
subproject: '{subproject}'
diff --git a/jobs/confs/yaml/templates/engine-upgrade-db.yaml
b/jobs/confs/yaml/templates/engine-upgrade-db.yaml
index ee38720..2a78425 100644
--- a/jobs/confs/yaml/templates/engine-upgrade-db.yaml
+++ b/jobs/confs/yaml/templates/engine-upgrade-db.yaml
@@ -41,6 +41,7 @@
files: '(.*/)?packaging/dbscripts/.*'
builders:
- whitelist
+ - cleanup-slave
- ovirt-engine_upgrade-db:
test-branch: '{test-branch}'
action: '{action}'
diff --git a/jobs/confs/yaml/templates/install-rpm-sanity.yaml
b/jobs/confs/yaml/templates/install-rpm-sanity.yaml
index 129b90c..2a2c761 100644
--- a/jobs/confs/yaml/templates/install-rpm-sanity.yaml
+++ b/jobs/confs/yaml/templates/install-rpm-sanity.yaml
@@ -13,6 +13,7 @@
branch: '{branch}'
files: '{trigger-files}'
builders:
+ - cleanup-slave
- mock-{mock-build-type}-install:
project: '{project}'
distro: '{distro}'
diff --git a/jobs/confs/yaml/templates/ovirt-node-job-templates.yaml
b/jobs/confs/yaml/templates/ovirt-node-job-templates.yaml
index 593c68c..1f48eb0 100644
--- a/jobs/confs/yaml/templates/ovirt-node-job-templates.yaml
+++ b/jobs/confs/yaml/templates/ovirt-node-job-templates.yaml
@@ -1,12 +1,10 @@
-##############################################################################
-### Job Templates
-##############################################################################
- job-template:
name: 'ovirt-node_{version}_create-iso-{distro}_{trigger}'
node : '{distro}'
builders :
- - ovirt-node_create-iso:
- distro: '{distro}'
+ - cleanup-slave
+ - ovirt-node_create-iso:
+ distro: '{distro}'
scm :
- ovirt-node
- ovirt-node-iso
diff --git a/jobs/confs/yaml/templates/standard-stage.yaml
b/jobs/confs/yaml/templates/standard-stage.yaml
index 9b8943d..9c7b865 100644
--- a/jobs/confs/yaml/templates/standard-stage.yaml
+++ b/jobs/confs/yaml/templates/standard-stage.yaml
@@ -13,6 +13,7 @@
project: '{project}'
branch: '{branch}'
builders:
+ - cleanup-slave
- '{stage}_builder':
project: '{project}'
distro: '{distro}'
--
To view, visit http://gerrit.ovirt.org/37977
To unsubscribe, visit http://gerrit.ovirt.org/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I7575d38367008f9d9e1eaeaedc7f639d95e5b723
Gerrit-PatchSet: 1
Gerrit-Project: jenkins
Gerrit-Branch: master
Gerrit-Owner: David Caro <dcaro...@redhat.com>
_______________________________________________
Engine-patches mailing list
Engine-patches@ovirt.org
http://lists.ovirt.org/mailman/listinfo/engine-patches
