commit: e2d27f67f5c832e30b4683917e2bd7e28f22cfb9
Author: Andreas K. Huettel <dilfridge <AT> gentoo <DOT> org>
AuthorDate: Sat Aug 22 19:11:51 2020 +0000
Commit: Andreas K. Hüttel <dilfridge <AT> gentoo <DOT> org>
CommitDate: Sat Aug 29 12:33:37 2020 +0000
URL: https://gitweb.gentoo.org/proj/releng.git/commit/?id=e2d27f67
catalyst-auto: Allow parallel building of spec sets
This assumes that the spec sets are logically independent from each other,
i.e., building different ABIs.
Code shamelessly adapted from locale-gen.
Signed-off-by: Andreas K. Huettel <dilfridge <AT> gentoo.org>
tools/catalyst-auto | 97 ++++++++++++++++++++++++++++++++++++-----------------
1 file changed, 66 insertions(+), 31 deletions(-)
diff --git a/tools/catalyst-auto b/tools/catalyst-auto
index 34238c01..484eaffc 100755
--- a/tools/catalyst-auto
+++ b/tools/catalyst-auto
@@ -23,6 +23,7 @@ testing=0
preclean=0
lastrun=0
lock_file=
+parallel_sets=1
usage() {
local msg=$1
@@ -38,6 +39,7 @@ Usage:
Options:
-c|--config Specifies the config file to use (required)
-C|--preclean Clean up loose artifacts from previous runs
+ -j|--jobs <n> Build <n> spec sets in parallel
-v|--verbose Send output of commands to console as well as log
-k|--keep-tmpdir Don't remove temp dir when build finishes
-t|--test Stop after mangling specs and copying files
@@ -133,6 +135,10 @@ parse_args() {
config_files+=("$1")
shift
;;
+ -j|--jobs)
+ parallel_sets="$1"
+ shift
+ ;;
-v|--verbose)
verbose=$(($verbose+1))
;;
@@ -385,47 +391,76 @@ run_catalyst_commands() {
timeprefix=()
which time >/dev/null && timeprefix=( "time" )
+ JOB_PIDS=()
+ JOB_RETS=()
+ JOB_IDX_S=0
+ JOB_IDX_E=0
+
for a in "" ${SETS}; do
- if [[ -z ${a} ]]; then
- specs_var="SPECS"
- optional_specs_var="OPTIONAL_SPECS"
- else
- specs_var="SET_${a}_SPECS"
- optional_specs_var="SET_${a}_OPTIONAL_SPECS"
+
+ if [[ $(( JOB_IDX_E - JOB_IDX_S )) == ${parallel_sets} ]] ; then
+ wait ${JOB_PIDS[$(( JOB_IDX_S++ ))]}
+ JOB_RETS+=( $? )
fi
- for i in ${!specs_var}; do
- LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:'
-e 's:\.spec$::').log"
- specpath=$(readlink -f "${i}")
- run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst -a -c
"${CATALYST_CONFIG}" -f "${specpath}"
- if [[ $? != 0 ]]; then
- build_failure=1
- send_email "Catalyst fatal build error - ${i}"
"" "${LOGFILE}"
- continue 2
- else
- trigger_post_build "${a}" "${i}"
- fi
- done
+ (
- for i in ${!optional_specs_var}; do
- LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e 's:/:_:'
-e 's:\.spec$::').log"
- specpath=$(readlink -f "${i}")
- run_cmd "${LOGFILE}" "${timeprefix[@]}" catalyst -a -c
"${CATALYST_CONFIG}" -f "${specpath}"
- if [[ $? != 0 ]]; then
- build_failure=1
- send_email "Catalyst non-fatal build error -
${i}" "" "${LOGFILE}"
- break
+ if [[ -z ${a} ]]; then
+ specs_var="SPECS"
+ optional_specs_var="OPTIONAL_SPECS"
else
- trigger_post_build "${a}" "${i}"
+ specs_var="SET_${a}_SPECS"
+ optional_specs_var="SET_${a}_OPTIONAL_SPECS"
fi
- done
- # Do not purge yet, because there might be interdendency
between specs
- # in different build sets!
+ for i in ${!specs_var}; do
+ LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e
's:/:_:' -e 's:\.spec$::').log"
+ specpath=$(readlink -f "${i}")
+ run_cmd "${LOGFILE}" "${timeprefix[@]}"
catalyst -a -c "${CATALYST_CONFIG}" -f "${specpath}"
+ if [[ $? != 0 ]]; then
+ build_failure=1
+ send_email "Catalyst fatal build error
- ${i}" "" "${LOGFILE}"
+ exit 1
+ else
+ trigger_post_build "${a}" "${i}"
+ fi
+ done
+
+ for i in ${!optional_specs_var}; do
+ LOGFILE="${TMPDIR}/log/$(echo "${i}" | sed -e
's:/:_:' -e 's:\.spec$::').log"
+ specpath=$(readlink -f "${i}")
+ run_cmd "${LOGFILE}" "${timeprefix[@]}"
catalyst -a -c "${CATALYST_CONFIG}" -f "${specpath}"
+ if [[ $? != 0 ]]; then
+ build_failure=1
+ send_email "Catalyst non-fatal build
error - ${i}" "" "${LOGFILE}"
+ break
+ else
+ trigger_post_build "${a}" "${i}"
+ fi
+ done
+
+ # Do not purge yet, because there might be
interdendency between specs
+ # in different build sets!
+
+ update_symlinks
+
+ exit ${build_failure}
+
+ )&
+
+ JOB_PIDS+=( $! )
+ : $(( ++JOB_IDX_E ))
+
- update_symlinks
done
+ for (( i = JOB_IDX_S; i < JOB_IDX_E; ++i )) ; do
+ wait ${JOB_PIDS[i]}
+ JOB_RETS+=( $? )
+ done
+ build_failure=$(( 0 ${JOB_RETS[@]/#/+} ))
+
+
# Now do the cleanup
for a in "" ${SETS}; do
if [[ -z ${a} ]]; then
