From: Prarit Bhargava <[email protected]> redhat/self-test: Provide better failure output
Currently when the tests fail there isn't a lot of information on which test failed. This has resulted in confusion by developers. Add a general bash function, check_status, that outputs some basic information on which test failed and how to run the test to reproduce the error. Signed-off-by: Prarit Bhargava <[email protected]> diff --git a/redhat/self-test/0001-shellcheck.bats b/redhat/self-test/0001-shellcheck.bats index blahblah..blahblah 100644 --- a/redhat/self-test/0001-shellcheck.bats +++ b/redhat/self-test/0001-shellcheck.bats @@ -1,9 +1,12 @@ #!/usr/bin/env bats # Purpose: This test runs shellcheck on all .sh files in the redhat directory. +load test-lib.bash + @test "shellcheck" { if ! test -x /usr/bin/shellcheck; then skip "The ShellCheck package is not installed" fi - shellcheck $(find $BATS_TEST_DIRNAME/.. -name "*.sh" -not -path "$BATS_TEST_DIRNAME/../rpm/*") + run shellcheck $(find $BATS_TEST_DIRNAME/.. -name "*.sh" -not -path "$BATS_TEST_DIRNAME/../rpm/*") + check_status } diff --git a/redhat/self-test/1001-rpmlint.bats b/redhat/self-test/1001-rpmlint.bats index blahblah..blahblah 100644 --- a/redhat/self-test/1001-rpmlint.bats +++ b/redhat/self-test/1001-rpmlint.bats @@ -1,6 +1,8 @@ #!/usr/bin/env bats # Purpose: This test runs rpmlint on the source rpm. +load test-lib.bash + @test "rpmlint" { if ! test -x /usr/bin/rpmlint; then skip "The rpmlint package is not installed" @@ -15,6 +17,5 @@ srpm=$(find "$BATS_TEST_DIRNAME"/.. -name "*.rpm") run rpmlint $srpm - status=$? - [ "$status" = 0 ] + check_result } diff --git a/redhat/self-test/1002-basic-structural-test.bats b/redhat/self-test/1002-basic-structural-test.bats index blahblah..blahblah 100644 --- a/redhat/self-test/1002-basic-structural-test.bats +++ b/redhat/self-test/1002-basic-structural-test.bats @@ -1,6 +1,12 @@ #!/usr/bin/env bats # Purpose: This test runs tests on the SRPM. +load test-lib.bash + +_SRPM_unpacks_OK() { + rpm2cpio "$srpm" | cpio -idm +} + @test "SRPM unpacks OK" { numsrpms=$(find "$BATS_TEST_DIRNAME"/.. -name "*.rpm" | wc -l) if [ "$numsrpms" != "1" ]; then @@ -14,9 +20,8 @@ fi mkdir SRPMS cd SRPMS - rpm2cpio "$srpm" | cpio -idm - status=$? - [ "$status" = 0 ] + run _SRPM_unpacks_OK + check_status popd >& /dev/null } @@ -30,7 +35,7 @@ numsrpms=$(find "$BATS_TEST_DIRNAME"/.. -name "*.rpm" | wc -l) ls | wc linuxname=$(ls linux*.tar.xz) run tar --extract --xz -f "$linuxname" - [ "$status" = 0 ] + check_status popd >& /dev/null } @@ -44,7 +49,7 @@ numsrpms=$(find "$BATS_TEST_DIRNAME"/.. -name "*.rpm" | wc -l) linuxtree=$(ls linux*.tar.xz) linuxtree=${linuxtree/.tar.xz} cd $linuxtree - test -d arch && \ + run test -d arch && \ test -d block && \ test -d certs && \ test -d crypto && \ @@ -66,7 +71,6 @@ numsrpms=$(find "$BATS_TEST_DIRNAME"/.. -name "*.rpm" | wc -l) test -d tools && \ test -d usr && \ test -d virt - status=$? + check_status popd >& /dev/null - [ "$status" = 0 ] } diff --git a/redhat/self-test/1003-rpminspect.bats b/redhat/self-test/1003-rpminspect.bats index blahblah..blahblah 100644 --- a/redhat/self-test/1003-rpminspect.bats +++ b/redhat/self-test/1003-rpminspect.bats @@ -1,6 +1,8 @@ #!/usr/bin/env bats # Purpose: This test runs rpminspect on the SRPM. +load test-lib.bash + @test "rpminspect" { if ! test -x /usr/bin/rpminspect; then skip "The rpminspect package is not installed" @@ -15,5 +17,5 @@ srpm=$(find "$BATS_TEST_DIRNAME"/.. -name "*.rpm") run rpminspect $srpm - [ "$status" = 0 ] + check_status } diff --git a/redhat/self-test/1005-dist-dump-variables.bats b/redhat/self-test/1005-dist-dump-variables.bats index blahblah..blahblah 100644 --- a/redhat/self-test/1005-dist-dump-variables.bats +++ b/redhat/self-test/1005-dist-dump-variables.bats @@ -3,6 +3,8 @@ # variables that are used in the specfile. This data is diff'd against a # "known good" set of data and if there is a difference an error is reported. +load test-lib.bash + @test "self-test-data check" { mkdir -p $BATS_TMPDIR/data RHDISTDATADIR=$BATS_TMPDIR/data make dist-self-test-data @@ -10,6 +12,7 @@ redhat=$(make dist-dump-variables | grep "REDHAT=" | cut -d"=" -f2 | xargs) echo "Diffing directories ${redhat}/self-test/data and $BATS_TMPDIR/data" - diff -urNp -x create-data.sh ${redhat}/self-test/data $BATS_TMPDIR/data + run diff -urNp -x create-data.sh ${redhat}/self-test/data $BATS_TMPDIR/data [ -d $BATS_TMPDIR ] && rm -rf $BATS_TMPDIR/data + check_status } diff --git a/redhat/self-test/1006-verify-SPEC-variables.bats b/redhat/self-test/1006-verify-SPEC-variables.bats index blahblah..blahblah 100644 --- a/redhat/self-test/1006-verify-SPEC-variables.bats +++ b/redhat/self-test/1006-verify-SPEC-variables.bats @@ -2,8 +2,9 @@ # Purpose: This test looks at the spec file variable replacement code in # redhat/genspec.sh and confirms that each variable begins with "SPEC". -@test "verify SPEC variables" { +load test-lib.bash +_verify_SPEC_variables() { # This looks at the code and replaces each / with a new-line character, removes # any whitespace and entry entries beginning with valid "%%SPEC" or $"SPEC". # "$SOURCES" lines are also okay as it is used to point to the changelog and @@ -23,3 +24,8 @@ do esac done } + +@test "verify SPEC variables" { + run _verify_SPEC_variables + check_status +} diff --git a/redhat/self-test/2001-dist-release.bats b/redhat/self-test/2001-dist-release.bats index blahblah..blahblah 100644 --- a/redhat/self-test/2001-dist-release.bats +++ b/redhat/self-test/2001-dist-release.bats @@ -2,7 +2,9 @@ # Purpose: These are general dist-release tests. They are run from a git # worktree created by the first test. -@test "dist-release prologue" { +load test-lib.bash + +@test "dist-release setup worktree" { git worktree add $BATS_TMPDIR/distrelease cd $BATS_TMPDIR/distrelease # All the tests start off with 'make dist-release', so we can pull @@ -21,7 +23,12 @@ logb=($(git log --oneline -n 2 | tail -1)) # If SHA1 in loga is the same as the SHA1 in logb, then no # 2nd commit has been created and the test has succeeded: - [ ${loga[0]} = ${logb[0]} ] + run [ ${loga[0]} = ${logb[0]} ] + check_status +} + +_dist-release_test_2() { + echo $pkgrelease | grep -q -w "$build" } @test "dist-release test 2" { @@ -39,11 +46,17 @@ ((build--)) echo "pkgrelease=$pkgrelease" echo "build=$build" - echo $pkgrelease | grep -q -w "$build" - status=$? - [ "$status" = 0 ] + run _dist-release_test_2 + check_status } +_dist-release_test_3() { + [ "$changelog" = "$gitlog" ] +} + +# Note, when running this test on the command line you may have to specifiy the +# RHEL_MAJOR and RHEL_MINOR variables, for example, +# RHEL_MAJOR=9 RHEL_MINOR=99 bats redhat/self-test/2001-dist-release.bats @test "dist-release test 3" { # Test whether the version in the commit message matches # the version in the change log. @@ -56,11 +69,12 @@ gitlog=${commit##*\[redhat\] } # This time, strip off "kernel-" also: gitlog=${gitlog/kernel-/} - echo "The kernel version in the changelog ($changelog) differs from the version in the git log ($gitlog)" - [ "$changelog" = "$gitlog" ] + echo "The kernel version in the changelog-${RHEL_MAJOR}.${RHEL_MINOR} ("${changelog}") differs from the version in the git log ($gitlog)" + run _dist-release_test_3 + check_status } -@test "dist-release epilogue" { +@test "dist-release cleanup worktree" { git worktree remove --force $BATS_TMPDIR/distrelease git branch -D distrelease } diff --git a/redhat/self-test/3001-Makefile-contents.bats b/redhat/self-test/3001-Makefile-contents.bats index blahblah..blahblah 100755 --- a/redhat/self-test/3001-Makefile-contents.bats +++ b/redhat/self-test/3001-Makefile-contents.bats @@ -2,18 +2,28 @@ # Purpose: This is a test that verifies that Makefile.variable variable # declarations are all declared with "?=" -@test "Makefile variable declarations" { - # By design, only the Makefile.variables file should have ?= declarations +load test-lib.bash + +_Makefile_variable_declarations_1() { + git grep "?=" $BATS_TEST_DIRNAME/../Makefile.variables | wc -l +} - value=$(git grep "?=" Makefile.variables | wc -l) - if [ $value -eq 0 ]; then +_Makefile_variable_declarations_2() { + git grep "?=" $BATS_TEST_DIRNAME/../Makefile | grep -v "\"?=" | wc -l +} + +@test "Makefile variable declarations" { + run _Makefile_variable_declarations_1 + if [ "$output" -eq 0 ]; then echo "Test failed: No ?= variables found in Makefile.variables" - exit 1 + status=1 fi + check_status - value=$(git grep "?=" Makefile | grep -v "\"?=" | wc -l) - if [ $value -gt 0 ]; then + run _Makefile_variable_declarations_2 + if [ "$output" -ne 0 ]; then echo "Test failed: Makefile should not ?= declarations." - exit 1 + status=1 fi + check_status } diff --git a/redhat/self-test/test-lib.bash b/redhat/self-test/test-lib.bash new file mode 100644 index blahblah..blahblah 100644 --- /dev/null +++ b/redhat/self-test/test-lib.bash @@ -0,0 +1,16 @@ +#!/usr/bin/bash + +# This function makes use of bats built-in run function and its status and output variables. +check_status() { + if [ "$status" -eq 0 ]; then + return 0 + fi + + # report the error + echo "$output" + echo "------------------" + expath=$(echo "$BATS_TEST_FILENAME" | rev | cut -d'/' -f-3 | rev) + echo -n "This redhat/self-test test has failed. You can run all tests by executing 'make dist-self-test', or just this test by executing 'bats $expath'." + exit 1 +} + -- https://gitlab.com/cki-project/kernel-ark/-/merge_requests/1881 _______________________________________________ kernel mailing list -- [email protected] To unsubscribe send an email to [email protected] Fedora Code of Conduct: https://docs.fedoraproject.org/en-US/project/code-of-conduct/ List Guidelines: https://fedoraproject.org/wiki/Mailing_list_guidelines List Archives: https://lists.fedoraproject.org/archives/list/[email protected] Do not reply to spam on the list, report it: https://pagure.io/fedora-infrastructure
