* tests/prove-runner: New helper script; mostly it wraps
our `*.test' test scripts to make them runnable by the
`prove' utility.
* tests/Makefile.am (EXTRA_DIST): Distribute it.
(PROVE, AM_PROVEFLAGS, AM_PROVECMD): New variables.
(prove, installprove): New targets, used run the automake
testsuite through the `prove' utility.
* tests/gen-testsuite-part: For any considered dependency,
generate a variables that groups all the tests depending on
it. This is especially useful for use in the new `prove'
and `installprove' targets, which cannot rely on make for
inter-test dependencies, and have to handle them "by hand".
---
ChangeLog | 16 ++++
tests/Makefile.am | 83 +++++++++++++++++++++
tests/Makefile.in | 159 +++++++++++++++++++++++++++++++++++++++--
tests/gen-testsuite-part | 13 ++--
tests/prove-runner | 50 +++++++++++++
5 files changed, 311 insertions(+), 10 deletions(-)
create mode 100755 tests/prove-runner
mode change 100644 => 100755 tests/test-logs-repeated.test
diff --git a/ChangeLog b/ChangeLog
index 39aa991..f4aa951 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,19 @@
+2011-09-10 Stefano Lattarini <stefano.lattar...@gmail.com>
+
+ tests: allow use of `prove' to run automake testsuite
+ * tests/prove-runner: New helper script; mostly it wraps
+ our `*.test' test scripts to make them runnable by the
+ `prove' utility.
+ * tests/Makefile.am (EXTRA_DIST): Distribute it.
+ (PROVE, AM_PROVEFLAGS, AM_PROVECMD): New variables.
+ (prove, installprove): New targets, used run the automake
+ testsuite through the `prove' utility.
+ * tests/gen-testsuite-part: For any considered dependency,
+ generate a variables that groups all the tests depending on
+ it. This is especially useful for use in the new `prove'
+ and `installprove' targets, which cannot rely on make for
+ inter-test dependencies, and have to handle them "by hand".
+
2011-09-09 Stefano Lattarini <stefano.lattar...@gmail.com>
test defs: more environment cleanup
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 51edc2f..1f4f9ab 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -183,3 +183,86 @@ clean-local-check:
find "$$@" -type d ! -perm -700 -exec chmod u+rwx {} ';'; \
rm -rf "$$@"; \
fi;
+
+## Run the testsuite through the `prove' utility. Mostly for maintainers,
+## and to find out possible incompatibilities between `prove' and the
+## Automake own TAP driver.
+
+PROVE = prove
+AM_PROVEFLAGS = --timer
+AM_PROVECMD = \
+ test -z "$$tests" || \
+ { \
+ $(AM_TESTS_ENVIRONMENT) $(TESTS_ENVIRONMENT) \
+ $(PROVE) $(AM_PROVEFLAGS) -e $(srcdir)/prove-runner $(PROVEFLAGS) \
+ $$tests $(AM_TESTS_FD_REDIRECT); \
+ }
+
+EXTRA_DIST += prove-runner
+
+.PHONY: prove installprove
+prove installprove: $(TESTS) prove-runner defs-static
+ @exit_status=0; \
+## The testsuite can be run either using the automake build tree or
+## the already-installed automake package.
+ case '$@' in \
+ install*) am_running_installcheck=yes;; \
+ *) am_running_installcheck=no;; \
+ esac; \
+ export am_running_installcheck; \
+## XXX Our poor-man way to handle inter-tests dependencies without the
+## XXX help of make. This is obviously fragile and doesn't scale,
+## XXX but for now it's good enough since we have only one possible
+## XXX inter-tests dependency at the moment.
+ need_tap_common_setup=no; \
+ lst=' $(TESTS) '; for t in $$lst; do \
+ case ' $(on_tap_with_common_setup_TESTS) ' in \
+ *" $$t "*) need_tap_common_setup=yes; break;; \
+ esac; \
+ done; \
+## For VPATH builds.
+ if test $$need_tap_common_setup = yes; then \
+ t=tap-common-setup.test; \
+ if test -f ./$$t; then \
+ tests=./$$t; \
+ else \
+ tests=$(srcdir)/$$t; \
+ fi; \
+## Do not stop if there's a failure here, we want to run the following
+## tests anyway.
+ echo "$@: running pre-requisite tests ..."; \
+ $(AM_PROVECMD) || exits_status=$$?; \
+ echo "$@: done. We'll shortly run the main testsuite."; \
+ fi; \
+## Handle VPATH builds, and ensure that the tests have a properly
+## qualified path.
+ tests=`\
+ for t in $(TESTS); do \
+## XXX In next line, another hack for inter-tests dependencies: this
+## XXX can't run concurrently to the tests that use the data files it
+## XXX generates.
+ case "$$need_tap_common_setup,/$$t" in \
+ yes,*/tap-common-setup.test) continue;; \
+ esac; \
+ if test -f ./$$t; then \
+ echo ./$$t; \
+ else \
+ echo $(srcdir)/$$t; \
+ fi; \
+ done`; \
+## Extra environment, mostly for use by `prove-runner'.
+## XFAIL_TESTS gets normalized to play nicer with VPATH builds.
+ XFAIL_TESTS=`\
+ for t in $(XFAIL_TESTS); do \
+ echo $$t; echo ./$$t; echo $(srcdir)/$$t; \
+ done | LC_ALL=C sort | uniq`; \
+ XFAIL_TESTS=`echo $$XFAIL_TESTS`; \
+ TAP_RUNNER='$(TAP_LOG_COMPILE)' \
+ TEST_RUNER='$(TEST_LOG_COMPILE)' \
+ export XFAIL_TESTS TAP_RUNNER TEST_RUNNER; \
+## For consistency with the Automake testsuite harness, that exports
+## this unconditionally in the test scripts' environment.
+ srcdir='$(srcdir)'; export srcdir; \
+## Here we go.
+ $(AM_PROVECMD) || exit_status=$$?; \
+ exit $$exit_status;
diff --git a/tests/Makefile.in b/tests/Makefile.in
index 5ae15b1..b131ecf 100644
--- a/tests/Makefile.in
+++ b/tests/Makefile.in
@@ -295,11 +295,12 @@ TAP_LOG_COMPILER = $(LOG_COMPILER)
TAP_LOG_DRIVER = AM_TAP_AWK='$(AWK)' $(SHELL) $(top_srcdir)/lib/tap-driver.sh
AM_TAP_LOG_DRIVER_FLAGS = --merge --comments --diagnostic-string \
`printf '\043%%\043\n'`
-EXTRA_DIST = ChangeLog-old $(handwritten_TESTS) tap-summary-aux.sh \
- tap-setup.sh extract-testsuite-summary \
+EXTRA_DIST = ChangeLog-old $(handwritten_TESTS) tap-setup.sh \
+ tap-summary-aux.sh extract-testsuite-summary \
testsuite-summary-checks.sh trivial-test-driver \
$(generated_TESTS) gen-testsuite-part tap-functions.sh \
- plain-functions.sh distcheck-hook-m4.am defs-static.in
+ plain-functions.sh distcheck-hook-m4.am defs-static.in \
+ prove-runner
# Hand-written tests.
@@ -1435,6 +1436,96 @@ yflags-force-override.test \
yflags-force-conditional.test \
yflags-var-expand.test
+on_tap_with_common_setup_TESTS = \
+ tap-ambiguous-directive.test \
+ tap-autonumber.test \
+ tap-bailout.test \
+ tap-bailout-and-logging.test \
+ tap-bailout-suppress-badexit.test \
+ tap-bailout-suppress-later-diagnostic.test \
+ tap-bailout-suppress-later-errors.test \
+ tap-color.test \
+ tap-deps.test \
+ tap-diagnostic.test \
+ tap-empty-diagnostic.test \
+ tap-empty.test \
+ tap-escape-directive.test \
+ tap-escape-directive-2.test \
+ tap-exit.test \
+ tap-signal.tap \
+ tap-fancy.test \
+ tap-fancy2.test \
+ tap-global-log.test \
+ tap-global-result.test \
+ tap-html.test \
+ tap-log.test \
+ tap-msg0-result.test \
+ tap-msg0-directive.test \
+ tap-msg0-planskip.test \
+ tap-msg0-bailout.test \
+ tap-msg0-misc.test \
+ tap-merge-stdout-stderr.test \
+ tap-no-merge-stdout-stderr.test \
+ tap-no-disable-hard-error.test \
+ tap-no-spurious-summary.test \
+ tap-no-spurious-numbers.test \
+ tap-no-spurious.test \
+ tap-not-ok-skip.test \
+ tap-number-wordboundary.test \
+ tap-numeric-description.test \
+ tap-negative-numbers.test \
+ tap-numbers-leading-zero.test \
+ tap-out-of-order.test \
+ tap-passthrough.test \
+ tap-passthrough-exit.test \
+ tap-plan.test \
+ tap-plan-corner.test \
+ tap-plan-errors.test \
+ tap-plan-middle.test \
+ tap-plan-whitespace.test \
+ tap-plan-leading-zero.test \
+ tap-plan-malformed.test \
+ tap-missing-plan-and-bad-exit.test \
+ tap-planskip.test \
+ tap-planskip-late.test \
+ tap-planskip-and-logging.test \
+ tap-planskip-unplanned.test \
+ tap-planskip-unplanned-corner.test \
+ tap-planskip-case-insensitive.test \
+ tap-planskip-whitespace.test \
+ tap-planskip-badexit.test \
+ tap-planskip-bailout.test \
+ tap-planskip-later-errors.test \
+ tap-realtime.test \
+ tap-test-number-0.test \
+ tap-recheck-logs.test \
+ tap-result-comment.test \
+ tap-todo-skip-together.test \
+ tap-todo-skip-whitespace.test \
+ tap-todo-skip.test \
+ tap-unplanned.test \
+ tap-whitespace-normalization.test \
+ tap-with-and-without-number.test \
+ tap-xfail-tests.test
+
+check_tap_testsuite_summary_TESTS = \
+ tap-summary.test \
+ tap-summary-color.test
+
+extract_testsuite_summary_TESTS = \
+ testsuite-summary-count-many.test
+
+check_testsuite_summary_TESTS = \
+ testsuite-summary-color.test \
+ testsuite-summary-count.test
+
+use_trivial_test_driver_TESTS = \
+ testsuite-summary-count-many.test \
+ test-driver-custom-multitest.test \
+ test-driver-custom-multitest-recheck.test \
+ test-driver-custom-multitest-recheck2.test \
+ test-driver-custom-html.test
+
generated_TESTS = ar-lib-w.test compile-w.test compile2-w.test \
compile3-w.test compile4-w.test compile5-w.test \
compile6-w.test instsh2-w.test instsh3-w.test mdate5-w.test \
@@ -1527,6 +1618,16 @@ CLEANFILES = defs-static
am__tmk = tests-in-makefile-list.tmp
am__tfs = tests-on-filesystem-list.tmp
am__tdf = diff-in-tests-lists.tmp
+PROVE = prove
+AM_PROVEFLAGS = --timer
+AM_PROVECMD = \
+ test -z "$$tests" || \
+ { \
+ $(AM_TESTS_ENVIRONMENT) $(TESTS_ENVIRONMENT) \
+ $(PROVE) $(AM_PROVEFLAGS) -e $(srcdir)/prove-runner $(PROVEFLAGS) \
+ $$tests $(AM_TESTS_FD_REDIRECT); \
+ }
+
all: all-am
.SUFFIXES:
@@ -1920,8 +2021,6 @@ uninstall-am:
print-list-of-tests:
@echo $(handwritten_TESTS)
.PHONY: print-list-of-tests
-tap-summary.log: tap-summary-aux.sh
-tap-summary-color.log: tap-summary-aux.sh
tap-ambiguous-directive.log: tap-setup.sh tap-common-setup.log
tap-autonumber.log: tap-setup.sh tap-common-setup.log
tap-bailout.log: tap-setup.sh tap-common-setup.log
@@ -1992,6 +2091,8 @@ tap-unplanned.log: tap-setup.sh tap-common-setup.log
tap-whitespace-normalization.log: tap-setup.sh tap-common-setup.log
tap-with-and-without-number.log: tap-setup.sh tap-common-setup.log
tap-xfail-tests.log: tap-setup.sh tap-common-setup.log
+tap-summary.log: tap-summary-aux.sh
+tap-summary-color.log: tap-summary-aux.sh
testsuite-summary-count-many.log: extract-testsuite-summary
testsuite-summary-color.log: testsuite-summary-checks.sh
testsuite-summary-count.log: testsuite-summary-checks.sh
@@ -2341,6 +2442,54 @@ clean-local-check:
rm -rf "$$@"; \
fi;
+.PHONY: prove installprove
+prove installprove: $(TESTS) prove-runner defs-static
+ @exit_status=0; \
+ case '$@' in \
+ install*) am_running_installcheck=yes;; \
+ *) am_running_installcheck=no;; \
+ esac; \
+ export am_running_installcheck; \
+ need_tap_common_setup=no; \
+ lst=' $(TESTS) '; for t in $$lst; do \
+ case ' $(on_tap_with_common_setup_TESTS) ' in \
+ *" $$t "*) need_tap_common_setup=yes; break;; \
+ esac; \
+ done; \
+ if test $$need_tap_common_setup = yes; then \
+ t=tap-common-setup.test; \
+ if test -f ./$$t; then \
+ tests=./$$t; \
+ else \
+ tests=$(srcdir)/$$t; \
+ fi; \
+ echo "$@: running pre-requisite tests ..."; \
+ $(AM_PROVECMD) || exits_status=$$?; \
+ echo "$@: done. We'll shortly run the main testsuite."; \
+ fi; \
+ tests=`\
+ for t in $(TESTS); do \
+ case "$$need_tap_common_setup,/$$t" in \
+ yes,*/tap-common-setup.test) continue;; \
+ esac; \
+ if test -f ./$$t; then \
+ echo ./$$t; \
+ else \
+ echo $(srcdir)/$$t; \
+ fi; \
+ done`; \
+ XFAIL_TESTS=`\
+ for t in $(XFAIL_TESTS); do \
+ echo $$t; echo ./$$t; echo $(srcdir)/$$t; \
+ done | LC_ALL=C sort | uniq`; \
+ XFAIL_TESTS=`echo $$XFAIL_TESTS`; \
+ TAP_RUNNER='$(TAP_LOG_COMPILE)' \
+ TEST_RUNER='$(TEST_LOG_COMPILE)' \
+ export XFAIL_TESTS TAP_RUNNER TEST_RUNNER; \
+ srcdir='$(srcdir)'; export srcdir; \
+ $(AM_PROVECMD) || exit_status=$$?; \
+ exit $$exit_status;
+
# Tell versions [3.59,3.63) of GNU make to not export all variables.
# Otherwise a system limit (for SysV at least) may be exceeded.
.NOEXPORT:
diff --git a/tests/gen-testsuite-part b/tests/gen-testsuite-part
index fc21487..1c1c55b 100755
--- a/tests/gen-testsuite-part
+++ b/tests/gen-testsuite-part
@@ -158,7 +158,7 @@ my %deps_extractor =
line_matcher => qr/\btap-summary-aux\.sh\b/,
dist_prereqs => "tap-summary-aux.sh",
},
- is_test_on_tap_with_common_setup =>
+ on_tap_with_common_setup =>
{
line_matcher => qr/\btap-setup\.sh\b/,
dist_prereqs => "tap-setup.sh",
@@ -246,7 +246,7 @@ my %test_generators =
parse_options @ARGV;
-my @tests = get_list_of_tests;
+my @all_tests = get_list_of_tests;
print "## -*- Makefile -*-\n";
print "## Generated by $me. DO NOT EDIT BY HAND!\n\n";
@@ -263,9 +263,12 @@ while (my ($k, $x) = each %deps_extractor)
{
my $dist_prereqs = $x->{dist_prereqs} || "";
my $nodist_prereqs = $x->{nodist_prereqs} || "";
- my @logs = grep { line_match $x->{line_matcher}, $_ } @tests;
- map { s/\.[^.]*$//; s/$/\.log/; } @logs;
+ my @tests = grep { line_match $x->{line_matcher}, $_ } @all_tests;
+ map { s/\.[^.]*$//; s/$/\.log/; } (my @logs = @tests);
print "## Added by deps-extracting key `$k'.\n";
+ ## The list of all tests which have a dependency detected by the
+ ## current key.
+ print join(" \\\n ", "${k}_TESTS =", @tests) . "\n";
print "EXTRA_DIST += $dist_prereqs\n";
map { print "$_: $dist_prereqs $nodist_prereqs\n" } @logs;
print "\n";
@@ -289,7 +292,7 @@ while (my ($k, $g) = each %test_generators)
my @wrapped_tests = grep {
line_match ($g->{line_matcher}, $_)
&& !line_match ($g->{line_rejecter}, $_)
- } @tests;
+ } @all_tests;
foreach my $wrapped_test (@wrapped_tests)
{
(my $base = $wrapped_test) =~ s/\.([^.]*)$//;
diff --git a/tests/prove-runner b/tests/prove-runner
new file mode 100755
index 0000000..41c4c52
--- /dev/null
+++ b/tests/prove-runner
@@ -0,0 +1,50 @@
+#!/bin/sh
+# Helper script to run the automake testsuite through the `prove' utility.
+# See the "prove" and "installprove" targets in tests/Makefile.am.
+
+set -u
+
+nl='
+'
+
+tst=$1
+shift
+
+case $tst in
+ *.test)
+ echo 1..1
+ # Protect the output contents of the test script, since it might
+ # sometimes contain TAP lines that would confuse `prove'. This
+ # This has already happened in practice, especially for the tests
+ # checking TAP support in automake-generated test harnesses.
+ (${TEST_RUNNER-} "$tst" ${1+"$@"}; echo exit status: $?) 2>&1 | awk '
+ BEGIN { lastine = "exit status: 255"; }
+ { print " " $0; lastline = $0; }
+ END { st = lastline; sub("^exit status: *", "", st); exit st; }
+ '
+ st=$?
+ if test $st -eq 99; then
+ echo "not ok - $tst: hard failure (exit status $st)"
+ elif test $st -eq 77; then
+ echo "ok - $tst # SKIP"
+ else
+ case " ${XFAIL_TESTS-} " in
+ *" $tst "*) xs=' # TODO';;
+ *) xs='';;
+ esac
+ if test $st -eq 0; then
+ echo "ok - test script $tst passed$xs"
+ else
+ echo "not ok - test script $tst failed (exit status $st)$xs"
+ fi
+ fi
+ exit 0
+ ;;
+ *.tap)
+ ${TAP_RUNNER-} "$tst" ${1+"$@"} 2>&1
+ exit $?
+ ;;
+esac
+
+echo "Bail out! Invalid test script name '$tst'"
+exit 99
diff --git a/tests/test-logs-repeated.test b/tests/test-logs-repeated.test
old mode 100644
new mode 100755
--
1.7.2.3
