regtest/HTMLReport.py | 197 ++++++++++++++++++++++++++++++++++++++ regtest/backends/__init__.py | 36 ++++++ regtest/backends/cairo.py | 2 regtest/backends/splash.py | 2 regtest/backends/text.py | 2 regtest/commands/create-report.py | 57 ++++++++++ 6 files changed, 290 insertions(+), 6 deletions(-)
New commits: commit 738b879ebb536cc84d7ec96543d484023b69e6d3 Author: Carlos Garcia Campos <[email protected]> Date: Sat Feb 25 20:53:58 2012 +0100 regtest: Add create-report command to generate html report of test results diff --git a/regtest/HTMLReport.py b/regtest/HTMLReport.py new file mode 100644 index 0000000..f64ec2d --- /dev/null +++ b/regtest/HTMLReport.py @@ -0,0 +1,197 @@ +# HTMLReport.py +# +# Copyright (C) 2012 Carlos Garcia Campos <[email protected]> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +from backends import get_backend, get_all_backends +from Config import Config +import os +import subprocess + +class BackendTestResult: + + def __init__(self, test, refsdir, outdir, backend, results): + self._test = test + self._refsdir = refsdir + self._outdir = outdir + self._backend = backend + + self._results = [] + + ref_path = os.path.join(self._refsdir, self._test) + ref_names = backend.get_ref_names(ref_path) + for result in results: + basename = os.path.basename(result) + if basename in ref_names: + self._results.append(basename) + + def is_failed(self): + return len(self._results) > 0 + + def is_crashed(self): + return self._backend.is_crashed(os.path.join(self._outdir, self._test)) + + def is_failed_to_run(self): + return self._backend.is_failed(os.path.join(self._outdir, self._test)) + + def get_stderr(self): + return self._backend.get_stderr(os.path.join(self._outdir, self._test)) + + def get_failed_html(self): + html = "" + for result in self._results: + actual = os.path.abspath(os.path.join(self._outdir, self._test, result)) + expected = os.path.abspath(os.path.join(self._refsdir, self._test, result)) + html += "<li><a href='%s'>actual</a> <a href='%s'>expected</a> " % (actual, expected) + if self._backend.has_diff(actual): + html += "<a href='%s'>diff</a>" % (os.path.abspath(actual + self._backend.get_diff_ext())) + html += "</li>\n" + + if html: + return "<ul>%s</ul>\n" % (html) + return "" + + +class TestResult: + + def __init__(self, docsdir, refsdir, outdir, resultdir, results, backends): + self._refsdir = refsdir + self._outdir = outdir + + self._test = resultdir[len(self._outdir):] + self._doc = os.path.join(docsdir, self._test) + self._results = {} + for backend in backends: + self._results[backend] = BackendTestResult(self._test, refsdir, outdir, backend, results) + + def get_test(self): + return self._test + + def is_failed(self): + for backend in self._results: + if self._results[backend].is_failed(): + return True + return False; + + def get_failed_html(self): + html = "" + for backend in self._results: + backend_html = self._results[backend].get_failed_html() + if not backend_html: + continue + + html += "<li>%s " % (backend.get_name()) + stderr = self._results[backend].get_stderr() + if os.path.exists(stderr): + html += "<a href='%s'>stderr</a>" % (os.path.abspath(stderr)) + html += "</li>\n%s" % (backend_html) + + if html: + return "<h2><a name='%s'><a href='%s'>%s</a></a></h2>\n<ul>%s</ul><a href='#top'>Top</a>\n" % (self._test, self._doc, self._test, html) + return "" + + def get_crashed_html(self): + html = "" + for backend in self._results: + if not self._results[backend].is_crashed(): + continue + + html += "<li><a href='%s'>%s</a> (%s)</li>\n" % (self._doc, self._test, backend.get_name()) + + if html: + return "<ul>%s</ul>\n" % (html) + return "" + + def get_failed_to_run_html(self): + html = "" + for backend in self._results: + status = self._results[backend].is_failed_to_run() + if not status: + continue + + html += "<li><a href='%s'>%s</a> [Status: %d] (%s)</li>\n" % (self._doc, self._test, status, backend.get_name()) + + if html: + return "<ul>%s</ul>\n" % (html) + return "" + +class HTMLReport: + + def __init__(self, docsdir, refsdir, outdir): + self._docsdir = docsdir + self._refsdir = refsdir + self._outdir = outdir + self.config = Config() + + def create(self): + html = "<html><body><a name='top'></a>" + if self.config.backends: + backends = [get_backend(name) for name in self.config.backends] + else: + backends = get_all_backends() + + results = {} + for root, dirs, files in os.walk(self._outdir, False): + if not files: + continue + if not root.lower().endswith('.pdf'): + continue + + results[root] = TestResult(self._docsdir, self._refsdir, self._outdir, root, files, backends) + + tests = results.keys() + tests.sort() + + failed_anchors = [] + failed = "" + crashed = "" + failed_to_run = "" + for test_name in tests: + test = results[test_name] + if test.is_failed(): + failed_anchors.append(test.get_test()) + failed += test.get_failed_html() + crashed += test.get_crashed_html() + failed_to_run += test.get_failed_to_run_html() + + if failed: + failed = "<h1><a name='failed'>Tests Failed (differences were found)</name></h1>\n%s" % (failed) + if crashed: + crashed = "<h1><a name='crashed'>Tests Crashed</a></h1>\n%s" % (crashed) + if failed_to_run: + failed_to_run = "<h1><a name='failed_to_run'>Tests that failed to run (command returned an error status)</a></h1>\n%s" % (failed_to_run) + + if failed or crashed or failed_to_run: + html += "<ul>\n" + if failed: + html += "<li><a href='#failed'>Tests Failed (differences were found)</a></li>\n<ul>" + for anchor in failed_anchors: + html += "<li><a href='#%s'>%s</a></li>" % (anchor, anchor) + html += "</ul>\n" + if crashed: + html += "<li><a href='#crashed'>Tests Crashed(differences were found)</a></li>\n" + if failed_to_run: + html += "<li><a href='#failed_to_run'>Tests that failed to run (command returned an error status)</a></li>\n" + html += "</ul>\n" + + html += failed + crashed + failed_to_run + "</body></html>" + + report_index = os.path.join(self._outdir, 'index.html') + f = open(report_index, 'wb') + f.write(html) + f.close() + + subprocess.Popen(['xdg-open', report_index]) diff --git a/regtest/backends/__init__.py b/regtest/backends/__init__.py index 77e5498..26be0b0 100644 --- a/regtest/backends/__init__.py +++ b/regtest/backends/__init__.py @@ -136,6 +136,21 @@ class Backend: return retval + def get_ref_names(self, refs_path): + retval = [] + md5_path = os.path.join(refs_path, self._name) + md5_file = open(md5_path + '.md5', 'r') + for line in md5_file.readlines(): + md5sum, ref_path = line.strip('\n').split(' ', 1) + basename = os.path.basename(ref_path) + if not self.__should_have_checksum(basename): + continue + + retval.append(basename) + md5_file.close() + + return retval + def has_md5(self, test_path): return os.path.exists(os.path.join(test_path, self._name + '.md5')) @@ -156,8 +171,11 @@ class Backend: def has_results(self, test_path): return self.has_md5(test_path) or self.is_crashed(test_path) or self.is_failed(test_path) + def get_stderr(self, test_path): + return os.path.join(test_path, self._name + '.stderr') + def has_stderr(self, test_path): - return os.path.exists(os.path.join(test_path, self._name + '.stderr')) + return os.path.exists(self.get_stderr(test_path)) def has_diff(self, test_result): if not self._diff_ext: diff --git a/regtest/commands/create-report.py b/regtest/commands/create-report.py new file mode 100644 index 0000000..89b6687 --- /dev/null +++ b/regtest/commands/create-report.py @@ -0,0 +1,57 @@ +# create-report.py +# +# Copyright (C) 2012 Carlos Garcia Campos <[email protected]> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +from commands import Command, register_command +from HTMLReport import HTMLReport +from Config import Config +import os +import tempfile + +class CreateReport(Command): + + name = 'create-report' + usage_args = '[ options ... ] tests ' + description = 'Create report of test results' + + def __init__(self): + Command.__init__(self) + parser = self._get_args_parser() + parser.add_argument('--refs-dir', + action = 'store', dest = 'refs_dir', default = os.path.join(tempfile.gettempdir(), 'refs'), + help = 'Directory containing the references') + parser.add_argument('-o', '--out-dir', + action = 'store', dest = 'out_dir', default = os.path.join(tempfile.gettempdir(), 'out'), + help = 'Directory containing the results') + parser.add_argument('tests') + + def run(self, options): + config = Config() + + doc = options['tests'] + if os.path.isdir(doc): + docs_dir = doc + else: + docs_dir = os.path.dirname(doc) + + report = HTMLReport(docs_dir, options['refs_dir'], options['out_dir']) + if doc == docs_dir: + report.create() + else: + report.create_for_file(os.path.basename(doc)) + +register_command('create-report', CreateReport) commit d5faabd509c2860ab199ee89b8ef9d4c14fa5118 Author: Carlos Garcia Campos <[email protected]> Date: Sat Feb 25 20:50:53 2012 +0100 regtest: Use diff.png extension for image diff files So that they are recognized as images diff --git a/regtest/backends/__init__.py b/regtest/backends/__init__.py index 47985fb..77e5498 100644 --- a/regtest/backends/__init__.py +++ b/regtest/backends/__init__.py @@ -33,13 +33,17 @@ class UnknownBackendError(Exception): class Backend: - def __init__(self, name): + def __init__(self, name, diff_ext = None): self._name = name + self._diff_ext = diff_ext self._utilsdir = Config().utils_dir def get_name(self): return self._name + def get_diff_ext(self): + return self._diff_ext + def __should_have_checksum(self, entry): if not entry.startswith(self._name): return False @@ -155,6 +159,14 @@ class Backend: def has_stderr(self, test_path): return os.path.exists(os.path.join(test_path, self._name + '.stderr')) + def has_diff(self, test_result): + if not self._diff_ext: + return False + basename = os.path.basename(test_result) + if not basename.startswith(self._name): + return False + return os.path.exists(test_result + self._diff_ext) + def __create_stderr_file(self, stderr, out_path): if not stderr: return @@ -216,7 +228,7 @@ class Backend: ref = Image.open(ref_path) result = Image.open(result_path) diff = ImageChops.difference(ref, result) - diff.save(result_path + '.diff', 'png') + diff.save(result_path + '.diff.png', 'png') def _create_diff(self, ref_path, result_path): raise NotImplementedError diff --git a/regtest/backends/cairo.py b/regtest/backends/cairo.py index a990ddf..304783e 100644 --- a/regtest/backends/cairo.py +++ b/regtest/backends/cairo.py @@ -23,7 +23,7 @@ import os class Cairo(Backend): def __init__(self, name): - Backend.__init__(self, name) + Backend.__init__(self, name, '.diff.png') self._pdftocairo = os.path.join(self._utilsdir, 'pdftocairo'); def create_refs(self, doc_path, refs_path): diff --git a/regtest/backends/splash.py b/regtest/backends/splash.py index 5fbfba6..3144bc7 100644 --- a/regtest/backends/splash.py +++ b/regtest/backends/splash.py @@ -23,7 +23,7 @@ import os class Splash(Backend): def __init__(self, name): - Backend.__init__(self, name) + Backend.__init__(self, name, '.diff.png') self._pdftoppm = os.path.join(self._utilsdir, 'pdftoppm'); def create_refs(self, doc_path, refs_path): diff --git a/regtest/backends/text.py b/regtest/backends/text.py index 14e7c03..10b660a 100644 --- a/regtest/backends/text.py +++ b/regtest/backends/text.py @@ -23,7 +23,7 @@ import os class Text(Backend): def __init__(self, name): - Backend.__init__(self, name) + Backend.__init__(self, name, '.diff') self._pdftotext = os.path.join(self._utilsdir, 'pdftotext'); def create_refs(self, doc_path, refs_path): _______________________________________________ poppler mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/poppler
