This patch converts all of the status_tests.py into nose style tests. Nose makes everything we bent over backwards for with unittest easy, reducing code in both amount and complexity.
Signed-off-by: Dylan Baker <[email protected]> --- framework/tests/helpers.py | 105 ---------------- framework/tests/status.py | 262 ---------------------------------------- framework/tests/status_tests.py | 88 ++++++++++++++ piglit-framework-tests.py | 47 ------- 4 files changed, 88 insertions(+), 414 deletions(-) delete mode 100644 framework/tests/helpers.py delete mode 100644 framework/tests/status.py create mode 100644 framework/tests/status_tests.py delete mode 100755 piglit-framework-tests.py diff --git a/framework/tests/helpers.py b/framework/tests/helpers.py deleted file mode 100644 index 2fd7c3d..0000000 --- a/framework/tests/helpers.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2013 Intel Corporation -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice (including the next -# paragraph) shall be included in all copies or substantial portions of the -# Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - - -import sys - -import framework.core as core - - -def test_iterations(*parameters): - """ Magic that allows a single method to create a whole bunch of functions. - - This is desirable because of the way unittest works: Each method gets a - name in the output, and each method stops on the first error. This makes - using a loop useless, if 10/20 iterations should fail, the first failure - stops the loop. The solution other than using a decorator is to create a - test for each iteration, which could result in hundreds of tests. - - """ - - def decorator(method, parameters=parameters): - def tuplify(x): - if not isinstance(x, tuple): - return (x, ) - return x - - for parameter in map(tuplify, parameters): - name_for_parameter = "{0}({1})".format(method.__name__, - ", ".join(map(repr, parameter))) - frame = sys._getframe(1) # pylint: disable-msg=W0212 - frame.f_locals[name_for_parameter] = \ - lambda self, m=method, p=parameter: m(self, *p) - return None - return decorator - - -def create_testresult(name, lspci="fake lspci", glxinfo="fake glxinfo", - tests={}): - """ Helper function that returns a complete TestResults file - - This takes one required argument - :name: This must be set, it names the run. If two runs have the same name - there can be problems in the summary code. - - This function also takes three optional arguments - :lspci: This is the lspci information in the file. Default="fake lspci" - :glxinfo: glxinfo in the file. Default="fake glxinfo" - :tests: A dictionary of tests - - """ - assert(isinstance(tests, dict)) - - return core.TestResult({"options": {"profile": "fake", - "filter": [], - "exclude_filter": []}, - "name": "{0}".format(name), - "lspci": "{0}".format(lspci), - "glxinfo": "{0}".format(glxinfo), - "time_elapsed": 10.23456789, - "tests": tests}) - - -def create_test(name, result, info="fake info", returncode=0, time=0.123456789, - command="fake command"): - """ Helper function that takes input and returns a dictionary of results - for a single tests - - Takes two required arguments: - :name: The name of the tests - :result: The result the test returned - - Additionally takes four optional arguments: - :info: The info entry. Default="fake info" - :returncode: The returncode of the test. Default=0 - :time: The amount of time the test ran. Default=0.123456789 - :command: The command that was executed. Default="fake command" - - """ - #FIXME: This should be able to create other entries for failed tests - - return {name: {"info": info, - "returncode": returncode, - "time": time, - "command": command, - "result": result}} diff --git a/framework/tests/status.py b/framework/tests/status.py deleted file mode 100644 index 3630abe..0000000 --- a/framework/tests/status.py +++ /dev/null @@ -1,262 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2013 Intel Corporation -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice (including the next -# paragraph) shall be included in all copies or substantial portions of the -# Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - - -import os -import os.path as path -import unittest -import itertools -import tempfile - -try: - import simplejson as json -except ImportError: - import json -import framework.summary as summary -from helpers import test_iterations, create_testresult, create_test - - -""" Status ordering from best to worst: - -See ../summary.py. - -""" - -# These lists normally would reside within a class, however, these are meant to -# be fed to decorators, which donot have access to class variables. - -# List of possible statuses. -# 'notrun' must be last or all kinds of things might break. -STATUSES = ['pass', 'fail', 'skip', 'warn', 'crash', 'dmesg-warn', - 'dmesg-fail', 'notrun'] - -# list of possible regressions -REGRESSIONS = [("pass", "warn"), - ("pass", "dmesg-warn"), - ("pass", "fail"), - ("pass", "dmesg-fail"), - ("pass", "crash"), - ("dmesg-warn", "warn"), - ("dmesg-warn", "dmesg-fail"), - ("dmesg-warn", "fail"), - ("dmesg-warn", "crash"), - ("warn", "fail"), - ("warn", "crash"), - ("warn", "dmesg-fail"), - ("dmesg-fail", "crash"), - ("dmesg-fail", "fail"), - ("fail", "crash"), - ("skip", "crash"), - ("skip", "fail"), - ("skip", "dmesg-fail"), - ("skip", "warn"), - ("skip", "dmesg-warn"), - ("notrun", "crash"), - ("notrun", "fail"), - ("notrun", "dmesg-fail"), - ("notrun", "warn"), - ("notrun", "dmesg-warn")] - - -# List of possible fixes -FIXES = [("crash", "fail"), - ("crash", "dmesg-fail"), - ("crash", "warn"), - ("crash", "dmesg-warn"), - ("crash", "pass"), - ("crash", "skip"), - ("crash", "notrun"), - ("fail", "dmesg-fail"), - ("fail", "warn"), - ("fail", "dmesg-warn"), - ("fail", "pass"), - ("fail", "skip"), - ("fail", "notrun"), - ("dmesg-fail", "warn"), - ("dmesg-fail", "dmesg-warn"), - ("dmesg-fail", "pass"), - ("dmesg-fail", "skip"), - ("dmesg-fail", "notrun"), - ("warn", "dmesg-warn"), - ("warn", "pass"), - ("warn", "skip"), - ("warn", "notrun"), - ("dmesg-warn", "pass"), - ("dmesg-warn", "skip"), - ("dmesg-warn", "notrun")] - - -# List of statuses that should be problems. -PROBLEMS = ["crash", "fail", "warn", "dmesg-warn", "dmesg-fail"] - - -class SummaryTestBase(unittest.TestCase): - """ Abstract base class for testing the Summary module""" - tmpdir = tempfile.mkdtemp("piglit") - files = [] - - @classmethod - def tearDownClass(cls): - """ Remove any folders which were created and are empty """ - for branch in cls.files: - os.remove(branch) # Delete the fie itself - while branch != '/': - branch = path.dirname(branch) - try: - os.rmdir(branch) - except OSError as exception: - # If there is an OSError the directory is not empty, so - # every additional attempt to call os.rmdir will fail. - if exception.errno == 39: - break - - @classmethod - def _makedirs(cls, dirs): - """ Create one more more folders, with some error checking - - This catches a "Directory Exists" error (OSError.errno == 17), and - passes that; but any other error will be raised - - """ - try: - os.makedirs(dirs) - except OSError as exception: - # os.mkdir (and by extension os.makedirs) quite annoyingly throws - # an error. - - # if the directory exists if the error was not a "Directory Exists" - # Error, go ahead and raise it. - if exception.errno != 17: - raise exception - - # you cannot pass in an if clause, since that would pass the if, - # not the except - pass - - def _generate_summary(self, files): - return summary.Summary([path.join(self.tmpdir, i) for i in files]) - - -class StatusTest(SummaryTestBase): - """ Test status comparisons in the Summary module. - - This test creates summary objects with one or two results that it - generates. These results have a single test in them forming known status - pairs. These pairs are explicitly a fix, regression, or change. The code - then creates a summary and ensures that the status has been added to the - proper field, and not to any additional fields. - - """ - - @classmethod - def setUpClass(cls): - """ Create all of the necissary files for running the test - - Create a set of temprorary result files, and add them to the files - attribute. This attribute will be deleted by the tearDownClass() method - after the tests have run - - """ - cls._makedirs(cls.tmpdir) - - # Create temporary files in the /tmp/piglit directory - for result in STATUSES[:-1]: # notrun cannot be generated here - tmpfile = path.join(cls.tmpdir, result, "main") - cls._makedirs(path.dirname(tmpfile)) - - with open(tmpfile, 'w') as f: - json.dump(create_testresult(name=result, - tests=create_test("test", result)), - f, indent=4) - - # Add the file to the list of files to be removed in tearDownClass - cls.files.append(tmpfile) - - # Create an additional file to test a Not Run status. This needs to be - # generated separately since the test result should not include the - # test that will have the not run status. - tmpfile = path.join(cls.tmpdir, "notrun", "main") - cls._makedirs(path.dirname(tmpfile)) - - with open(tmpfile, 'w') as f: - # If there are not tests buildDictionary fails - json.dump(create_testresult(name="notrun", - tests=create_test("diff", "pass")), - f, indent=4) - - cls.files.append(tmpfile) - - @test_iterations(*REGRESSIONS) - def test_is_regression(self, x, y): - """ Only explicitly defined regressions should be treated as such - - Test that all combinations that are not explicitly a regression are not - treated as regressions - - """ - result = self._generate_summary([x, y]) - self.assertEqual( - len(result.tests['regressions']), 1, - "{0} -> {1} is not a regression but should be".format(x, y)) - - @test_iterations(*FIXES) - def test_is_fix(self, x, y): - """ Statuses that are explicitly defined as fixes should be fixes - - Test that ensures that all tests in the FIXES list are treated as fixes - - """ - result = self._generate_summary([x, y]) - self.assertEqual( - len(result.tests['fixes']), 1, - "{0} -> {1} is not a fix but should be".format(x, y)) - - #XXX: is "notrun" -> not(notrun) a change? - @test_iterations(*[i for i in itertools.product(STATUSES[:-1], STATUSES[:-1]) - if i[0] != i[1]]) - def test_is_change(self, x, y): - """ Check that all changes are treated as changes - - Given two statuses if status1 != status2 then that should be a change - - """ - result = self._generate_summary([x, y]) - self.assertEqual( - len(result.tests['changes']), 1, - "{0} -> {1} is a not a change but should be".format(x, y)) - - @test_iterations(*PROBLEMS) - def test_is_problem(self, x): - """ Only statuses in the PROBLEMS list should be added to problems """ - result = self._generate_summary([x]) - self.assertEqual( - len(result.tests['problems']), 1, - "{0} is not a problem but should be".format(x)) - - @test_iterations("skip") - def test_is_skip(self, x): - """ Ensure that skip is being added to the skip list """ - result = self._generate_summary([x]) - self.assertEqual( - len(result.tests['skipped']), 1, - "Skip is not being added to the list of skips") diff --git a/framework/tests/status_tests.py b/framework/tests/status_tests.py new file mode 100644 index 0000000..a15bec1 --- /dev/null +++ b/framework/tests/status_tests.py @@ -0,0 +1,88 @@ +# Copyright (c) 2014 Intel Corperation + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" Tests for the Status module + +Note: see framework/status.py for the authoritative list of fixes, regression, +etc + +""" + +import itertools +import framework.status as status + +# Statuses from worst to last. NotRun is intentionally not in this list and +# tested separately because of upcoming features for it +STATUSES = ["notrun", "pass", "dmesg-warn", "warn", "dmesg-fail", "fail", + "crash", "timeout"] + +# Create lists of fixes and regressions programmatically based on the STATUSES +# list. This means less code, and easier expansion changes. +REGRESSIONS = itertools.combinations(STATUSES, 2) +FIXES = itertools.combinations(reversed(STATUSES), 2) + +# all statuses except pass are problems +PROBLEMS = STATUSES[1:] + + +def is_regression(x, y): + # Test for regressions + assert status.status_lookup(x) < status.status_lookup(y) + + +def is_fix(x, y): + # Test for fix + assert status.status_lookup(x) > status.status_lookup(y) + + +def is_equivalent(x, y): + # Test if status is equivalent. Note that this does not mean 'same', two + # statuses could be equivalent in terms of fixes and regressions, but that + # doesn't require that they are the same status + assert status.status_lookup(x) == status.status_lookup(y) + + +def is_not_equivalent(x, y): + # Test that status is not equivalent. + assert status.status_lookup(x) != status.status_lookup(y) + + +def test_is_regression(): + # Generate all tests for regressions + for x, y in REGRESSIONS: + yield is_regression, x, y + + +def test_is_fix(): + # Generates all tests for fixes + for x, y in FIXES: + yield is_fix, x, y + + +def test_is_equivalent(): + # test the assertion that NotRun, Pass and Skip should be considered + # equivalent for regression testing. + for x, y in itertools.izip(STATUSES, STATUSES): + yield is_equivalent, x, y + + +def test_is_change(): + for x, y in itertools.permutations(STATUSES, 2): + yield is_not_equivalent, x, y diff --git a/piglit-framework-tests.py b/piglit-framework-tests.py deleted file mode 100755 index aa4d9e6..0000000 --- a/piglit-framework-tests.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) 2013 Intel Corporation -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice (including the next -# paragraph) shall be included in all copies or substantial portions of the -# Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import argparse -import unittest - -import framework.tests.status - -# Create a dictionary of all of tests. Do this before the parser so we can use -# it as a list of optional arguments for the parser -tests = {"status": unittest.TestLoader().loadTestsFromModule(framework.tests.status)} - -parser = argparse.ArgumentParser() -parser.add_argument("tests", - action="append", - choices=tests.keys(), - help="Testing profiles for the framework") -parser.add_argument("-v", "--verbose", - action="store", - choices=['0', '1', '2'], - default='1', - help="Set the level of verbosity to run tests at") -args = parser.parse_args() - -# Run the tests -map(unittest.TextTestRunner(verbosity=int(args.verbose)).run, - [tests[x] for x in args.tests]) -- 1.8.5.3 _______________________________________________ Piglit mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/piglit
