This adds a few tests for the dmesg class. It also adds a test hook into core.Test. This is a little ugly, but I just couldn't come up with any other solution that didn't involve making the dmesg code more invasive or harder to read.
V2 - Adds tests to simulate dmesg 'wrapping' Signed-off-by: Dylan Baker <[email protected]> --- framework/core.py | 5 + framework/tests/dmesg_tests.py | 245 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 250 insertions(+) create mode 100644 framework/tests/dmesg_tests.py diff --git a/framework/core.py b/framework/core.py index a193a2d..da2a716 100644 --- a/framework/core.py +++ b/framework/core.py @@ -417,6 +417,10 @@ class Test(object): self.runConcurrent = runConcurrent self.skip_test = False + # This is a hook for doing some testing on execute right before + # self.run is called. + self._test_hook_execute_run = lambda: None + def run(self): raise NotImplementedError @@ -436,6 +440,7 @@ class Test(object): try: status("running") time_start = time.time() + self._test_hook_execute_run() result = self.run(env) time_end = time.time() if 'time' not in result: diff --git a/framework/tests/dmesg_tests.py b/framework/tests/dmesg_tests.py new file mode 100644 index 0000000..869d81a --- /dev/null +++ b/framework/tests/dmesg_tests.py @@ -0,0 +1,245 @@ +# Copyright (c) 2014 Intel Corporation + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" Provides tests for the dmesg class """ + +import os +import subprocess +from nose.plugins.skip import SkipTest +from framework.dmesg import DummyDmesg, PosixDmesg, get_dmesg +from framework.core import TestResult, PiglitJSONEncoder, Environment +from framework.exectest import PlainExecTest +from framework.gleantest import GleanTest +from framework.shader_test import ShaderTest +from framework.glsl_parser_test import GLSLParserTest + + +def _get_dmesg(): + """ checks to ensure dmesg is not DummyDmesg, raises skip if it is + + If we are on a non-posix system we will get a dummy dmesg, go ahead and + skip in that case + """ + test = get_dmesg() + if isinstance(test, DummyDmesg): + raise SkipTest("A DummyDmesg was returned, testing dmesg impossible.") + return test + + +def _write_dev_kmesg(): + """ Try to write to /dev/kmesg, skip if not possible + + Writing to the dmesg ringbuffer at /dev/kmesg ensures that we varies + functionality in the PosixDmesg class will go down the + dmesg-has-new-entries path. + + If we anything goes wrong here just skip. + """ + err = subprocess.call(['sudo', 'sh', '-c', 'echo "piglit dmesg test" > /dev/kmsg']) + if err != 0: + raise SkipTest("Writing to the ringbuffer failed") + + +def test_get_dmesg_dummy(): + """ Test that get_dmesg function returns a Dummy when asked """ + dummy = get_dmesg(not_dummy=False) + assert isinstance(dummy, DummyDmesg) + + +def test_get_dmesg_posix(): + """ Test that get_dmesg function returns a Posix when asked """ + assert isinstance(_get_dmesg(), PosixDmesg) + + +def test_update_dmesg(): + """ Tests that update_dmesg actually updates + + This will automatically pass on a non-posix systems, since there is no way + to actually test this. + + Because this test needs to write into the dmesg ringbuffer to assure that + the ringbuffer has changed and that our class successfully catches that + change it requires root access, gained by sudo. In the event that it cannot + get sudo it will skip. + + """ + dmesg = _get_dmesg() + _write_dev_kmesg() + + dmesg.update_dmesg() + assert(dmesg._new_messages) + + +def test_dmesg_wrap_partial(): + """ Test that dmesg still works after dmesg wraps partially + + We can overwrite the DMESG_COMMAND class variable to emluate dmesg being + filled up and overflowing. What this test does is starts with a string that + looks like this: "a\nb\nc\n" (this is used to emluate the contents of + dmesg), we then replace that with "b\nc\nd\n", and ensure that the update + of dmesg contains only 'd'. + + """ + # We don't want weird side effects of changing DMESG_COMMAND globally, so + # instead we set it as a class instance and manually clear the + # _last_messages attribute + dmesg = PosixDmesg() + dmesg.DMESG_COMMAND = ['echo', 'a\nb\nc\n'] + dmesg._last_messages = [] + dmesg.update_dmesg() + + # Udamte the DMESG_COMMAND to add d\n and remove a\n, this simluates the + # wrap + dmesg.DMESG_COMMAND = ['echo', 'b\nc\nd\n'] + dmesg.update_dmesg() + + print dmesg._new_messages + assert(dmesg._new_messages == ['d']) + + +def test_dmesg_wrap_complete(): + """ Test that dmesg still works after dmesg wraps completely + + just like the partial version, but with nothingin common. + + """ + # We don't want weird side effects of changing DMESG_COMMAND globally, so + # instead we set it as a class instance and manually clear the + # _last_messages attribute + dmesg = PosixDmesg() + dmesg.DMESG_COMMAND = ['echo', 'a\nb\nc\n'] + dmesg._last_messages = [] + dmesg.update_dmesg() + + # Udamte the DMESG_COMMAND to add d\n and remove a\n, this simluates the + # wrap + dmesg.DMESG_COMMAND = ['echo', '1\n2\n3\n'] + dmesg.update_dmesg() + + print dmesg._new_messages + assert(dmesg._new_messages == ['1', '2', '3']) + + +def test_update_result_replace(): + """ Generates tests for update_result """ + dmesg = _get_dmesg() + + for res in ['pass', 'fail', 'crash', 'warn', 'skip', 'notrun']: + result = TestResult() + result['result'] = res + result['subtest'] = {} + result['subtest']['test'] = res + + _write_dev_kmesg() + new_result = dmesg.update_result(result) + + # Create a yieldable and set the description for useful per-test names + yieldable = check_update_result + yieldable.description = "Test update_result: {0}".format(res) + yield yieldable, new_result['result'], res + + yieldable.description = "Test update_result subtest: {0}".format(res) + yield yieldable, new_result['subtest']['test'], res + + +def check_update_result(result, status): + """ Tests that update result replaces results correctly + + Dmesg.update_results() should take a TestResult instance and replace the + result instance with a dmesg-statuses when appropriate. Appropriate + instances to be replaced are: pass, warn, fail. + + """ + if status == "pass": + assert result == 'dmesg-warn' + elif status in ['warn', 'fail']: + assert result == 'dmesg-fail' + else: + assert result == status + + +def test_update_result_add_dmesg(): + """ Tests update_result's addition of dmesg attribute """ + dmesg = _get_dmesg() + + result = TestResult() + result['result'] = 'pass' + + _write_dev_kmesg() + _ = dmesg.update_result(result) + + assert 'dmesg' in result + return result + + +def test_json_serialize_updated_result(): + """ Test that a TestResult that has been updated is json serializable """ + encoder = PiglitJSONEncoder() + result = test_update_result_add_dmesg() + encoded = encoder.encode(result) + + +def test_testclasses_dmesg(): + """ Generator that creates tests for """ + env = Environment() + + lists = [(PlainExecTest, ['attribs', '-auto', '-fbo'], 'PlainExecTest'), + (GleanTest, 'basic', "GleanTest"), + (ShaderTest, + ['tests/shaders/loopfunc.shader_test', '-auto', '-fbo'], + 'ShaderTest'), + (GLSLParserTest, 'tests/glslparsertest/shaders/main1.vert', + 'GLSLParserTest')] + + for tclass, tfile, desc in lists: + yieldable = check_classes_dmesg + yieldable.description = "Test dmesg in {}".format(desc) + yield yieldable, tclass, tfile, env + + +def check_classes_dmesg(test_class, test_args, env): + """ Do the actual check on the provided test class for dmesg """ + if not os.path.exists('bin'): + raise SkipTest("This tests requires a working, built version of " + "piglit") + + class DummyJsonWriter(): + """ A very simple dummy for json writer """ + def __init__(self): + pass + + def write_dict_item(self, path, result): + self.result = result + + def _write_dmesg(): + """ Small helper to write dmesg """ + subprocess.call(['sudo', 'sh', '-c', 'echo "piglit test" > /dev/kmsg']) + + dmesg = _get_dmesg() + + # Create the test and then write to dmesg to ensure that it actually works + test = test_class(test_args) + test._test_hook_execute_run = _write_dmesg + + json = DummyJsonWriter() + + test.execute(env, None, json, dmesg) + + assert(json.result['result'] in ['dmesg-warn', 'dmesg-fail']) -- 1.8.5.3 _______________________________________________ Piglit mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/piglit
