Currently, if a test from provided testlist fails to be discovered by the framework, piglit blows up with an exception.
This is both good - for consistency/early errors - and bad - for handling some CI/automation scenarios (e.g autobisecting the tests). So let's keep the current default, but allow some flexibility with the new option that reports the missing tests as 'notrun'. v2: - change switch name to 'ignore', as skip is too suggestive - use DummyTest to get 'notrun' result instead of warnings bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=99649 Cc: Dylan Baker <[email protected]> Cc: Tomi Sarvela <[email protected]> Cc: Martin Peres <[email protected]> Tested-by: Petri Latvala <[email protected]> Signed-off-by: Arkadiusz Hiler <[email protected]> --- framework/profile.py | 10 +++++++--- framework/programs/run.py | 6 ++++++ framework/test/base.py | 12 ++++++++++++ 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/framework/profile.py b/framework/profile.py index a625318..bb6c39e 100644 --- a/framework/profile.py +++ b/framework/profile.py @@ -42,11 +42,12 @@ import re import six -from framework import grouptools, exceptions +from framework import grouptools, exceptions, status from framework.dmesg import get_dmesg from framework.log import LogManager from framework.monitoring import Monitoring -from framework.test.base import Test +from framework.test.base import Test, DummyTest +from framework.options import OPTIONS __all__ = [ 'RegexFilter', @@ -314,7 +315,10 @@ class TestProfile(object): if self.forced_test_list: opts = collections.OrderedDict() for n in self.forced_test_list: - opts[n] = self.test_list[n] + if OPTIONS.ignore_missing and n not in self.test_list: + opts[n] = DummyTest(name=n, result=status.NOTRUN) + else: + opts[n] = self.test_list[n] else: opts = self.test_list # pylint: disable=redefined-variable-type diff --git a/framework/programs/run.py b/framework/programs/run.py index 6444cfe..d30e6a4 100644 --- a/framework/programs/run.py +++ b/framework/programs/run.py @@ -208,6 +208,10 @@ def _run_parser(input_): 'isolation. This allows, but does not require, ' 'tests to run multiple tests per process. ' 'This value can also be set in piglit.conf.') + parser.add_argument("--ignore-missing", + dest="ignore_missing", + action="store_true", + help="missing tests are considered as 'notrun'") parser.add_argument("test_profile", metavar="<Profile path(s)>", nargs='+', @@ -291,6 +295,7 @@ def run(input_): options.OPTIONS.sync = args.sync options.OPTIONS.deqp_mustpass = args.deqp_mustpass options.OPTIONS.process_isolation = args.process_isolation + options.OPTIONS.ignore_missing = args.ignore_missing # Set the platform to pass to waffle options.OPTIONS.env['PIGLIT_PLATFORM'] = args.platform @@ -389,6 +394,7 @@ def resume(input_): options.OPTIONS.sync = results.options['sync'] options.OPTIONS.deqp_mustpass = results.options['deqp_mustpass'] options.OPTIONS.proces_isolation = results.options['process_isolation'] + options.OPTIONS.ignore_missing = results.options['ignore_missing'] core.get_config(args.config_file) diff --git a/framework/test/base.py b/framework/test/base.py index d73dee9..e74ea3d 100644 --- a/framework/test/base.py +++ b/framework/test/base.py @@ -381,6 +381,18 @@ class Test(object): return not self == other +class DummyTest(Test): + def __init__(self, name, result): + super(DummyTest, self).__init__([name]) + self.result.result = result + + def execute(self, path, log, options): + pass + + def interpret_result(self): + pass + + class WindowResizeMixin(object): """ Mixin class that deals with spurious window resizes -- 2.9.4 _______________________________________________ Piglit mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/piglit
