On Friday, June 20, 2014 07:33:29 PM Ilia Mirkin wrote: > Just tested your updated branch, I now see > > 4539/4888 instead of 4444/4768 > > for the file I referenced. I think the doubling is still happening, for > example: > > spec/EXT_framebuffer_object/fbo-readpixels-depth-formats/GL_DEPTH24_STENCIL8 > _EXT/GL_DEPTH24_STENCIL8_EXT/GL_UNSIGNED_SHORT > > IIRC this is one of the stupid ones with *2* levels in the subtest > name (that's why I had that extra-lame hack in my version of splitting > things up).
gah. I'll have a look at two level subtests then. > > On Fri, Jun 20, 2014 at 6:22 PM, Dylan Baker <[email protected]> wrote: > > This patch updates our json to version 1. Changes from version 0 to > > version 1 are as follows: > > > > - renamed 'main' to 'results.json' > > - dmesg must be a string (It was stored a list in some version 0 > > > > results) > > > > - subtests are never stored as duplicate entries, a single instance of > > > > the test is recorded, (In version 0 both are possible) > > > > - there is no info entry in version 1, err, out, and returncode are > > > > always split into seperate entries > > > > This patch adds support to the results module for handling updates to > > the results in a sane way. It does this by adding a result_version > > attribute to the TestrunResult (which is stored as json), and > > implementing the ability to incrementally update results between > > versions. > > > > It does this automatically on load, non-destructively, moving the old > > results to results.json.old, but does write the updated results to disk, > > making the cost of this update a one time cost. > > > > v2: - Handle non-writable directories and files, this also fixes using > > > > file-descriptors as inputs > > > > v4: - Fix bug that caused update_results to always return early > > > > - Fix bug that caused tests with one subtest to have their name > > > > mangled > > > > - Set TestrunResult.results_version by default, set it to None > > > > during load if the results file doesn't have a results_vesrion > > value. Otherwise results could end up with no version when they > > should have one > > > > - Fix a bug that raises an uncaught exception when a test's info has > > > > more elements than the expected 3 (mostly glean) > > > > - split the version 0 to 1 tests out of the results_tests module. > > > > There are enough tests and enough data specific to these tests it > > makes more sense to make a separate test module > > > > Signed-off-by: Dylan Baker <[email protected]> > > --- > > > > framework/programs/run.py | 4 +- > > framework/results.py | 154 > > ++++++++++++++++++++++++++++++++--- framework/tests/results_tests.py > > | 17 ++-- > > framework/tests/results_v0_tests.py | 157 > > ++++++++++++++++++++++++++++++++++++ framework/tests/utils.py > > | 2 + > > 5 files changed, 318 insertions(+), 16 deletions(-) > > create mode 100644 framework/tests/results_v0_tests.py > > > > diff --git a/framework/programs/run.py b/framework/programs/run.py > > index 9255f64..ff13974 100644 > > --- a/framework/programs/run.py > > +++ b/framework/programs/run.py > > > > @@ -157,7 +157,7 @@ def run(input_): > > results.name = path.basename(args.results_path) > > > > # Begin json. > > > > - result_filepath = path.join(args.results_path, 'main') > > + result_filepath = path.join(args.results_path, 'results.json') > > > > result_file = open(result_filepath, 'w') > > json_writer = framework.results.JSONWriter(result_file) > > > > @@ -212,7 +212,7 @@ def resume(input_): > > if results.options.get('platform'): > > env.env['PIGLIT_PLATFORM'] = results.options['platform'] > > > > - results_path = path.join(args.results_path, "main") > > + results_path = path.join(args.results_path, 'results.json') > > > > json_writer = framework.results.JSONWriter(open(results_path, 'w+')) > > json_writer.initialize_json(results.options, results.name, > > > > core.collect_system_info()) > > > > diff --git a/framework/results.py b/framework/results.py > > index 1edc423..4ac2f37 100644 > > --- a/framework/results.py > > +++ b/framework/results.py > > @@ -23,6 +23,7 @@ > > > > from __future__ import print_function > > import os > > > > +import sys > > > > from cStringIO import StringIO > > > > try: > > import simplejson as json > > > > @@ -39,6 +40,9 @@ __all__ = [ > > > > 'load_results', > > > > ] > > > > +# The current version of the JSON results > > +CURRENT_JSON_VERSION = 1 > > + > > > > def _piglit_encoder(obj): > > """ Encoder for piglit that can transform additional classes into > > json > > > > @@ -119,7 +123,7 @@ class JSONWriter(object): > > self.__is_collection_empty = [] > > > > def initialize_json(self, options, name, env): > > - """ Write boilerplate json code > > + """ Write boilerplate json code > > > > This writes all of the json except the actuall tests. > > > > @@ -132,6 +136,7 @@ class JSONWriter(object): > > """ > > self.open_dict() > > > > + self.write_dict_item('results_version', CURRENT_JSON_VERSION) > > > > self.write_dict_item('name', name) > > > > self.write_dict_key('options') > > > > @@ -229,6 +234,7 @@ class TestrunResult(object): > > 'wglinfo', > > 'glxinfo', > > 'lspci', > > > > + 'results_version', > > > > 'time_elapsed'] > > > > self.name = None > > self.uname = None > > > > @@ -236,6 +242,7 @@ class TestrunResult(object): > > self.glxinfo = None > > self.lspci = None > > self.time_elapsed = None > > > > + self.results_version = CURRENT_JSON_VERSION > > > > self.tests = {} > > > > if resultfile: > > @@ -246,6 +253,12 @@ class TestrunResult(object): > > except ValueError: > > raw_dict = json.load(self.__repair_file(resultfile)) > > > > + # If the results don't have a results version set it to None, > > + # otherwise they will pass through the update_results > > function > > + # without being updated > > + if not getattr(raw_dict, 'results_version', False): > > + self.results_version = None > > + > > > > # Check that only expected keys were unserialized. > > > > for key in raw_dict: > > if key not in self.serialized_keys: > > @@ -338,12 +351,135 @@ def load_results(filename): > > "main" > > > > """ > > > > + # This will load any file or file-like thing. That would include > > pipes and + # file descriptors > > + if not os.path.isdir(filename): > > + filepath = filename > > + else: > > + # If there are both old and new results in a directory pick the > > new + # ones first > > + if os.path.exists(os.path.join(filename, 'results.json')): > > + filepath = os.path.join(filename, 'results.json') > > + # Version 0 results are called 'main' > > + elif os.path.exists(os.path.join(filename, 'main')): > > + filepath = os.path.join(filename, 'main') > > + else: > > + raise Exception("No results found") > > + > > + with open(filepath, 'r') as f: > > + testrun = TestrunResult(f) > > + > > + return update_results(testrun, filepath) > > + > > + > > +def update_results(results, filepath): > > + """ Update results to the lastest version > > + > > + This function is a wraper for other update_* functions, providing > > + incremental updates from one version to another. > > + > > + """ > > + # If the results version is the current version there is no need to > > + # update, just return the results > > + if getattr(results, 'results_version', 0) == CURRENT_JSON_VERSION: > > + return results > > + > > + # If there is no version then the results are version 0, and need to > > be + # updated to version 1 > > + if getattr(results, 'results_version') is None: > > + results = _update_zero_to_one(results) > > + > > + # Move the old results, and write the current results > > + filedir = os.path.dirname(filepath) > > > > try: > > - with open(filename, 'r') as resultsfile: > > - testrun = TestrunResult(resultsfile) > > - except IOError: > > - with open(os.path.join(filename, "main"), 'r') as resultsfile: > > - testrun = TestrunResult(resultsfile) > > - > > - assert testrun.name is not None > > - return testrun > > + os.rename(filepath, os.path.join(filedir, 'results.json.old')) > > + results.write(os.path.join(filedir, 'results.json')) > > + except OSError: > > + print("WARNING: Could not write updated results > > {}".format(filepath), + file=sys.stderr) > > + > > + return results > > + > > + > > +def _update_zero_to_one(results): > > + """ Update version zero results to version 1 results > > + > > + Changes from version 0 to version 1 > > + > > + - dmesg is sometimes stored as a list, sometimes stored as a string. > > In + version 1 it is always stored as a string > > + - in version 0 subtests are somtimes stored as duplicates, sometimes > > stored + only with a single entry, in version 1 tests with subtests > > are only + recorded once, always. > > + - Version 0 can have an info entry, or returncode, out, and err > > entries, + Version 1 will only have the latter > > + - version 0 results are called 'main', while version 1 results are > > called + 'results.json' (This is not handled internally, it's either > > handled by + update_results() which will write the file back to > > disk, or needs to be + handled manually by the user) > > + > > + """ > > + updated_results = {} > > + remove = set() > > + > > + for name, test in results.tests.iteritems(): > > + # fix dmesg errors if any > > + if isinstance(test.get('dmesg'), list): > > + test['dmesg'] = '\n'.join(test['dmesg']) > > + > > + # If a test as an info attribute, we want to remove it, if it > > doesn't + # have a returncode, out, or attribute we'll want to get > > those out of + # info first > > + if (None in [test.get('out'), test.get('err'), > > test.get('returncode')] + and test.get('info')): > > + # Some tests put even more stuff in 'info', which means it > > cannot + # be exploded (well, not in python2 anyway). > > + # the values of split will be: returncode, stderr, stdout, > > *extra + split = test['info'].split('\n\n') > > + > > + # returncode can be 0, and 0 is falsy, so ensure it is > > actually + # None > > + if test.get('returncode') is None: > > + # In some cases the returncode might not be set (like the > > test + # skipped), in that case it will be None, so set it > > + # apropriately > > + try: > > + test['returncode'] = int(split[0].split()[1].strip()) > > + except ValueError: > > + test['returncode'] = None > > + if not test.get('err'): > > + test['err'] = split[1].strip()[7:] > > + if not test.get('out'): > > + test['out'] = split[2].strip()[7:] > > + > > + # Remove the unused info key > > + if test.get('info'): > > + del test['info'] > > + > > + # Walk through the list of tests, if any of them contain > > duplicate > > + # subtests, add those subtests to the remove set, and then write > > a > > + # single test result to the update_results dictionary, which will > > be + # merged into results > > + # > > + # this must be the last thing done in this loop, or there will be > > pain + if test.get('subtest'): > > + remove.add(name) > > + > > + # only one entry was written (with the correct name) if there > > was + # only one subtest. > > + if len(test['subtest']) > 1: > > + testname = os.path.dirname(name) > > + else: > > + testname = name > > + > > + if testname not in updated_results.iterkeys(): > > + updated_results[testname] = test > > + > > + for name in remove: > > + del results.tests[name] > > + results.tests.update(updated_results) > > + > > + # set the results version > > + results.results_version = 1 > > + > > + return results > > diff --git a/framework/tests/results_tests.py > > b/framework/tests/results_tests.py index b31c505..73594ac 100644 > > --- a/framework/tests/results_tests.py > > +++ b/framework/tests/results_tests.py > > > > @@ -67,21 +67,28 @@ def test_initialize_jsonwriter(): > > assert isinstance(func, results.JSONWriter) > > > > -def test_load_results_folder(): > > > > +def test_load_results_folder_as_main(): > > """ Test that load_results takes a folder with a file named main in > > it """ > > > > with utils.tempdir() as tdir: > > with open(os.path.join(tdir, 'main'), 'w') as tfile: > > tfile.write(json.dumps(utils.JSON_DATA)) > > > > - results_ = results.load_results(tdir) > > - assert results_ > > + results.load_results(tdir) > > + > > + > > +def test_load_results_folder(): > > + """ Test that load_results takes a folder with a file named > > results.json """ + with utils.tempdir() as tdir: > > + with open(os.path.join(tdir, 'results.json'), 'w') as tfile: > > + tfile.write(json.dumps(utils.JSON_DATA)) > > + > > + results.load_results(tdir) > > > > def test_load_results_file(): > > """ Test that load_results takes a file """ > > > > with utils.resultfile() as tfile: > > - results_ = results.load_results(tfile.name) > > - assert results_ > > + results.load_results(tfile.name) > > > > def test_testresult_to_status(): > > diff --git a/framework/tests/results_v0_tests.py > > b/framework/tests/results_v0_tests.py new file mode 100644 > > index 0000000..c05583a > > --- /dev/null > > +++ b/framework/tests/results_v0_tests.py > > @@ -0,0 +1,157 @@ > > +# Copyright (c) 2014 Intel Corporation > > + > > +# Permission is hereby granted, free of charge, to any person obtaining a > > copy +# of this software and associated documentation files (the > > "Software"), to deal +# in the Software without restriction, including > > without limitation the rights +# to use, copy, modify, merge, publish, > > distribute, sublicense, and/or sell +# copies of the Software, and to > > permit persons to whom the Software is +# furnished to do so, subject to > > the following conditions: > > + > > +# The above copyright notice and this permission notice shall be included > > in +# all copies or substantial portions of the Software. > > + > > +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS > > OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF > > MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. > > IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY > > CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, > > TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE > > SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. > > + > > +""" Module provides tests for converting version zero results to version > > 1 """ + > > +import json > > +import copy > > +import framework.results as results > > +import framework.tests.utils as utils > > + > > + > > +DATA = copy.deepcopy(utils.JSON_DATA) > > +DATA['tests']['sometest']['dmesg'] = ['this', 'is', 'dmesg'] > > +DATA['tests']['sometest']['info'] = \ > > + 'Returncode: 1\n\nErrors:stderr\n\nOutput: stdout\n' > > +DATA['tests'].update({ > > + 'group1/groupA/test/subtest 1': { > > + 'info': 'Returncode: 1\n\nErrors:stderr\n\nOutput:stdout\n', > > + 'subtest': { > > + 'subtest 1': 'pass', > > + 'subtest 2': 'pass' > > + }, > > + 'returncode': 0, > > + 'command': 'this is a command', > > + 'result': 'pass', > > + 'time': 0.1 > > + }, > > + 'group1/groupA/test/subtest 2': { > > + 'info': 'Returncode: 1\n\nErrors:stderr\n\nOutput:stdout\n', > > + 'subtest': { > > + 'subtest 1': 'pass', > > + 'subtest 2': 'pass' > > + }, > > + 'returncode': 0, > > + 'command': 'this is a command', > > + 'result': 'pass', > > + 'time': 0.1 > > + }, > > + 'single/test/thing': { > > + 'info': 'Returncode: 1\n\nErrors:stderr\n\nOutput:stdout\n', > > + 'subtest': { > > + 'subtest 1': 'pass', > > + }, > > + 'returncode': 0, > > + 'command': 'this is a command', > > + 'result': 'pass', > > + 'time': 0.1 > > + } > > +}) > > + > > +with utils.with_tempfile(json.dumps(DATA)) as f: > > + RESULT = results._update_zero_to_one(results.load_results(f)) > > + > > + > > +def test_dmesg(): > > + """ version 1: dmesg is converted from a list to a string """ > > + assert RESULT.tests['sometest']['dmesg'] == 'this\nis\ndmesg' > > + > > + > > +def test_subtests_remove_duplicates(): > > + """ Version 1: Removes duplicate entries """ > > + assert 'group1/groupA/test/subtest 1' not in RESULT.tests > > + assert 'group1/groupA/test/subtest 2' not in RESULT.tests > > + > > + > > +def test_subtests_add_test(): > > + """ Version 1: Add an entry for the actual test """ > > + assert RESULT.tests.get('group1/groupA/test') > > + > > + > > +def test_subtests_test_is_testresult(): > > + """ Version 1: The result of the new test is a TestResult Instance > > """ > > + assert isinstance( > > + RESULT.tests['group1/groupA/test'], > > + results.TestResult) > > + > > + > > +def test_info_delete(): > > + """ Version 1: Remove the info name from results """ > > + for value in RESULT.tests.itervalues(): > > + assert 'info' not in value > > + > > + > > +def test_returncode_from_info(): > > + """ Version 1: Use the returncode from info if there is no returncode > > """ + assert RESULT.tests['sometest']['returncode'] == 1 > > + > > + > > +def test_returncode_no_override(): > > + """ Version 1: Do not clobber returncode with info > > + > > + The returncode from info should not overwrite an existing returcnode > > + attribute, this test only tests that the value in info isn't used > > when > > + there is a value in returncode already > > + > > + """ > > + assert RESULT.tests['group1/groupA/test']['returncode'] != 1 > > + > > + > > +def test_err_from_info(): > > + """ Version 1: add an err attribute from info """ > > + assert RESULT.tests['group1/groupA/test']['err'] == 'stderr' > > + > > + > > +def test_out_from_info(): > > + """ Version 1: add an out attribute from info """ > > + assert RESULT.tests['group1/groupA/test']['out'] == 'stdout' > > + > > + > > +def test_set_version(): > > + """ Version 1: Set the version to 1 """ > > + assert RESULT.results_version == 1 > > + > > + > > +def test_dont_break_single_subtest(): > > + """ Version 1: Don't break single subtest entries > > + > > + A test with a single subtest was written correctly before, dont break > > it by + removing the name of the test. ex: > > + test/foo/bar: { > > + ... > > + subtest: { > > + 1x1: pass > > + } > > + } > > + > > + should remain test/foo/bar since bar is the name of the test not a > > subtest + > > + """ > > + assert RESULT.tests['single/test/thing'] > > + > > + > > +def test_info_split(): > > + """ Version 1: info can split into any number of elements """ > > + data = copy.copy(DATA) > > + data['tests']['sometest']['info'] = \ > > + 'Returncode: 1\n\nErrors:stderr\n\nOutput: > > stdout\n\nmore\n\nstuff' + > > + with utils.with_tempfile(json.dumps(data)) as f: > > + results._update_zero_to_one(results.load_results(f)) > > diff --git a/framework/tests/utils.py b/framework/tests/utils.py > > index f337b1e..252ced6 100644 > > --- a/framework/tests/utils.py > > +++ b/framework/tests/utils.py > > > > @@ -33,6 +33,7 @@ try: > > import simplejson as json > > > > except ImportError: > > import json > > > > +import framework.results > > > > __all__ = [ > > > > @@ -49,6 +50,7 @@ JSON_DATA = { > > > > "filter": [], > > "exclude_filter": [] > > > > }, > > > > + "results_version": framework.results.CURRENT_JSON_VERSION, > > > > "name": "fake-tests", > > "lspci": "fake", > > "glxinfo": "fake", > > > > -- > > 2.0.0
signature.asc
Description: This is a digitally signed message part.
_______________________________________________ Piglit mailing list [email protected] http://lists.freedesktop.org/mailman/listinfo/piglit
