Author: labath Date: Wed Dec 16 06:09:45 2015 New Revision: 255763 URL: http://llvm.org/viewvc/llvm-project?rev=255763&view=rev Log: [test] Add ability to expect timeouts
Summary: This adds ability to mark test that do not complete due to hangs, crashes, etc., as "expected", to avoid flagging the build red for a known problem. Functionally, this extends the scope of the existing expectedFailureXXX decorators to cover these states as well. Once this is in, I will start replacing the magic list of failing tests in dosep.py with our regular annotations which should hopefully make code simpler. Reviewers: tfiala Subscribers: lldb-commits Differential Revision: http://reviews.llvm.org/D15530 Modified: lldb/trunk/packages/Python/lldbsuite/test/dosep.py lldb/trunk/packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park lldb/trunk/packages/Python/lldbsuite/test/lldbtest.py lldb/trunk/packages/Python/lldbsuite/test/result_formatter.py Modified: lldb/trunk/packages/Python/lldbsuite/test/dosep.py URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/dosep.py?rev=255763&r1=255762&r2=255763&view=diff ============================================================================== --- lldb/trunk/packages/Python/lldbsuite/test/dosep.py (original) +++ lldb/trunk/packages/Python/lldbsuite/test/dosep.py Wed Dec 16 06:09:45 2015 @@ -1143,7 +1143,6 @@ def getExpectedTimeouts(platform_name): target = m.group(1) expected_timeout = set() - expected_timeout.add("TestExpectedTimeout.py") if target.startswith("linux"): expected_timeout |= { Modified: lldb/trunk/packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park?rev=255763&r1=255762&r2=255763&view=diff ============================================================================== --- lldb/trunk/packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park (original) +++ lldb/trunk/packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park Wed Dec 16 06:09:45 2015 @@ -10,6 +10,7 @@ class ExpectedTimeoutTestCase(lldbtest.T """Forces test timeout.""" mydir = lldbtest.TestBase.compute_mydir(__file__) + @lldbtest.expectedFailureAll() def test_buildbot_sees_expected_timeout(self): """Tests that expected timeout logic kicks in and is picked up.""" while True: Modified: lldb/trunk/packages/Python/lldbsuite/test/lldbtest.py URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/lldbtest.py?rev=255763&r1=255762&r2=255763&view=diff ============================================================================== --- lldb/trunk/packages/Python/lldbsuite/test/lldbtest.py (original) +++ lldb/trunk/packages/Python/lldbsuite/test/lldbtest.py Wed Dec 16 06:09:45 2015 @@ -598,6 +598,10 @@ def expectedFailure(expected_fn, bugnumb from unittest2 import case self = args[0] if expected_fn(self): + if configuration.results_formatter_object is not None: + # Mark this test as expected to fail. + configuration.results_formatter_object.handle_event( + EventBuilder.event_for_mark_test_expected_failure(self)) xfail_func = unittest2.expectedFailure(func) xfail_func(*args, **kwargs) else: Modified: lldb/trunk/packages/Python/lldbsuite/test/result_formatter.py URL: http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/result_formatter.py?rev=255763&r1=255762&r2=255763&view=diff ============================================================================== --- lldb/trunk/packages/Python/lldbsuite/test/result_formatter.py (original) +++ lldb/trunk/packages/Python/lldbsuite/test/result_formatter.py Wed Dec 16 06:09:45 2015 @@ -165,6 +165,7 @@ class EventBuilder(object): TYPE_TEST_RESULT = "test_result" TYPE_TEST_START = "test_start" TYPE_MARK_TEST_RERUN_ELIGIBLE = "test_eligible_for_rerun" + TYPE_MARK_TEST_EXPECTED_FAILURE = "test_expected_failure" TYPE_SESSION_TERMINATE = "terminate" RESULT_TYPES = set([ @@ -528,6 +529,20 @@ class EventBuilder(object): return event @staticmethod + def event_for_mark_test_expected_failure(test): + """Creates an event that indicates the specified test is expected + to fail. + + @param test the TestCase instance to which this pertains. + + @return an event that specifies the given test is expected to fail. + """ + event = EventBuilder._event_dictionary_common( + test, + EventBuilder.TYPE_MARK_TEST_EXPECTED_FAILURE) + return event + + @staticmethod def add_entries_to_all_events(entries_dict): """Specifies a dictionary of entries to add to all test events. @@ -681,6 +696,11 @@ class ResultsFormatter(object): # timeout test status for this. self.expected_timeouts_by_basename = set() + # Tests which have reported that they are expecting to fail. These will + # be marked as expected failures even if they return a failing status, + # probably because they crashed or deadlocked. + self.expected_failures = set() + # Keep track of rerun-eligible tests. # This is a set that contains tests saved as: # {test_filename}:{test_class}:{test_name} @@ -721,6 +741,15 @@ class ResultsFormatter(object): component_count += 1 return key + def _mark_test_as_expected_failure(self, test_result_event): + key = self._make_key(test_result_event) + if key is not None: + self.expected_failures.add(key) + else: + sys.stderr.write( + "\nerror: test marked as expected failure but " + "failed to create key.\n") + def _mark_test_for_rerun_eligibility(self, test_result_event): key = self._make_key(test_result_event) if key is not None: @@ -796,6 +825,20 @@ class ResultsFormatter(object): # Convert to an expected timeout. event["status"] = EventBuilder.STATUS_EXPECTED_TIMEOUT + def _maybe_remap_expected_failure(self, event): + if event is None: + return + + key = self._make_key(event) + if key not in self.expected_failures: + return + + status = event.get("status", None) + if status in EventBuilder.TESTRUN_ERROR_STATUS_VALUES: + event["status"] = EventBuilder.STATUS_EXPECTED_FAILURE + elif status == EventBuilder.STATUS_SUCCESS: + event["status"] = EventBuilder.STATUS_UNEXPECTED_SUCCESS + def handle_event(self, test_event): """Handles the test event for collection into the formatter output. @@ -824,6 +867,7 @@ class ResultsFormatter(object): # Remap timeouts to expected timeouts. if event_type in EventBuilder.RESULT_TYPES: self._maybe_remap_expected_timeout(test_event) + self._maybe_remap_expected_failure(test_event) event_type = test_event.get("event", "") if event_type == "terminate": @@ -887,6 +931,8 @@ class ResultsFormatter(object): elif event_type == EventBuilder.TYPE_MARK_TEST_RERUN_ELIGIBLE: self._mark_test_for_rerun_eligibility(test_event) + elif event_type == EventBuilder.TYPE_MARK_TEST_EXPECTED_FAILURE: + self._mark_test_as_expected_failure(test_event) def set_expected_timeouts_by_basename(self, basenames): """Specifies a list of test file basenames that are allowed to timeout _______________________________________________ lldb-commits mailing list lldb-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits
