labath created this revision.
labath added a reviewer: tfiala.
labath added a subscriber: lldb-commits.
This adds ability to mark test that do not complete due to hangs, crashes,
etc., as "expected",
to avoid flagging the build red for a known problem. Functionally, this extends
the scope of the
existing expectedFailureXXX decorators to cover these states as well. Once this
is in, I will
start replacing the magic list of failing tests in dosep.py with our regular
annotations which
should hopefully make code simpler.
http://reviews.llvm.org/D15530
Files:
packages/Python/lldbsuite/test/dosep.py
packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park
packages/Python/lldbsuite/test/lldbtest.py
packages/Python/lldbsuite/test/result_formatter.py
Index: packages/Python/lldbsuite/test/result_formatter.py
===================================================================
--- packages/Python/lldbsuite/test/result_formatter.py
+++ packages/Python/lldbsuite/test/result_formatter.py
@@ -163,6 +163,7 @@
TYPE_TEST_RESULT = "test_result"
TYPE_TEST_START = "test_start"
TYPE_MARK_TEST_RERUN_ELIGIBLE = "test_eligible_for_rerun"
+ TYPE_MARK_TEST_EXPECTED_FAILURE = "test_expected_failure"
TYPE_SESSION_TERMINATE = "terminate"
RESULT_TYPES = set([
@@ -526,6 +527,20 @@
return event
@staticmethod
+ def event_for_mark_test_expected_failure(test):
+ """Creates an event that indicates the specified test is expected
+ to fail.
+
+ @param test the TestCase instance to which this pertains.
+
+ @return an event that specifies the given test is expected to fail.
+ """
+ event = EventBuilder._event_dictionary_common(
+ test,
+ EventBuilder.TYPE_MARK_TEST_EXPECTED_FAILURE)
+ return event
+
+ @staticmethod
def add_entries_to_all_events(entries_dict):
"""Specifies a dictionary of entries to add to all test events.
@@ -668,6 +683,11 @@
# timeout test status for this.
self.expected_timeouts_by_basename = set()
+ # Tests which have reported that they are expecting to fail. These will
+ # be marked as expected failures even if they return a failing status,
+ # probably because they crashed or deadlocked.
+ self.expected_failures = set()
+
# Keep track of rerun-eligible tests.
# This is a set that contains tests saved as:
# {test_filename}:{test_class}:{test_name}
@@ -698,6 +718,15 @@
component_count += 1
return key
+ def _mark_test_as_expected_failure(self, test_result_event):
+ key = self._make_rerun_eligibility_key(test_result_event)
+ if key is not None:
+ self.expected_failures.add(key)
+ else:
+ sys.stderr.write(
+ "\nerror: test marked as expected failure but "
+ "failed to create key.\n")
+
def _mark_test_for_rerun_eligibility(self, test_result_event):
key = self._make_rerun_eligibility_key(test_result_event)
if key is not None:
@@ -767,6 +796,20 @@
# Convert to an expected timeout.
event["status"] = EventBuilder.STATUS_EXPECTED_TIMEOUT
+ def _maybe_remap_expected_failure(self, event):
+ if event is None:
+ return
+
+ key = self._make_rerun_eligibility_key(event)
+ if key not in self.expected_failures:
+ return
+
+ status = event.get("status", None)
+ if status in EventBuilder.TESTRUN_ERROR_STATUS_VALUES:
+ event["status"] = EventBuilder.STATUS_EXPECTED_FAILURE
+ elif status == EventBuilder.STATUS_SUCCESS:
+ event["status"] = EventBuilder.STATUS_UNEXPECTED_SUCCESS
+
def handle_event(self, test_event):
"""Handles the test event for collection into the formatter output.
@@ -795,6 +838,7 @@
# Remap timeouts to expected timeouts.
if event_type in EventBuilder.RESULT_TYPES:
self._maybe_remap_expected_timeout(test_event)
+ self._maybe_remap_expected_failure(test_event)
event_type = test_event.get("event", "")
if event_type == "terminate":
@@ -825,6 +869,8 @@
self.started_tests_by_worker[worker_index] = test_event
elif event_type == EventBuilder.TYPE_MARK_TEST_RERUN_ELIGIBLE:
self._mark_test_for_rerun_eligibility(test_event)
+ elif event_type == EventBuilder.TYPE_MARK_TEST_EXPECTED_FAILURE:
+ self._mark_test_as_expected_failure(test_event)
def set_expected_timeouts_by_basename(self, basenames):
"""Specifies a list of test file basenames that are allowed to timeout
Index: packages/Python/lldbsuite/test/lldbtest.py
===================================================================
--- packages/Python/lldbsuite/test/lldbtest.py
+++ packages/Python/lldbsuite/test/lldbtest.py
@@ -598,6 +598,10 @@
from unittest2 import case
self = args[0]
if expected_fn(self):
+ if configuration.results_formatter_object is not None:
+ # Mark this test as expected to fail.
+ configuration.results_formatter_object.handle_event(
+ EventBuilder.event_for_mark_test_expected_failure(self))
xfail_func = unittest2.expectedFailure(func)
xfail_func(*args, **kwargs)
else:
Index: packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park
===================================================================
--- packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park
+++ packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park
@@ -10,6 +10,7 @@
"""Forces test timeout."""
mydir = lldbtest.TestBase.compute_mydir(__file__)
+ @lldbtest.expectedFailureAll()
def test_buildbot_sees_expected_timeout(self):
"""Tests that expected timeout logic kicks in and is picked up."""
while True:
Index: packages/Python/lldbsuite/test/dosep.py
===================================================================
--- packages/Python/lldbsuite/test/dosep.py
+++ packages/Python/lldbsuite/test/dosep.py
@@ -1143,7 +1143,6 @@
target = m.group(1)
expected_timeout = set()
- expected_timeout.add("TestExpectedTimeout.py")
if target.startswith("linux"):
expected_timeout |= {
_______________________________________________
lldb-commits mailing list
[email protected]
http://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits
