Author: tfiala
Date: Fri Dec 11 13:44:23 2015
New Revision: 255363
URL: http://llvm.org/viewvc/llvm-project?rev=255363&view=rev
Log:
Add expected timeout support to test event architecture.
Added:
lldb/trunk/packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park
Modified:
lldb/trunk/packages/Python/lldbsuite/test/basic_results_formatter.py
lldb/trunk/packages/Python/lldbsuite/test/dosep.py
lldb/trunk/packages/Python/lldbsuite/test/result_formatter.py
Modified: lldb/trunk/packages/Python/lldbsuite/test/basic_results_formatter.py
URL:
http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/basic_results_formatter.py?rev=255363&r1=255362&r2=255363&view=diff
==============================================================================
--- lldb/trunk/packages/Python/lldbsuite/test/basic_results_formatter.py
(original)
+++ lldb/trunk/packages/Python/lldbsuite/test/basic_results_formatter.py Fri
Dec 11 13:44:23 2015
@@ -315,6 +315,11 @@ class BasicResultsFormatter(result_forma
[result_formatter.EventBuilder.STATUS_SKIP, "Skip", False, None],
[result_formatter.EventBuilder.STATUS_TIMEOUT,
"Timeout", True, "TIMEOUT"],
+ [result_formatter.EventBuilder.STATUS_EXPECTED_TIMEOUT,
+ # Intentionally using the unusual hyphenation in TIME-OUT to
+ # prevent buildbots from thinking it is an issue when scanning
+ # for TIMEOUT.
+ "Expected Timeout", True, "EXPECTED TIME-OUT"]
]
# Partition all the events by test result status
Modified: lldb/trunk/packages/Python/lldbsuite/test/dosep.py
URL:
http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/dosep.py?rev=255363&r1=255362&r2=255363&view=diff
==============================================================================
--- lldb/trunk/packages/Python/lldbsuite/test/dosep.py (original)
+++ lldb/trunk/packages/Python/lldbsuite/test/dosep.py Fri Dec 11 13:44:23 2015
@@ -1128,6 +1128,7 @@ def getExpectedTimeouts(platform_name):
target = m.group(1)
expected_timeout = set()
+ expected_timeout.add("TestExpectedTimeout.py")
if target.startswith("linux"):
expected_timeout |= {
@@ -1475,6 +1476,12 @@ def main(num_threads, test_subdir, test_
system_info = " ".join(platform.uname())
+ # Figure out which test files should be enabled for expected
+ # timeout
+ expected_timeout = getExpectedTimeouts(dotest_options.lldb_platform_name)
+ if results_formatter is not None:
+ results_formatter.set_expected_timeouts_by_basename(expected_timeout)
+
# Figure out which testrunner strategy we'll use.
runner_strategies_by_name = get_test_runner_strategies(num_threads)
@@ -1514,7 +1521,6 @@ def main(num_threads, test_subdir, test_
os.rename(core, os.path.join(session_dir, dst))
# remove expected timeouts from failures
- expected_timeout = getExpectedTimeouts(dotest_options.lldb_platform_name)
for xtime in expected_timeout:
if xtime in timed_out:
timed_out.remove(xtime)
Added:
lldb/trunk/packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park
URL:
http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park?rev=255363&view=auto
==============================================================================
---
lldb/trunk/packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park
(added)
+++
lldb/trunk/packages/Python/lldbsuite/test/issue_verification/TestExpectedTimeout.py.park
Fri Dec 11 13:44:23 2015
@@ -0,0 +1,19 @@
+"""Tests that a timeout is detected by the testbot."""
+from __future__ import print_function
+
+import time
+
+import lldbsuite.test.lldbtest as lldbtest
+
+
+class ExpectedTimeoutTestCase(lldbtest.TestBase):
+ """Forces test timeout."""
+ mydir = lldbtest.TestBase.compute_mydir(__file__)
+
+ def test_buildbot_sees_expected_timeout(self):
+ """Tests that expected timeout logic kicks in and is picked up."""
+ while True:
+ try:
+ time.sleep(1)
+ except:
+ print("ignoring exception during sleep")
Modified: lldb/trunk/packages/Python/lldbsuite/test/result_formatter.py
URL:
http://llvm.org/viewvc/llvm-project/lldb/trunk/packages/Python/lldbsuite/test/result_formatter.py?rev=255363&r1=255362&r2=255363&view=diff
==============================================================================
--- lldb/trunk/packages/Python/lldbsuite/test/result_formatter.py (original)
+++ lldb/trunk/packages/Python/lldbsuite/test/result_formatter.py Fri Dec 11
13:44:23 2015
@@ -163,11 +163,16 @@ class EventBuilder(object):
TYPE_TEST_START = "test_start"
TYPE_MARK_TEST_RERUN_ELIGIBLE = "test_eligible_for_rerun"
+ RESULT_TYPES = set([
+ TYPE_JOB_RESULT,
+ TYPE_TEST_RESULT])
+
# Test/Job Status Tags
STATUS_EXCEPTIONAL_EXIT = "exceptional_exit"
STATUS_SUCCESS = "success"
STATUS_FAILURE = "failure"
STATUS_EXPECTED_FAILURE = "expected_failure"
+ STATUS_EXPECTED_TIMEOUT = "expected_timeout"
STATUS_UNEXPECTED_SUCCESS = "unexpected_success"
STATUS_SKIP = "skip"
STATUS_ERROR = "error"
@@ -622,6 +627,7 @@ class ResultsFormatter(object):
self.result_status_counts = {
EventBuilder.STATUS_SUCCESS: 0,
EventBuilder.STATUS_EXPECTED_FAILURE: 0,
+ EventBuilder.STATUS_EXPECTED_TIMEOUT: 0,
EventBuilder.STATUS_SKIP: 0,
EventBuilder.STATUS_UNEXPECTED_SUCCESS: 0,
EventBuilder.STATUS_FAILURE: 0,
@@ -642,6 +648,13 @@ class ResultsFormatter(object):
# entirely consistent from the outside.
self.lock = threading.Lock()
+ # Keeps track of the test base filenames for tests that
+ # are expected to timeout. If a timeout occurs in any test
+ # basename that matches this list, that result should be
+ # converted into a non-issue. We'll create an expected
+ # timeout test status for this.
+ self.expected_timeouts_by_basename = set()
+
def _maybe_remap_job_result_event(self, test_event):
"""Remaps timeout/exceptional exit job results to last test method
running.
@@ -678,6 +691,21 @@ class ResultsFormatter(object):
if "test_filename" in test_start:
test_event["test_filename"] = test_start["test_filename"]
+ def _maybe_remap_expected_timeout(self, event):
+ if event is None:
+ return
+
+ status = event.get("status", None)
+ if status is None or status != EventBuilder.STATUS_TIMEOUT:
+ return
+
+ # Check if the timeout test's basename is in the expected timeout
+ # list. If so, convert to an expected timeout.
+ basename = os.path.basename(event.get("test_filename", ""))
+ if basename in self.expected_timeouts_by_basename:
+ # Convert to an expected timeout.
+ event["status"] = EventBuilder.STATUS_EXPECTED_TIMEOUT
+
def handle_event(self, test_event):
"""Handles the test event for collection into the formatter output.
@@ -703,6 +731,11 @@ class ResultsFormatter(object):
self._maybe_remap_job_result_event(test_event)
event_type = test_event.get("event", "")
+ # Remap timeouts to expected timeouts.
+ if event_type in EventBuilder.RESULT_TYPES:
+ self._maybe_remap_expected_timeout(test_event)
+ event_type = test_event.get("event", "")
+
if event_type == "terminate":
self.terminate_called = True
elif (event_type == EventBuilder.TYPE_TEST_RESULT or
@@ -724,6 +757,16 @@ class ResultsFormatter(object):
if worker_index is not None:
self.started_tests_by_worker[worker_index] = test_event
+ def set_expected_timeouts_by_basename(self, basenames):
+ """Specifies a list of test file basenames that are allowed to timeout
+ without being called out as a timeout issue.
+
+ These fall into a new status category called STATUS_EXPECTED_TIMEOUT.
+ """
+ if basenames is not None:
+ for basename in basenames:
+ self.expected_timeouts_by_basename.add(basename)
+
def track_start_time(self, test_class, test_name, start_time):
"""tracks the start time of a test so elapsed time can be computed.
_______________________________________________
lldb-commits mailing list
lldb-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits
