DavidSpickett created this revision.
Herald added a project: LLDB.
Herald added a subscriber: lldb-commits.
DavidSpickett requested review of this revision.
Herald added a subscriber: JDevlieghere.
This updates the errors reported by expect()
to something like:
'''
Ran command:
"help"
Got output:
Debugger commands:
<...>
Expecting start string: "Debugger commands:" (was found)
Expecting end string: "foo" (was not found)
'''
(see added tests for more examples)
This shows the user exactly what was run,
what checks passed and which failed. Along with
whether that check was supposed to pass.
(including what regex patterns matched)
These lines are also output to the test
trace file, whether the test passes or not.
Note that expect() will still fail at the first failed
check, in line with previous behaviour.
Also I have flipped the wording of the assert
message functions (.*_MSG) to describe failures
not successes. This makes more sense as they are
only shown on assert failures.
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D86792
Files:
lldb/packages/Python/lldbsuite/test/lldbtest.py
lldb/test/API/assert_messages_test/TestAssertMessages.py
Index: lldb/test/API/assert_messages_test/TestAssertMessages.py
===================================================================
--- /dev/null
+++ lldb/test/API/assert_messages_test/TestAssertMessages.py
@@ -0,0 +1,113 @@
+"""
+Test the format of API test suite assert failure messages
+"""
+
+
+import lldb
+import lldbsuite.test.lldbutil as lldbutil
+from lldbsuite.test.lldbtest import *
+from textwrap import dedent
+
+
+class AssertMessagesTestCase(TestBase):
+
+ mydir = TestBase.compute_mydir(__file__)
+ NO_DEBUG_INFO_TESTCASE = True
+
+ def catch_assert_fail(self, *args, **kwargs):
+ try:
+ self.expect(*args, **kwargs)
+ except AssertionError as e:
+ return e
+ else:
+ self.fail("Expect should have raised AssertionError!")
+
+ def expect_assert_msg(self, exception, substrs):
+ self.expect(str(exception), exe=False, substrs=substrs)
+
+ def test_expect(self):
+ """Test format of messages produced by expect(...)"""
+
+ # When an expect passes the messages are sent to the trace
+ # file which we can't access here. So really, these only
+ # check what failures look like, but it *should* be the same
+ # content for the trace log too.
+
+ # Will stop at startstr fail
+ e = self.catch_assert_fail("help", startstr="dog", endstr="cat")
+ self.expect_assert_msg(e, [dedent("""\
+ Ran command:
+ "help"
+
+ Got output:
+ Debugger commands:"""),
+ """Expecting start string: "dog" (was not found)"""])
+
+ # startstr passes, endstr fails
+ # We see both reported
+ e = self.catch_assert_fail("help",
+ startstr="Debugger commands:", endstr="foo")
+ self.expect_assert_msg(e, [dedent("""\
+ Ran command:
+ "help"
+
+ Got output:
+ Debugger commands:"""),
+ """Expecting start string: "Debugger commands:" (was found)""",
+ """Expecting end string: "foo" (was not found)"""])
+
+ # Same thing for substrs, regex patterns ignored because of substr failure
+ # Any substr after the first missing is also ignored
+ e = self.catch_assert_fail("abcdefg", substrs=["abc", "ijk", "xyz"],
+ patterns=["foo", "bar"], exe=False)
+ self.expect_assert_msg(e, [dedent("""\
+ Checking string:
+ "abcdefg"
+
+ Expecting sub string: "abc" (was found)
+ Expecting sub string: "ijk" (was not found)""")])
+
+ # Regex patterns also stop at first failure, subsequent patterns ignored
+ # They are last in the chain so no other check gets skipped
+ # Including the rest of the conditions here to prove they are run and shown
+ e = self.catch_assert_fail("0123456789",
+ startstr="012", endstr="789", substrs=["345", "678"],
+ patterns=["[0-9]+", "[a-f]+", "a|b|c"], exe=False)
+ self.expect_assert_msg(e, [dedent("""\
+ Checking string:
+ "0123456789"
+
+ Expecting start string: "012" (was found)
+ Expecting end string: "789" (was found)
+ Expecting sub string: "345" (was found)
+ Expecting sub string: "678" (was found)
+ Expecting regex pattern: "[0-9]+" (was found, matched "0123456789")
+ Expecting regex pattern: "[a-f]+" (was not found)""")])
+
+ # This time we dont' want matches but we do get them
+ e = self.catch_assert_fail("the quick brown fox",
+ startstr="cat", endstr="rabbit", substrs=["abc", "def"],
+ # Note that the second pattern *will* match
+ patterns=["[0-9]+", "fox"],
+ exe=False, matching=False)
+ self.expect_assert_msg(e, [dedent("""\
+ Checking string:
+ "the quick brown fox"
+
+ Not expecting start string: "cat" (was not found)
+ Not expecting end string: "rabbit" (was not found)
+ Not expecting sub string: "abc" (was not found)
+ Not expecting sub string: "def" (was not found)
+ Not expecting regex pattern: "[0-9]+" (was not found)
+ Not expecting regex pattern: "fox" (was found, matched "fox")""")])
+
+ # Extra assert messages are only printed when we get a failure
+ # So I can't test that from here, just how it looks when it's printed
+ e = self.catch_assert_fail("mouse", startstr="cat", exe=False,
+ msg="Reason for check goes here!")
+ self.expect_assert_msg(e, [dedent("""\
+ Checking string:
+ "mouse"
+
+ Expecting start string: "cat" (was not found)
+ Reason for check goes here!""")])
Index: lldb/packages/Python/lldbsuite/test/lldbtest.py
===================================================================
--- lldb/packages/Python/lldbsuite/test/lldbtest.py
+++ lldb/packages/Python/lldbsuite/test/lldbtest.py
@@ -179,12 +179,12 @@
def CMD_MSG(str):
- '''A generic "Command '%s' returns successfully" message generator.'''
- return "Command '%s' returns successfully" % str
+ '''A generic "Command '%s' did not return successfully" message generator.'''
+ return "Command '%s' did not return successfully" % str
def COMPLETION_MSG(str_before, str_after, completions):
- '''A generic message generator for the completion mechanism.'''
+ '''A generic assertion failed message generator for the completion mechanism.'''
return ("'%s' successfully completes to '%s', but completions were:\n%s"
% (str_before, str_after, "\n".join(completions)))
@@ -198,8 +198,8 @@
def SETTING_MSG(setting):
- '''A generic "Value of setting '%s' is correct" message generator.'''
- return "Value of setting '%s' is correct" % setting
+ '''A generic "Value of setting '%s' is not correct" message generator.'''
+ return "Value of setting '%s' is not correct" % setting
def line_number(filename, string_to_match):
@@ -2434,58 +2434,76 @@
with recording(self, trace) as sbuf:
print("looking at:", output, file=sbuf)
- # The heading says either "Expecting" or "Not expecting".
- heading = "Expecting" if matching else "Not expecting"
+ expecting_str = "Expecting" if matching else "Not expecting"
+ def found_str(matched):
+ return "was found" if matched else "was not found"
+
+ # To be used as assert fail message and/or trace content
+ log_lines = [
+ "{}:".format("Ran command" if exe else "Checking string"),
+ "\"{}\"".format(str),
+ # Space out command and output
+ "",
+ ]
+ if exe:
+ # Newline before output to make large strings more readable
+ log_lines.append("Got output:\n{}".format(output))
- # Start from the startstr, if specified.
- # If there's no startstr, set the initial state appropriately.
- matched = output.startswith(startstr) if startstr else (
- True if matching else False)
+ # Assume that we start matched if we want a match
+ # Meaning if you have no conditions, matching or
+ # not matching will always pass
+ matched = matching
+ # We will stop checking on first failure
if startstr:
- with recording(self, trace) as sbuf:
- print("%s start string: %s" % (heading, startstr), file=sbuf)
- print("Matched" if matched else "Not matched", file=sbuf)
+ matched = output.startswith(startstr)
+ log_lines.append("{} start string: \"{}\" ({})".format(
+ expecting_str, startstr, found_str(matched)))
- # Look for endstr, if specified.
- keepgoing = matched if matching else not matched
- if endstr:
+ if endstr and matched == matching:
matched = output.endswith(endstr)
- with recording(self, trace) as sbuf:
- print("%s end string: %s" % (heading, endstr), file=sbuf)
- print("Matched" if matched else "Not matched", file=sbuf)
+ log_lines.append("{} end string: \"{}\" ({})".format(
+ expecting_str, endstr, found_str(matched)))
- # Look for sub strings, if specified.
- keepgoing = matched if matching else not matched
- if substrs and keepgoing:
+ if substrs and matched == matching:
start = 0
for substr in substrs:
index = output[start:].find(substr)
start = start + index if ordered and matching else 0
matched = index != -1
- with recording(self, trace) as sbuf:
- print("%s sub string: %s" % (heading, substr), file=sbuf)
- print("Matched" if matched else "Not matched", file=sbuf)
- keepgoing = matched if matching else not matched
- if not keepgoing:
+ log_lines.append("{} sub string: \"{}\" ({})".format(
+ expecting_str, substr, found_str(matched)))
+
+ if matched != matching:
break
- # Search for regular expression patterns, if specified.
- keepgoing = matched if matching else not matched
- if patterns and keepgoing:
+ if patterns and matched == matching:
for pattern in patterns:
- # Match Objects always have a boolean value of True.
- matched = bool(re.search(pattern, output))
- with recording(self, trace) as sbuf:
- print("%s pattern: %s" % (heading, pattern), file=sbuf)
- print("Matched" if matched else "Not matched", file=sbuf)
- keepgoing = matched if matching else not matched
- if not keepgoing:
+ matched = re.search(pattern, output)
+
+ pattern_line = "{} regex pattern: \"{}\" ({}".format(
+ expecting_str, pattern, found_str(matched))
+ if matched:
+ pattern_line += ", matched \"{}\"".format(
+ matched.group(0))
+ pattern_line += ")"
+ log_lines.append(pattern_line)
+
+ # Convert to bool because match objects
+ # are True-ish but != True itself
+ matched = bool(matched)
+ if matched != matching:
break
- self.assertTrue(matched if matching else not matched,
- msg + "\nCommand output:\n" + EXP_MSG(str, output, exe)
- if msg else EXP_MSG(str, output, exe))
+ # If a check failed, add any extra assert message
+ if msg is not None and matched != matching:
+ log_lines.append(msg)
+
+ log_msg = "\n".join(log_lines)
+ with recording(self, trace) as sbuf:
+ print(log_msg, file=sbuf)
+ if matched != matching:
+ self.fail(log_msg)
def expect_expr(
self,
_______________________________________________
lldb-commits mailing list
lldb-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits
