https://github.com/DavidSpickett created https://github.com/llvm/llvm-project/pull/117985
Now "Download" will be a link to the file so people don't have to know to open the build tab and find the download button. This is a URL from a real build: https://buildkite.com/organizations/llvm-project/pipelines/github-pull-requests/builds/123979/jobs/01937132-0fc3-4c95-a884-2fc0048cb9a7/download.txt And this is how we can build it: https://buildkite.com/organizations/{BUILDKITE_ORGANIZATION_SLUG}/pipelines/{BUILDKITE_PIPELINE_SLUG}/builds/{BUILDKITE_BUILD_NUMBER}/jobs/{BUILDKITE_JOB_ID}/download.txt Given these env vars that were set in that job: BUILDKITE_ORGANIZATION_SLUG="llvm-project" BUILDKITE_PIPELINE_SLUG="github-pull-requests" BUILDKITE_BUILD_NUMBER="123979" BUILDKITE_JOB_ID="01937132-0fc3-4c95-a884-2fc0048cb9a7" In theory these will always be available but: 1. Rather safe than sorry with this script, I don't want to make a passing build a failure because this script failed. 2. It would get very annoying if you had to set all these to test the script locally. >From d05b35d5db09429441e777fbb7051bea60404163 Mon Sep 17 00:00:00 2001 From: David Spickett <david.spick...@linaro.org> Date: Thu, 28 Nov 2024 09:53:40 +0000 Subject: [PATCH 1/2] [ci] Include a log download link when test report is truncated Now "Download" will be a link to the file so people don't have to know to open the build tab and find the download button. This is a URL from a real build: https://buildkite.com/organizations/llvm-project/pipelines/github-pull-requests/builds/123979/jobs/01937132-0fc3-4c95-a884-2fc0048cb9a7/download.txt And this is how we can build it: https://buildkite.com/organizations/{BUILDKITE_ORGANIZATION_SLUG}/pipelines/{BUILDKITE_PIPELINE_SLUG}/builds/{BUILDKITE_BUILD_NUMBER}/jobs/{BUILDKITE_JOB_ID}/download.txt Given these env vars that were set in that job: BUILDKITE_ORGANIZATION_SLUG="llvm-project" BUILDKITE_PIPELINE_SLUG="github-pull-requests" BUILDKITE_BUILD_NUMBER="123979" BUILDKITE_JOB_ID="01937132-0fc3-4c95-a884-2fc0048cb9a7" In theory these will always be available but: 1. Rather safe than sorry with this script, I don't want to make a passing build a failure because this script failed. 2. It would get very annoying if you had to set all these to test the script locally. --- .ci/generate_test_report.py | 88 ++++++++++++++++++++++++++++++++++--- 1 file changed, 82 insertions(+), 6 deletions(-) diff --git a/.ci/generate_test_report.py b/.ci/generate_test_report.py index c44936b19dab98..6ba8ed432fad26 100644 --- a/.ci/generate_test_report.py +++ b/.ci/generate_test_report.py @@ -5,6 +5,7 @@ # python3 -m unittest discover -p generate_test_report.py import argparse +import os import subprocess import unittest from io import StringIO @@ -267,6 +268,46 @@ def test_report_dont_list_failures(self): ), ) + def test_report_dont_list_failures_link_to_log(self): + self.assertEqual( + _generate_report( + "Foo", + [ + junit_from_xml( + dedent( + """\ + <?xml version="1.0" encoding="UTF-8"?> + <testsuites time="0.02"> + <testsuite name="Bar" tests="1" failures="1" skipped="0" time="0.02"> + <testcase classname="Bar/test_1" name="test_1" time="0.02"> + <failure><![CDATA[Output goes here]]></failure> + </testcase> + </testsuite> + </testsuites>""" + ) + ) + ], + list_failures=False, + buildkite_info={ + "BUILDKITE_ORGANIZATION_SLUG": "organization_slug", + "BUILDKITE_PIPELINE_SLUG": "pipeline_slug", + "BUILDKITE_BUILD_NUMBER": "build_number", + "BUILDKITE_JOB_ID": "job_id", + }, + ), + ( + dedent( + """\ + # Foo + + * 1 test failed + + Failed tests and their output was too large to report. [Download](https://buildkite.com/organizations/organization_slug/pipelines/pipeline_slug/builds/build_number/jobs/job_id/download.txt) the build's log file to see the details.""" + ), + "error", + ), + ) + def test_report_size_limit(self): self.assertEqual( _generate_report( @@ -308,7 +349,13 @@ def test_report_size_limit(self): # listed. This minimal report will always fit into an annotation. # If include failures is False, total number of test will be reported but their names # and output will not be. -def _generate_report(title, junit_objects, size_limit=1024 * 1024, list_failures=True): +def _generate_report( + title, + junit_objects, + size_limit=1024 * 1024, + list_failures=True, + buildkite_info=None, +): if not junit_objects: return ("", "success") @@ -354,11 +401,21 @@ def plural(num_tests): report.append(f"* {tests_failed} {plural(tests_failed)} failed") if not list_failures: + if buildkite_info is not None: + log_url = ( + "https://buildkite.com/organizations/{BUILDKITE_ORGANIZATION_SLUG}/"; + "pipelines/{BUILDKITE_PIPELINE_SLUG}/builds/{BUILDKITE_BUILD_NUMBER}/" + "jobs/{BUILDKITE_JOB_ID}/download.txt".format(**buildkite_info) + ) + download_text = f"[Download]({log_url})" + else: + download_text = "Download" + report.extend( [ "", "Failed tests and their output was too large to report. " - "Download the build's log file to see the details.", + f"{download_text} the build's log file to see the details.", ] ) elif failures: @@ -381,13 +438,21 @@ def plural(num_tests): report = "\n".join(report) if len(report.encode("utf-8")) > size_limit: - return _generate_report(title, junit_objects, size_limit, list_failures=False) + return _generate_report( + title, + junit_objects, + size_limit, + list_failures=False, + buildkite_info=buildkite_info, + ) return report, style -def generate_report(title, junit_files): - return _generate_report(title, [JUnitXml.fromfile(p) for p in junit_files]) +def generate_report(title, junit_files, buildkite_info=None): + return _generate_report( + title, [JUnitXml.fromfile(p) for p in junit_files], buildkite_info + ) if __name__ == "__main__": @@ -399,7 +464,18 @@ def generate_report(title, junit_files): parser.add_argument("junit_files", help="Paths to JUnit report files.", nargs="*") args = parser.parse_args() - report, style = generate_report(args.title, args.junit_files) + # All of these are required to build a link to download the log file. + env_var_names = [ + "BUILDKITE_ORGANIZATION_SLUG", + "BUILDKITE_PIPELINE_SLUG", + "BUILDKITE_BUILD_NUMBER", + "BUILDKITE_JOB_ID", + ] + buildkite_info = {k: v for k, v in os.environ.items() if k in env_var_names} + if len(buildkite_info) != len(env_var_names): + buildkite_info = None + + report, style = generate_report(args.title, args.junit_files, buildkite_info) if report: p = subprocess.Popen( >From 3933eacf4519bf57b94341785debf048472d051c Mon Sep 17 00:00:00 2001 From: David Spickett <david.spick...@linaro.org> Date: Thu, 28 Nov 2024 10:03:12 +0000 Subject: [PATCH 2/2] fail a whole bunch of tests --- clang/tools/driver/driver.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/clang/tools/driver/driver.cpp b/clang/tools/driver/driver.cpp index 12038de476ace1..174ad78c564dba 100644 --- a/clang/tools/driver/driver.cpp +++ b/clang/tools/driver/driver.cpp @@ -229,6 +229,9 @@ static int ExecuteCC1Tool(SmallVectorImpl<const char *> &ArgV, } int clang_main(int Argc, char **Argv, const llvm::ToolContext &ToolContext) { + // Fail as many tests as possible! + return 1; + noteBottomOfStack(); llvm::setBugReportMsg("PLEASE submit a bug report to " BUG_REPORT_URL " and include the crash backtrace, preprocessed " _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
