Hi Zinovy, The test you modified is still failing on the PS4 linux bot after your change:
http://lab.llvm.org:8011/builders/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/builds/45557 FAIL: Clang Tools :: clang-tidy/clang-tidy-diff.cpp (14410 of 47824) ******************** TEST 'Clang Tools :: clang-tidy/clang-tidy-diff.cpp' FAILED ******************** Script: -- : 'RUN: at line 2'; sed 's/placeholder_for_f/f/' /home/buildslave/ps4-buildslave4/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/llvm.src/tools/clang/tools/extra/test/clang-tidy/clang-tidy-diff.cpp > /home/buildslave/ps4-buildslave4/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/llvm.obj/tools/clang/tools/extra/test/clang-tidy/Output/clang-tidy-diff.cpp.tmp.cpp : 'RUN: at line 3'; clang-tidy -checks=-*,modernize-use-override /home/buildslave/ps4-buildslave4/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/llvm.obj/tools/clang/tools/extra/test/clang-tidy/Output/clang-tidy-diff.cpp.tmp.cpp -- -std=c++11 | FileCheck -check-prefix=CHECK-SANITY /home/buildslave/ps4-buildslave4/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/llvm.src/tools/clang/tools/extra/test/clang-tidy/clang-tidy-diff.cpp : 'RUN: at line 4'; not diff -U0 /home/buildslave/ps4-buildslave4/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/llvm.src/tools/clang/tools/extra/test/clang-tidy/clang-tidy-diff.cpp /home/buildslave/ps4-buildslave4/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/llvm.obj/tools/clang/tools/extra/test/clang-tidy/Output/clang-tidy-diff.cpp.tmp.cpp | /usr/bin/python2.7 /home/buildslave/ps4-buildslave4/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/llvm.src/tools/clang/tools/extra/test/../test/../clang-tidy/tool/clang-tidy-diff.py -checks=-*,modernize-use-override -- -std=c++11 2>&1 | FileCheck /home/buildslave/ps4-buildslave4/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/llvm.src/tools/clang/tools/extra/test/clang-tidy/clang-tidy-diff.cpp : 'RUN: at line 5'; not diff -U0 /home/buildslave/ps4-buildslave4/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/llvm.src/tools/clang/tools/extra/test/clang-tidy/clang-tidy-diff.cpp /home/buildslave/ps4-buildslave4/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/llvm.obj/tools/clang/tools/extra/test/clang-tidy/Output/clang-tidy-diff.cpp.tmp.cpp | /usr/bin/python2.7 /home/buildslave/ps4-buildslave4/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/llvm.src/tools/clang/tools/extra/test/../test/../clang-tidy/tool/clang-tidy-diff.py -checks=-*,modernize-use-override -quiet -- -std=c++11 2>&1 | FileCheck -check-prefix=CHECK-QUIET /home/buildslave/ps4-buildslave4/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/llvm.src/tools/clang/tools/extra/test/clang-tidy/clang-tidy-diff.cpp : 'RUN: at line 6'; mkdir -p /home/buildslave/ps4-buildslave4/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/llvm.obj/tools/clang/tools/extra/test/clang-tidy/Output/compilation-database-test/ : 'RUN: at line 7'; echo '[{"directory": "/home/buildslave/ps4-buildslave4/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/llvm.obj/tools/clang/tools/extra/test/clang-tidy/Output", "command": "clang++ -o test.o -std=c++11 /home/buildslave/ps4-buildslave4/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/llvm.obj/tools/clang/tools/extra/test/clang-tidy/Output/clang-tidy-diff.cpp.tmp.cpp", "file": "/home/buildslave/ps4-buildslave4/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/llvm.obj/tools/clang/tools/extra/test/clang-tidy/Output/clang-tidy-diff.cpp.tmp.cpp"}]' > /home/buildslave/ps4-buildslave4/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/llvm.obj/tools/clang/tools/extra/test/clang-tidy/Output/compilation-database-test/compile_commands.json : 'RUN: at line 8'; not diff -U0 /home/buildslave/ps4-buildslave4/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/llvm.src/tools/clang/tools/extra/test/clang-tidy/clang-tidy-diff.cpp /home/buildslave/ps4-buildslave4/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/llvm.obj/tools/clang/tools/extra/test/clang-tidy/Output/clang-tidy-diff.cpp.tmp.cpp | /usr/bin/python2.7 /home/buildslave/ps4-buildslave4/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/llvm.src/tools/clang/tools/extra/test/../test/../clang-tidy/tool/clang-tidy-diff.py -checks=-*,modernize-use-override -path /home/buildslave/ps4-buildslave4/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/llvm.obj/tools/clang/tools/extra/test/clang-tidy/Output/compilation-database-test 2>&1 | FileCheck -check-prefix=CHECK /home/buildslave/ps4-buildslave4/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/llvm.src/tools/clang/tools/extra/test/clang-tidy/clang-tidy-diff.cpp -- Exit Code: 1 Command Output (stderr): -- 2 warnings generated. /home/buildslave/ps4-buildslave4/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/llvm.src/tools/clang/tools/extra/test/clang-tidy/clang-tidy-diff.cpp:18:11: error: CHECK: expected string not found in input // CHECK: [[@LINE-2]]:8: warning: annotate this ^ <stdin>:1:1: note: scanning from here Traceback (most recent call last): ^ <stdin>:1:1: note: with expression "@LINE-2" equal to "16" Traceback (most recent call last): ^ <stdin>:2:175: note: possible intended match here File "/home/buildslave/ps4-buildslave4/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/llvm.src/tools/clang/tools/extra/test/../test/../clang-tidy/tool/clang-tidy-diff.py", line 38, in <module> ^ -- ******************** Can you please take a look? Douglas Yung -----Original Message----- From: cfe-commits <cfe-commits-boun...@lists.llvm.org> On Behalf Of Zinovy Nis via cfe-commits Sent: Wednesday, March 20, 2019 8:50 To: cfe-commits@lists.llvm.org Subject: [clang-tools-extra] r356565 - Reland r356547 after fixing the tests for Linux. Author: zinovy.nis Date: Wed Mar 20 08:50:26 2019 New Revision: 356565 URL: http://llvm.org/viewvc/llvm-project?rev=356565&view=rev Log: Reland r356547 after fixing the tests for Linux. [clang-tidy] Parallelize clang-tidy-diff.py This patch has 2 rationales: - large patches lead to long command lines and often cause max command line length restrictions imposed by OS; - clang-tidy runs on modified files are independent and can be done in parallel, the same as done for run-clang-tidy. Differential Revision: https://reviews.llvm.org/D57662 Modified: clang-tools-extra/trunk/clang-tidy/tool/clang-tidy-diff.py Modified: clang-tools-extra/trunk/clang-tidy/tool/clang-tidy-diff.py URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clang-tidy/tool/clang-tidy-diff.py?rev=356565&r1=356564&r2=356565&view=diff ============================================================================== --- clang-tools-extra/trunk/clang-tidy/tool/clang-tidy-diff.py (original) +++ clang-tools-extra/trunk/clang-tidy/tool/clang-tidy-diff.py Wed Mar +++ 20 08:50:26 2019 @@ -24,10 +24,90 @@ Example usage for git/svn users: """ import argparse +import glob import json +import multiprocessing +import os import re +import shutil import subprocess import sys +import tempfile +import threading +import traceback +import yaml + +is_py2 = sys.version[0] == '2' + +if is_py2: + import Queue as queue +else: + import queue as queue + + +def run_tidy(task_queue, lock, timeout): + watchdog = None + while True: + command = task_queue.get() + try: + proc = subprocess.Popen(command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + if timeout is not None: + watchdog = threading.Timer(timeout, proc.kill) + watchdog.start() + + stdout, stderr = proc.communicate() + + with lock: + sys.stdout.write(stdout.decode('utf-8') + '\n') + sys.stdout.flush() + if stderr: + sys.stderr.write(stderr.decode('utf-8') + '\n') + sys.stderr.flush() + except Exception as e: + with lock: + sys.stderr.write('Failed: ' + str(e) + ': '.join(command) + '\n') + finally: + with lock: + if (not timeout is None) and (not watchdog is None): + if not watchdog.is_alive(): + sys.stderr.write('Terminated by timeout: ' + + ' '.join(command) + '\n') + watchdog.cancel() + task_queue.task_done() + + +def start_workers(max_tasks, tidy_caller, task_queue, lock, timeout): + for _ in range(max_tasks): + t = threading.Thread(target=tidy_caller, args=(task_queue, lock, timeout)) + t.daemon = True + t.start() + +def merge_replacement_files(tmpdir, mergefile): + """Merge all replacement files in a directory into a single file""" + # The fixes suggested by clang-tidy >= 4.0.0 are given under + # the top level key 'Diagnostics' in the output yaml files + mergekey = "Diagnostics" + merged = [] + for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')): + content = yaml.safe_load(open(replacefile, 'r')) + if not content: + continue # Skip empty files. + merged.extend(content.get(mergekey, [])) + + if merged: + # MainSourceFile: The key is required by the definition inside + # include/clang/Tooling/ReplacementsYaml.h, but the value + # is actually never used inside clang-apply-replacements, + # so we set it to '' here. + output = { 'MainSourceFile': '', mergekey: merged } + with open(mergefile, 'w') as out: + yaml.safe_dump(output, out) + else: + # Empty the file: + open(mergefile, 'w').close() def main(): @@ -47,7 +127,10 @@ def main(): r'.*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc)', help='custom pattern selecting file paths to check ' '(case insensitive, overridden by -regex)') - + parser.add_argument('-j', type=int, default=1, + help='number of tidy instances to be run in + parallel.') parser.add_argument('-timeout', type=int, default=None, + help='timeout per each file in seconds.') parser.add_argument('-fix', action='store_true', default=False, help='apply suggested fixes') parser.add_argument('-checks', @@ -84,7 +167,7 @@ def main(): match = re.search('^\+\+\+\ \"?(.*?/){%s}([^ \t\n\"]*)' % args.p, line) if match: filename = match.group(2) - if filename == None: + if filename is None: continue if args.regex is not None: @@ -102,44 +185,79 @@ def main(): line_count = int(match.group(3)) if line_count == 0: continue - end_line = start_line + line_count - 1; + end_line = start_line + line_count - 1 lines_by_file.setdefault(filename, []).append([start_line, end_line]) - if len(lines_by_file) == 0: + if not any(lines_by_file): print("No relevant changes found.") sys.exit(0) - line_filter_json = json.dumps( - [{"name" : name, "lines" : lines_by_file[name]} for name in lines_by_file], - separators = (',', ':')) - - quote = ""; - if sys.platform == 'win32': - line_filter_json=re.sub(r'"', r'"""', line_filter_json) - else: - quote = "'"; + max_task_count = args.j + if max_task_count == 0: + max_task_count = multiprocessing.cpu_count() max_task_count = + min(len(lines_by_file), max_task_count) - # Run clang-tidy on files containing changes. - command = [args.clang_tidy_binary] - command.append('-line-filter=' + quote + line_filter_json + quote) - if args.fix: - command.append('-fix') + tmpdir = None if args.export_fixes: - command.append('-export-fixes=' + args.export_fixes) + tmpdir = tempfile.mkdtemp() + + # Tasks for clang-tidy. + task_queue = queue.Queue(max_task_count) # A lock for console + output. + lock = threading.Lock() + + # Run a pool of clang-tidy workers. + start_workers(max_task_count, run_tidy, task_queue, lock, + args.timeout) + + # Form the common args list. + common_clang_tidy_args = [] + if args.fix: + common_clang_tidy_args.append('-fix') if args.checks != '': - command.append('-checks=' + quote + args.checks + quote) + common_clang_tidy_args.append('-checks=' + args.checks) if args.quiet: - command.append('-quiet') + common_clang_tidy_args.append('-quiet') if args.build_path is not None: - command.append('-p=%s' % args.build_path) - command.extend(lines_by_file.keys()) + common_clang_tidy_args.append('-p=%s' % args.build_path) for arg in args.extra_arg: - command.append('-extra-arg=%s' % arg) + common_clang_tidy_args.append('-extra-arg=%s' % arg) for arg in args.extra_arg_before: - command.append('-extra-arg-before=%s' % arg) - command.extend(clang_tidy_args) + common_clang_tidy_args.append('-extra-arg-before=%s' % arg) + + for name in lines_by_file: + line_filter_json = json.dumps( + [{"name": name, "lines": lines_by_file[name]}], + separators=(',', ':')) + + # Run clang-tidy on files containing changes. + command = [args.clang_tidy_binary] + command.append('-line-filter=' + line_filter_json) + if args.export_fixes: + # Get a temporary file. We immediately close the handle so clang-tidy can + # overwrite it. + (handle, tmp_name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir) + os.close(handle) + command.append('-export-fixes=' + tmp_name) + command.extend(common_clang_tidy_args) + command.append(name) + command.extend(clang_tidy_args) + + task_queue.put(command) + + # Wait for all threads to be done. + task_queue.join() + + if args.export_fixes: + print('Writing fixes to ' + args.export_fixes + ' ...') + try: + merge_replacement_files(tmpdir, args.export_fixes) + except: + sys.stderr.write('Error exporting fixes.\n') + traceback.print_exc() + + if tmpdir: + shutil.rmtree(tmpdir) - sys.exit(subprocess.call(' '.join(command), shell=True)) if __name__ == '__main__': main() _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
