Hi Zinovy, I have reverted this change in r356630 in order to get the build bots back to green.
I was able to reproduce the issue locally on my machine, it appears that your use of “import yaml” is not part of the standard python distribution and so is not found. Douglas Yung From: cfe-commits <cfe-commits-boun...@lists.llvm.org> On Behalf Of Galina Kistanova via cfe-commits Sent: Wednesday, March 20, 2019 14:03 To: Zinovy Nis <zinovy....@gmail.com> Cc: cfe-commits <cfe-commits@lists.llvm.org> Subject: Re: [clang-tools-extra] r356565 - Reland r356547 after fixing the tests for Linux. Hello Zinovy, This commit broke test on the next builder: http://lab.llvm.org:8011/builders/llvm-clang-lld-x86_64-scei-ps4-ubuntu-fast/builds/45557 . . . Failing Tests (1): Clang Tools :: clang-tidy/clang-tidy-diff.cpp Please have a look? Thanks Galina On Wed, Mar 20, 2019 at 8:49 AM Zinovy Nis via cfe-commits <cfe-commits@lists.llvm.org<mailto:cfe-commits@lists.llvm.org>> wrote: Author: zinovy.nis Date: Wed Mar 20 08:50:26 2019 New Revision: 356565 URL: http://llvm.org/viewvc/llvm-project?rev=356565&view=rev Log: Reland r356547 after fixing the tests for Linux. [clang-tidy] Parallelize clang-tidy-diff.py This patch has 2 rationales: - large patches lead to long command lines and often cause max command line length restrictions imposed by OS; - clang-tidy runs on modified files are independent and can be done in parallel, the same as done for run-clang-tidy. Differential Revision: https://reviews.llvm.org/D57662 Modified: clang-tools-extra/trunk/clang-tidy/tool/clang-tidy-diff.py Modified: clang-tools-extra/trunk/clang-tidy/tool/clang-tidy-diff.py URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clang-tidy/tool/clang-tidy-diff.py?rev=356565&r1=356564&r2=356565&view=diff ============================================================================== --- clang-tools-extra/trunk/clang-tidy/tool/clang-tidy-diff.py (original) +++ clang-tools-extra/trunk/clang-tidy/tool/clang-tidy-diff.py Wed Mar 20 08:50:26 2019 @@ -24,10 +24,90 @@ Example usage for git/svn users: """ import argparse +import glob import json +import multiprocessing +import os import re +import shutil import subprocess import sys +import tempfile +import threading +import traceback +import yaml + +is_py2 = sys.version[0] == '2' + +if is_py2: + import Queue as queue +else: + import queue as queue + + +def run_tidy(task_queue, lock, timeout): + watchdog = None + while True: + command = task_queue.get() + try: + proc = subprocess.Popen(command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + if timeout is not None: + watchdog = threading.Timer(timeout, proc.kill) + watchdog.start() + + stdout, stderr = proc.communicate() + + with lock: + sys.stdout.write(stdout.decode('utf-8') + '\n') + sys.stdout.flush() + if stderr: + sys.stderr.write(stderr.decode('utf-8') + '\n') + sys.stderr.flush() + except Exception as e: + with lock: + sys.stderr.write('Failed: ' + str(e) + ': '.join(command) + '\n') + finally: + with lock: + if (not timeout is None) and (not watchdog is None): + if not watchdog.is_alive(): + sys.stderr.write('Terminated by timeout: ' + + ' '.join(command) + '\n') + watchdog.cancel() + task_queue.task_done() + + +def start_workers(max_tasks, tidy_caller, task_queue, lock, timeout): + for _ in range(max_tasks): + t = threading.Thread(target=tidy_caller, args=(task_queue, lock, timeout)) + t.daemon = True + t.start() + +def merge_replacement_files(tmpdir, mergefile): + """Merge all replacement files in a directory into a single file""" + # The fixes suggested by clang-tidy >= 4.0.0 are given under + # the top level key 'Diagnostics' in the output yaml files + mergekey = "Diagnostics" + merged = [] + for replacefile in glob.iglob(os.path.join(tmpdir, '*.yaml')): + content = yaml.safe_load(open(replacefile, 'r')) + if not content: + continue # Skip empty files. + merged.extend(content.get(mergekey, [])) + + if merged: + # MainSourceFile: The key is required by the definition inside + # include/clang/Tooling/ReplacementsYaml.h, but the value + # is actually never used inside clang-apply-replacements, + # so we set it to '' here. + output = { 'MainSourceFile': '', mergekey: merged } + with open(mergefile, 'w') as out: + yaml.safe_dump(output, out) + else: + # Empty the file: + open(mergefile, 'w').close() def main(): @@ -47,7 +127,10 @@ def main(): r'.*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc)', help='custom pattern selecting file paths to check ' '(case insensitive, overridden by -regex)') - + parser.add_argument('-j', type=int, default=1, + help='number of tidy instances to be run in parallel.') + parser.add_argument('-timeout', type=int, default=None, + help='timeout per each file in seconds.') parser.add_argument('-fix', action='store_true', default=False, help='apply suggested fixes') parser.add_argument('-checks', @@ -84,7 +167,7 @@ def main(): match = re.search('^\+\+\+\ \"?(.*?/){%s}([^ \t\n\"]*)' % args.p, line) if match: filename = match.group(2) - if filename == None: + if filename is None: continue if args.regex is not None: @@ -102,44 +185,79 @@ def main(): line_count = int(match.group(3)) if line_count == 0: continue - end_line = start_line + line_count - 1; + end_line = start_line + line_count - 1 lines_by_file.setdefault(filename, []).append([start_line, end_line]) - if len(lines_by_file) == 0: + if not any(lines_by_file): print("No relevant changes found.") sys.exit(0) - line_filter_json = json.dumps( - [{"name" : name, "lines" : lines_by_file[name]} for name in lines_by_file], - separators = (',', ':')) - - quote = ""; - if sys.platform == 'win32': - line_filter_json=re.sub(r'"', r'"""', line_filter_json) - else: - quote = "'"; + max_task_count = args.j + if max_task_count == 0: + max_task_count = multiprocessing.cpu_count() + max_task_count = min(len(lines_by_file), max_task_count) - # Run clang-tidy on files containing changes. - command = [args.clang_tidy_binary] - command.append('-line-filter=' + quote + line_filter_json + quote) - if args.fix: - command.append('-fix') + tmpdir = None if args.export_fixes: - command.append('-export-fixes=' + args.export_fixes) + tmpdir = tempfile.mkdtemp() + + # Tasks for clang-tidy. + task_queue = queue.Queue(max_task_count) + # A lock for console output. + lock = threading.Lock() + + # Run a pool of clang-tidy workers. + start_workers(max_task_count, run_tidy, task_queue, lock, args.timeout) + + # Form the common args list. + common_clang_tidy_args = [] + if args.fix: + common_clang_tidy_args.append('-fix') if args.checks != '': - command.append('-checks=' + quote + args.checks + quote) + common_clang_tidy_args.append('-checks=' + args.checks) if args.quiet: - command.append('-quiet') + common_clang_tidy_args.append('-quiet') if args.build_path is not None: - command.append('-p=%s' % args.build_path) - command.extend(lines_by_file.keys()) + common_clang_tidy_args.append('-p=%s' % args.build_path) for arg in args.extra_arg: - command.append('-extra-arg=%s' % arg) + common_clang_tidy_args.append('-extra-arg=%s' % arg) for arg in args.extra_arg_before: - command.append('-extra-arg-before=%s' % arg) - command.extend(clang_tidy_args) + common_clang_tidy_args.append('-extra-arg-before=%s' % arg) + + for name in lines_by_file: + line_filter_json = json.dumps( + [{"name": name, "lines": lines_by_file[name]}], + separators=(',', ':')) + + # Run clang-tidy on files containing changes. + command = [args.clang_tidy_binary] + command.append('-line-filter=' + line_filter_json) + if args.export_fixes: + # Get a temporary file. We immediately close the handle so clang-tidy can + # overwrite it. + (handle, tmp_name) = tempfile.mkstemp(suffix='.yaml', dir=tmpdir) + os.close(handle) + command.append('-export-fixes=' + tmp_name) + command.extend(common_clang_tidy_args) + command.append(name) + command.extend(clang_tidy_args) + + task_queue.put(command) + + # Wait for all threads to be done. + task_queue.join() + + if args.export_fixes: + print('Writing fixes to ' + args.export_fixes + ' ...') + try: + merge_replacement_files(tmpdir, args.export_fixes) + except: + sys.stderr.write('Error exporting fixes.\n') + traceback.print_exc() + + if tmpdir: + shutil.rmtree(tmpdir) - sys.exit(subprocess.call(' '.join(command), shell=True)) if __name__ == '__main__': main() _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org<mailto:cfe-commits@lists.llvm.org> https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
_______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
