On Mon, Aug 26, 2019 at 7:53 PM Richard Smith <rich...@metafoo.co.uk> wrote: > > Hi Aaron, > > I tried using this script to update a json dump test, and it seems to not > really work very well: > > 1) it's really inconvenient to invoke; you need to pass in a --clang, copy > some options out from two different places in the test file (from the RUN > line and from a "--filters" line), and guess how to form the proper command > line for the tool > 2) it generates a file with the wrong name (adding a -json before the > extension) > 3) it doesn't strip out the CHECK lines from the previous run of the tool > 4) it adds in a bunch of trailing whitespace, causing the file to not match > the one in the repository > > Have you had a chance to look at making this more usable? I think the > approach taken by utils/make-ast-dump-check.sh helps a lot: run the test in > its normal configuration to generate the output, and then replace FileCheck > with a tool that updates the CHECK lines instead of checking them. (That > saves you needing to pass in --clang and --opts, at least, and makes it > really easy to update a whole bunch of tests at once by running them all with > lit.)
Unfortunately, I've not had the chance to look into improving the script and the original author of the script has since moved on to other opportunities as well. I'll see if I can spend some time improving it, but shell scripting is not my strong suit (I'm predominately on Windows, so the Python script approach is a bit easier for me to grok). Does this have to use a shell script, or is Python still fine? ~Aaron > > On Wed, 19 Jun 2019 at 08:21, Aaron Ballman via cfe-commits > <cfe-commits@lists.llvm.org> wrote: >> >> Author: aaronballman >> Date: Wed Jun 19 08:25:24 2019 >> New Revision: 363820 >> >> URL: http://llvm.org/viewvc/llvm-project?rev=363820&view=rev >> Log: >> Add a script to help generate expected test output for dumping the AST to >> JSON. >> >> Patch by Abhishek Bhaskar. >> >> Added: >> cfe/trunk/test/AST/gen_ast_dump_json_test.py >> >> Added: cfe/trunk/test/AST/gen_ast_dump_json_test.py >> URL: >> http://llvm.org/viewvc/llvm-project/cfe/trunk/test/AST/gen_ast_dump_json_test.py?rev=363820&view=auto >> ============================================================================== >> --- cfe/trunk/test/AST/gen_ast_dump_json_test.py (added) >> +++ cfe/trunk/test/AST/gen_ast_dump_json_test.py Wed Jun 19 08:25:24 2019 >> @@ -0,0 +1,137 @@ >> +#!/usr/bin/env python >> + >> +from collections import OrderedDict >> +from sets import Set >> +from shutil import copyfile >> +import argparse >> +import json >> +import os >> +import pprint >> +import re >> +import subprocess >> + >> +def normalize(dict_var): >> + for k, v in dict_var.items(): >> + if isinstance(v, OrderedDict): >> + normalize(v) >> + elif isinstance(v, list): >> + for e in v: >> + if isinstance(e, OrderedDict): >> + normalize(e) >> + elif type(v) is unicode: >> + st = v.encode('utf-8') >> + if re.match(r"0x[0-9A-Fa-f]+", v): >> + dict_var[k] = u'0x{{.*}}' >> + elif os.path.isfile(v): >> + dict_var[k] = u'{{.*}}' >> + else: >> + splits = (v.split(u' ')) >> + out_splits = [] >> + for split in splits: >> + inner_splits = split.rsplit(u':',2) >> + if os.path.isfile(inner_splits[0]): >> + out_splits.append( >> + u'{{.*}}:%s:%s' >> + %(inner_splits[1], >> + inner_splits[2])) >> + continue >> + out_splits.append(split) >> + >> + dict_var[k] = ' '.join(out_splits) >> + >> +def filter_json(dict_var, filters, out): >> + for k, v in dict_var.items(): >> + if type(v) is unicode: >> + st = v.encode('utf-8') >> + if st in filters: >> + out.append(dict_var) >> + break >> + elif isinstance(v, OrderedDict): >> + filter_json(v, filters, out) >> + elif isinstance(v, list): >> + for e in v: >> + if isinstance(e, OrderedDict): >> + filter_json(e, filters, out) >> + >> +def main(): >> + parser = argparse.ArgumentParser() >> + parser.add_argument("--clang", help="The clang binary (could be a >> relative or absolute path)", >> + action="store", required=True) >> + parser.add_argument("--opts", help="other options", >> + action="store", default='', type=str) >> + parser.add_argument("--source", help="the source file. Command used to >> generate the json will be of the format <clang> -cc1 -ast-dump=json <opts> >> <source>", >> + action="store", required=True) >> + parser.add_argument("--filters", help="comma separated list of AST >> filters. Ex: --filters=TypedefDecl,BuiltinType", >> + action="store", default='') >> + >> + args = parser.parse_args() >> + >> + if not args.source: >> + print("Specify the source file to give to clang.") >> + return -1 >> + >> + clang_binary = os.path.abspath(args.clang) >> + if not os.path.isfile(clang_binary): >> + print("clang binary specified not present.") >> + return -1 >> + >> + options = args.opts.split(' ') >> + filters = Set(args.filters.split(',')) if args.filters else Set([]) >> + >> + cmd = [clang_binary, "-cc1"] >> + cmd.extend(options) >> + >> + using_ast_dump_filter = 'ast-dump-filter' in args.opts >> + >> + cmd.extend(["-ast-dump=json", args.source]) >> + >> + try: >> + json_str = subprocess.check_output(cmd) >> + except Exception as ex: >> + print("The clang command failed with %s" % ex) >> + return -1 >> + >> + out_asts = [] >> + if using_ast_dump_filter: >> + splits = re.split('Dumping .*:\n', json_str) >> + if len(splits) > 1: >> + for split in splits[1:]: >> + j = json.loads(split.decode('utf-8'), >> object_pairs_hook=OrderedDict) >> + normalize(j) >> + out_asts.append(j) >> + else: >> + j = json.loads(json_str.decode('utf-8'), >> object_pairs_hook=OrderedDict) >> + normalize(j) >> + >> + if len(filters) == 0: >> + out_asts.append(j) >> + else: >> + #assert using_ast_dump_filter is False,\ >> + # "Does not support using compiler's ast-dump-filter "\ >> + # "and the tool's filter option at the same time yet." >> + >> + filter_json(j, filters, out_asts) >> + >> + partition = args.source.rpartition('.') >> + dest_path = '%s-json%s%s' % (partition[0], partition[1], partition[2]) >> + >> + print("Writing json appended source file to %s." %(dest_path)) >> + copyfile(args.source, dest_path) >> + with open(dest_path, "a") as f: >> + for out_ast in out_asts: >> + append_str = json.dumps(out_ast, indent=1, ensure_ascii=False) >> + out_str = '\n\n' >> + index = 0 >> + for append_line in append_str.splitlines()[2:]: >> + if index == 0: >> + out_str += '// CHECK: %s\n' %(append_line) >> + index += 1 >> + else: >> + out_str += '// CHECK-NEXT: %s\n' %(append_line) >> + >> + f.write(out_str) >> + >> + return 0 >> + >> +if __name__ == '__main__': >> + main() >> >> >> _______________________________________________ >> cfe-commits mailing list >> cfe-commits@lists.llvm.org >> https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
