bin/find-unneeded-includes | 208 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 200 insertions(+), 8 deletions(-)
New commits: commit a741dd2cd5059e3d6a06307fb7c6c016b4099c81 Author: Gabor Kelemen <[email protected]> AuthorDate: Thu Mar 14 12:40:53 2024 +0100 Commit: Miklos Vajna <[email protected]> CommitDate: Mon Mar 18 09:21:27 2024 +0100 find-unneeded-includes: add option to detect unneeded namespaces Sometimes there are unneeded namespaces included, these can be detected from the IWYU output. This mode makes a suggestion to remove these, then in a subsequent normal run some more headers can be detected as unnecessary, whose presence was justified by the "using namespace" statement. Change-Id: I45616537925ec0d09039edf3d9237ffbd13e2410 Reviewed-on: https://gerrit.libreoffice.org/c/core/+/164939 Tested-by: Jenkins Reviewed-by: Miklos Vajna <[email protected]> diff --git a/bin/find-unneeded-includes b/bin/find-unneeded-includes index 12659fa82a31..439bb5230418 100755 --- a/bin/find-unneeded-includes +++ b/bin/find-unneeded-includes @@ -153,11 +153,12 @@ def unwrapInclude(include): return include[1:-1] -def processIWYUOutput(iwyuOutput, moduleRules, fileName, noexclude): +def processIWYUOutput(iwyuOutput, moduleRules, fileName, noexclude, checknamespaces): inAdd = False toAdd = [] inRemove = False toRemove = [] + inFull = False currentFileName = None for line in iwyuOutput: @@ -174,6 +175,9 @@ def processIWYUOutput(iwyuOutput, moduleRules, fileName, noexclude): if inAdd: inAdd = False continue + if inFull: + inFull = False + continue shouldAdd = fileName + " should add these lines:" match = re.match(shouldAdd, line) @@ -189,6 +193,12 @@ def processIWYUOutput(iwyuOutput, moduleRules, fileName, noexclude): inRemove = True continue + if checknamespaces: + match = re.match("The full include-list for " + fileName, line) + if match: + inFull = True + continue + if inAdd: match = re.match('#include ([^ ]+)', line) if match: @@ -198,7 +208,7 @@ def processIWYUOutput(iwyuOutput, moduleRules, fileName, noexclude): # Forward declaration. toAdd.append(line) - if inRemove: + if inRemove and not checknamespaces: match = re.match("- #include (.*) // lines (.*)-.*", line) if match: # Only suggest removals for now. Removing fwd decls is more complex: they may be @@ -209,25 +219,203 @@ def processIWYUOutput(iwyuOutput, moduleRules, fileName, noexclude): if not ignoreRemoval(include, toAdd, currentFileName, moduleRules, noexclude): toRemove.append("%s:%s: %s" % (currentFileName, lineno, include)) + if inFull: + if checknamespaces: + # match for all possible URE/UNO namespaces, created with: + # find udkapi/com/sun/star/ -type d | sort| xargs basename -a | tr ' ' '|' + # find offapi/com/sun/star/ -type d | sort | xargs basename -a | tr ' ' '|' + # and ooo::vba namespaces + # plus a few popular ones about other modules + ns = re.compile( + '.*for\ (' + # URE namespaces + 'beans|' + 'bridge|oleautomation|' + 'connection|' + 'container|' + 'io|' + 'java|' + 'lang|' + 'loader|' + 'reflection|' + 'registry|' + 'script|' + 'security|' + 'task|' + 'uno|' + 'uri|' + 'util|' + # UNO namespaces + 'accessibility|' + 'animations|' + 'auth|' + 'awt|tab|tree|grid|' + 'chart|' + 'chart2|data|' + 'configuration|bootstrap|backend|xml|' + 'cui|' + 'datatransfer|clipboard|dnd|' + 'deployment|test|ui|' + 'document|' + 'drawing|framework|' + 'embed|' + 'form|binding|runtime|control|inspection|submission|component|validation|' + 'formula|' + 'frame|status|' + 'gallery|' + 'geometry|' + 'graphic|' + 'i18n|' + 'image|' + 'inspection|' + 'ldap|' + 'linguistic2|' + 'logging|' + 'mail|' + 'media|' + 'mozilla|' + 'office|' + 'packages|zip|manifest|' + 'presentation|textfield|' + 'qa|' + 'rdf|' + 'rendering|' + 'report|inspection|meta|' + 'resource|' + 'scanner|' + 'script|vba|browse|provider|' + 'sdb|application|tools|' + 'sdbc|' + 'sdbcx|' + 'security|' + 'setup|' + 'sheet|opencl|' + 'smarttags|' + 'style|' + 'svg|' + 'system|windows|' + 'table|' + 'task|' + 'text|textfield|docinfo|fieldmaster|' + 'tiledrendering|' + 'ucb|' + 'ui|dialogs|test|' + 'util|' + 'view|' + 'xforms|' + 'xml|xslt|wrapper|csax|sax|input|xpath|dom|views|events|crypto|sax|' + 'xsd|' + # ooo::vba and its namespaces + 'ooo|vba|excel|powerpoint|adodb|access|office|word|stdole|msforms|dao|' + # use of module namespaces, as spotted in the code + 'analysis|pricing' # sca internals + 'apphelper|CloneHelper|DataSeriesProperties|SceneProperties|wrapper|' # for chart internals + 'basegfx|utils|' + 'boost|posix_time|gregorian' + 'cairo|' + 'canvas|' + 'chelp|' + 'comphelper|' + 'connectivity|' + 'cpp|java|' # for codemaker:: + 'cppu|' + 'dbaccess|dbahsql|dbaui|dbtools|' + 'desktop|dp_misc|' + 'drawinglayer|attribute|geometry|primitive2d|processor2d|' + 'editeng|' + 'emscripten|' + 'formula|' + 'framework|' + 'frm|' + 'http_dav_ucp|tdoc_ucp|package_ucp|hierarchy_ucp|gio|fileaccess|ucb_impl|hcp_impl|ucb_cmdenv|' # for ucb internal + 'i18npool|' + 'internal|ColorComponentTag|' # for slideshow internals + 'jfw_plugin|' + 'jni_uno|' + 'librevenge|' + 'linguistic|' + 'lok|' + 'mtv|' # for mdds::mtv + 'nsSwDocInfoSubType|SWUnoHelper|nsHdFtFlags|' # sw internal + 'o3tl|' + 'odfflatxml|' # filter internal + 'oox|core|drawingml|ole|vml|' + 'OpenStormBento|' + 'osl|' + 'PackageKit|' + 'pdfi|pdfparse|' + 'ppt|' + 'pyuno|' + 'reportdesign|' + 'rptui|' + 'rtl|math|textenc|' + 'salhelper|' + 'sax_fastparser|' + 'sax|' # for xml::sax + 'sc|' + 'SchXMLTools|' # for xmloff + 'sd|slidesorter|cache|controller|model|view|' + 'sf_misc|' + 'sfx2|DocTempl|' + 'sidebar|' # for sfx2::sidebar + 'skeletonmaker|' + 'std|chrono_literals|literals|' + 'stoc_sec|' + 'store|' + 'svl|impl|' + 'svt|' + 'svtools|' + 'svx|sdr|contact|table|' + 'sw|access|annotation|mark|types|util|' + 'toolkit|' + 'treeview|' + 'ucbhelper|' + 'unodevtools' + 'unopkg|' + 'util|db|qe|' # for xmlsearch:: + 'utl|' + 'vcl|' + 'writerfilter|' + 'xforms|' + 'xmloff|token|EnhancedCustomShapeToken' # for xmloff:: + 'ZipUtils' + ')$', re.VERBOSE + ) + + reason = re.match(ns, line) + if reason: + # Warn about namespaces: if a header is suggested only '// for $namespace', then the namespace is not used + # otherwise the used classes name would show up after the '// for' + # Cleaning out the respective header (if ther is any + # - which is not always the case) is for the next run! + nameSpace = reason.group(1).split(' ')[0] + print("WARNING:", fileName, "This 'using namespace' is likely unnecessary:", nameSpace) + + # Get the row number, normal IWYU output does not contain this info + subprocess.run(["git", "grep", "-n", "using namespace.*"+nameSpace+";", fileName]) + for remove in sorted(toRemove): print("ERROR: %s: remove not needed include" % remove) return len(toRemove) -def run_tool(task_queue, failed_files, dontstop, noexclude): +def run_tool(task_queue, failed_files, dontstop, noexclude, checknamespaces): while True: invocation, moduleRules = task_queue.get() if not len(failed_files): print("[IWYU] " + invocation.split(' ')[-1]) p = subprocess.Popen(invocation, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - retcode = processIWYUOutput(p.communicate()[0].decode('utf-8').splitlines(), moduleRules, invocation.split(' ')[-1], noexclude) - if retcode == -1: + retcode = processIWYUOutput(p.communicate()[0].decode('utf-8').splitlines(), moduleRules, invocation.split(' ')[-1], noexclude, checknamespaces) + if retcode == -1 and not checknamespaces: print("ERROR: A file is probably not self contained, check this commands output: " + invocation) elif retcode > 0: print("ERROR: The following command found unused includes: " + invocation) if not dontstop: failed_files.append(invocation) task_queue.task_done() + if checknamespaces: + # Workaround: sometimes running git grep makes the letters typed into the terminal disappear after the script is finished + os.system('stty sane') def isInUnoIncludeFile(path): @@ -243,7 +431,7 @@ def isInUnoIncludeFile(path): or path.startswith("include/uno/") -def tidy(compileCommands, paths, dontstop, noexclude): +def tidy(compileCommands, paths, dontstop, noexclude,checknamespaces): return_code = 0 try: @@ -251,7 +439,7 @@ def tidy(compileCommands, paths, dontstop, noexclude): task_queue = queue.Queue(max_task) failed_files = [] for _ in range(max_task): - t = threading.Thread(target=run_tool, args=(task_queue, failed_files, dontstop, noexclude)) + t = threading.Thread(target=run_tool, args=(task_queue, failed_files, dontstop, noexclude,checknamespaces)) t.daemon = True t.start() @@ -319,6 +507,10 @@ def main(argv): help='Check header files. If omitted, check source files. Use with --recursive.') parser.add_argument('--noexclude', action='store_true', help='Ignore excludelist. Useful to check whether its exclusions are still all valid.') + parser.add_argument('--ns', action='store_true', + help='Warn about unused "using namespace" statements. ' + 'Removing these may uncover more removable headers ' + 'in a subsequent normal run') args = parser.parse_args() @@ -367,7 +559,7 @@ def main(argv): if not file.exists(): print("WARNING: File listed in " + rulePath + " no longer exists: " + pathname) - tidy(compileCommands, paths=list_of_files, dontstop=vars(args)["continue"], noexclude=args.noexclude) + tidy(compileCommands, paths=list_of_files, dontstop=vars(args)["continue"], noexclude=args.noexclude, checknamespaces=args.ns) if __name__ == '__main__': main(sys.argv[1:])
