bin/find-unneeded-includes

   1 #!/usr/bin/env python3
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6 #
   7 # This parses the output of 'include-what-you-use', focusing on just removing
   8 # not needed includes and providing a relatively conservative output by
   9 # filtering out a number of LibreOffice-specific false positives.
  10 #
  11 # It assumes you have a 'compile_commands.json' around (similar to clang-tidy),
  12 # you can generate one with 'make vim-ide-integration'.
  13 #
  14 # Design goals:
  15 # - excludelist mechanism, so a warning is either fixed or excluded
  16 # - works in a plugins-enabled clang build
  17 # - no custom configure options required
  18 # - no need to generate a dummy library to build a header
  19
  20 import glob
  21 import json
  22 import multiprocessing
  23 import os
  24 import queue
  25 import re
  26 import subprocess
  27 import sys
  28 import threading
  29 import yaml
  30 import argparse
  31 import pathlib
  32
  33
  34 def ignoreRemoval(include, toAdd, absFileName, moduleRules, noexclude):
  35     # global rules
  36
  37     # Avoid replacing .hpp with .hdl in the com::sun::star and  ooo::vba namespaces.
  38     if ( include.startswith("com/sun/star") or include.startswith("ooo/vba") ) and include.endswith(".hpp"):
  39         hdl = include.replace(".hpp", ".hdl")
  40         if hdl in toAdd:
  41             return True
  42
  43     # Avoid debug STL.
  44     debugStl = {
  45         "array": ("debug/array", ),
  46         "bitset": ("debug/bitset", ),
  47         "deque": ("debug/deque", ),
  48         "forward_list": ("debug/forward_list", ),
  49         "list": ("debug/list", ),
  50         "map": ("debug/map.h", "debug/multimap.h"),
  51         "set": ("debug/set.h", "debug/multiset.h"),
  52         "unordered_map": ("debug/unordered_map", ),
  53         "unordered_set": ("debug/unordered_set", ),
  54         "vector": ("debug/vector", ),
  55     }
  56     for k, values in debugStl.items():
  57         if include == k:
  58             for value in values:
  59                 if value in toAdd:
  60                     return True
  61
  62     # Avoid proposing to use libstdc++ internal headers.
  63     bits = {
  64         "exception": "bits/exception.h",
  65         "memory": "bits/shared_ptr.h",
  66         "functional": "bits/std_function.h",
  67         "cmath": "bits/std_abs.h",
  68         "ctime": "bits/types/clock_t.h",
  69         "cstdint": "bits/stdint-uintn.h",
  70     }
  71     for k, v in bits.items():
  72         if include == k and v in toAdd:
  73             return True
  74
  75     # Avoid proposing o3tl fw declaration
  76     o3tl = {
  77         "o3tl/typed_flags_set.hxx" : "namespace o3tl { template <typename T> struct typed_flags; }",
  78         "o3tl/deleter.hxx" : "namespace o3tl { template <typename T> struct default_delete; }",
  79         "o3tl/span.hxx" : "namespace o3tl { template <typename T> class span; }",
  80     }
  81     for k, v, in o3tl.items():
  82         if include == k and v in toAdd:
  83             return True
  84
  85     # Follow boost documentation.
  86     if include == "boost/optional.hpp" and "boost/optional/optional.hpp" in toAdd:
  87         return True
  88     if include == "boost/intrusive_ptr.hpp" and "boost/smart_ptr/intrusive_ptr.hpp" in toAdd:
  89         return True
  90     if include == "boost/shared_ptr.hpp" and "boost/smart_ptr/shared_ptr.hpp" in toAdd:
  91         return True
  92     if include == "boost/variant.hpp" and "boost/variant/variant.hpp" in toAdd:
  93         return True
  94     if include == "boost/unordered_map.hpp" and "boost/unordered/unordered_map.hpp" in toAdd:
  95         return True
  96     if include == "boost/functional/hash.hpp" and "boost/container_hash/extensions.hpp" in toAdd:
  97         return True
  98
  99     # Avoid .hxx to .h proposals in basic css/uno/* API
 100     unoapi = {
 101         "com/sun/star/uno/Any.hxx": "com/sun/star/uno/Any.h",
 102         "com/sun/star/uno/Reference.hxx": "com/sun/star/uno/Reference.h",
 103         "com/sun/star/uno/Sequence.hxx": "com/sun/star/uno/Sequence.h",
 104         "com/sun/star/uno/Type.hxx": "com/sun/star/uno/Type.h"
 105     }
 106     for k, v in unoapi.items():
 107         if include == k and v in toAdd:
 108             return True
 109
 110     # 3rd-party, non-self-contained headers.
 111     if include == "libepubgen/libepubgen.h" and "libepubgen/libepubgen-decls.h" in toAdd:
 112         return True
 113     if include == "librevenge/librevenge.h" and "librevenge/RVNGPropertyList.h" in toAdd:
 114         return True
 115     if include == "libetonyek/libetonyek.h" and "libetonyek/EtonyekDocument.h" in toAdd:
 116         return True
 117
 118     noRemove = (
 119         # <https://www.openoffice.org/tools/CodingGuidelines.sxw> insists on not
 120         # removing this.
 121         "sal/config.h",
 122         # Works around a build breakage specific to the broken Android
 123         # toolchain.
 124         "android/compatibility.hxx",
 125         # Removing this would change the meaning of '#if defined OSL_BIGENDIAN'.
 126         "osl/endian.h",
 127     )
 128     if include in noRemove:
 129         return True
 130
 131     # Ignore when <foo> is to be replaced with "foo".
 132     if include in toAdd:
 133         return True
 134
 135     fileName = os.path.relpath(absFileName, os.getcwd())
 136
 137     # Skip headers used only for compile test
 138     if fileName == "cppu/qa/cppumaker/test_cppumaker.cxx":
 139         if include.endswith(".hpp"):
 140             return True
 141
 142     # yaml rules, except when --noexclude is given
 143
 144     if "excludelist" in moduleRules.keys() and not noexclude:
 145         excludelistRules = moduleRules["excludelist"]
 146         if fileName in excludelistRules.keys():
 147             if include in excludelistRules[fileName]:
 148                 return True
 149
 150     return False
 151
 152
 153 def unwrapInclude(include):
 154     # Drop <> or "" around the include.
 155     return include[1:-1]
 156
 157
 158 def processIWYUOutput(iwyuOutput, moduleRules, fileName, noexclude):
 159     inAdd = False
 160     toAdd = []
 161     inRemove = False
 162     toRemove = []
 163     currentFileName = None
 164
 165     for line in iwyuOutput:
 166         line = line.strip()
 167
 168         # Bail out if IWYU gave an error due to non self-containedness
 169         if re.match ("(.*): error: (.*)", line):
 170             return -1
 171
 172         if len(line) == 0:
 173             if inRemove:
 174                 inRemove = False
 175                 continue
 176             if inAdd:
 177                 inAdd = False
 178                 continue
 179
 180         shouldAdd = fileName + " should add these lines:"
 181         match = re.match(shouldAdd, line)
 182         if match:
 183             currentFileName = match.group(0).split(' ')[0]
 184             inAdd = True
 185             continue
 186
 187         shouldRemove = fileName + " should remove these lines:"
 188         match = re.match(shouldRemove, line)
 189         if match:
 190             currentFileName = match.group(0).split(' ')[0]
 191             inRemove = True
 192             continue
 193
 194         if inAdd:
 195             match = re.match('#include ([^ ]+)', line)
 196             if match:
 197                 include = unwrapInclude(match.group(1))
 198                 toAdd.append(include)
 199             else:
 200                 # Forward declaration.
 201                 toAdd.append(line)
 202
 203         if inRemove:
 204             match = re.match("- #include (.*)  // lines (.*)-.*", line)
 205             if match:
 206                 # Only suggest removals for now. Removing fwd decls is more complex: they may be
 207                 # indeed unused or they may removed to be replaced with an include. And we want to
 208                 # avoid the later.
 209                 include = unwrapInclude(match.group(1))
 210                 lineno = match.group(2)
 211                 if not ignoreRemoval(include, toAdd, currentFileName, moduleRules, noexclude):
 212                     toRemove.append("%s:%s: %s" % (currentFileName, lineno, include))
 213
 214     for remove in sorted(toRemove):
 215         print("ERROR: %s: remove not needed include" % remove)
 216     return len(toRemove)
 217
 218
 219 def run_tool(task_queue, failed_files, dontstop, noexclude):
 220     while True:
 221         invocation, moduleRules = task_queue.get()
 222         if not len(failed_files):
 223             print("[IWYU] " + invocation.split(' ')[-1])
 224             p = subprocess.Popen(invocation, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 225             retcode = processIWYUOutput(p.communicate()[0].decode('utf-8').splitlines(), moduleRules, invocation.split(' ')[-1], noexclude)
 226             if retcode == -1:
 227                 print("ERROR: A file is probably not self contained, check this commands output:\n" + invocation)
 228             elif retcode > 0:
 229                 print("ERROR: The following command found unused includes:\n" + invocation)
 230                 if not dontstop:
 231                     failed_files.append(invocation)
 232         task_queue.task_done()
 233
 234
 235 def isInUnoIncludeFile(path):
 236     return path.startswith("include/com/") \
 237             or path.startswith("include/cppu/") \
 238             or path.startswith("include/cppuhelper/") \
 239             or path.startswith("include/osl/") \
 240             or path.startswith("include/rtl/") \
 241             or path.startswith("include/sal/") \
 242             or path.startswith("include/salhelper/") \
 243             or path.startswith("include/systools/") \
 244             or path.startswith("include/typelib/") \
 245             or path.startswith("include/uno/")
 246
 247
 248 def tidy(compileCommands, paths, dontstop, noexclude):
 249     return_code = 0
 250
 251     try:
 252         max_task = multiprocessing.cpu_count()
 253         task_queue = queue.Queue(max_task)
 254         failed_files = []
 255         for _ in range(max_task):
 256             t = threading.Thread(target=run_tool, args=(task_queue, failed_files, dontstop, noexclude))
 257             t.daemon = True
 258             t.start()
 259
 260         for path in sorted(paths):
 261             if isInUnoIncludeFile(path):
 262                 continue
 263
 264             moduleName = path.split("/")[0]
 265
 266             rulePath = os.path.join(moduleName, "IwyuFilter_" + moduleName + ".yaml")
 267             moduleRules = {}
 268             if os.path.exists(rulePath):
 269                 moduleRules = yaml.full_load(open(rulePath))
 270             assume = None
 271             pathAbs = os.path.abspath(path)
 272             compileFile = pathAbs
 273             matches = [i for i in compileCommands if i["file"] == compileFile]
 274             if not len(matches):
 275                 # Only use assume-filename for headers, so we don't try to analyze e.g. Windows-only
 276                 # code on Linux.
 277                 if "assumeFilename" in moduleRules.keys() and not path.endswith("cxx"):
 278                     assume = moduleRules["assumeFilename"]
 279                 if assume:
 280                     assumeAbs = os.path.abspath(assume)
 281                     compileFile = assumeAbs
 282                     matches = [i for i in compileCommands if i["file"] == compileFile]
 283                     if not len(matches):
 284                         print("WARNING: no compile commands for '" + path + "' (assumed filename: '" + assume + "'")
 285                         continue
 286                 else:
 287                     print("WARNING: no compile commands for '" + path + "'")
 288                     continue
 289
 290             _, _, args = matches[0]["command"].partition(" ")
 291             if assume:
 292                 args = args.replace(assumeAbs, "-x c++ " + pathAbs)
 293
 294             invocation = "include-what-you-use -Xiwyu --no_fwd_decls -Xiwyu --max_line_length=200 " + args
 295             task_queue.put((invocation, moduleRules))
 296
 297         task_queue.join()
 298         if len(failed_files):
 299             return_code = 1
 300
 301     except KeyboardInterrupt:
 302         print('\nCtrl-C detected, goodbye.')
 303         os.kill(0, 9)
 304
 305     sys.exit(return_code)
 306
 307
 308 def main(argv):
 309     parser = argparse.ArgumentParser(description='Check source files for unneeded includes.')
 310     parser.add_argument('--continue', action='store_true',
 311                     help='Don\'t stop on errors. Useful for periodic re-check of large amount of files')
 312     parser.add_argument('Files' , nargs='*',
 313                     help='The files to be checked')
 314     parser.add_argument('--recursive', metavar='DIR', nargs=1, type=str,
 315                     help='Recursively search a directory for source files to check')
 316     parser.add_argument('--headers', action='store_true',
 317                     help='Check header files. If omitted, check source files. Use with --recursive.')
 318     parser.add_argument('--noexclude', action='store_true',
 319                     help='Ignore excludelist. Useful to check whether its exclusions are still all valid.')
 320
 321     args = parser.parse_args()
 322
 323     if not len(argv):
 324         parser.print_help()
 325         return
 326
 327     list_of_files = []
 328     if args.recursive:
 329         for root, dirs, files in os.walk(args.recursive[0]):
 330             for file in files:
 331                 if args.headers:
 332                     if (file.endswith(".hxx") or file.endswith(".hrc") or file.endswith(".h")):
 333                         list_of_files.append(os.path.join(root,file))
 334                 else:
 335                     if (file.endswith(".cxx") or file.endswith(".c")):
 336                         list_of_files.append(os.path.join(root,file))
 337     else:
 338         list_of_files = args.Files
 339
 340     try:
 341         with open("compile_commands.json", 'r') as compileCommandsSock:
 342             compileCommands = json.load(compileCommandsSock)
 343     except FileNotFoundError:
 344         print ("File 'compile_commands.json' does not exist, please run:\nmake vim-ide-integration")
 345         sys.exit(-1)
 346
 347     # quickly sanity check whether files with exceptions in yaml still exists
 348     # only check for the module of the very first filename passed
 349
 350     # Verify there are files selected for checking, with --recursive it
 351     # may happen that there are in fact no C/C++ files in a module directory
 352     if not list_of_files:
 353         print("No files found to check!")
 354         sys.exit(-2)
 355
 356     moduleName = sorted(list_of_files)[0].split("/")[0]
 357     rulePath = os.path.join(moduleName, "IwyuFilter_" + moduleName + ".yaml")
 358     moduleRules = {}
 359     if os.path.exists(rulePath):
 360         moduleRules = yaml.full_load(open(rulePath))
 361     if "excludelist" in moduleRules.keys():
 362         excludelistRules = moduleRules["excludelist"]
 363         for pathname in excludelistRules.keys():
 364             file = pathlib.Path(pathname)
 365             if not file.exists():
 366                 print("WARNING: File listed in " + rulePath + " no longer exists: " + pathname)
 367
 368     tidy(compileCommands, paths=list_of_files, dontstop=vars(args)["continue"], noexclude=args.noexclude)
 369
 370 if __name__ == '__main__':
 371     main(sys.argv[1:])
 372
 373 # vim:set shiftwidth=4 softtabstop=4 expandtab: