bin/find-unneeded-includes

   1 #!/usr/bin/env python3
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6 #
   7 # This parses the output of 'include-what-you-use', focusing on just removing
   8 # not needed includes and providing a relatively conservative output by
   9 # filtering out a number of LibreOffice-specific false positives.
  10 #
  11 # It assumes you have a 'compile_commands.json' around (similar to clang-tidy),
  12 # you can generate one with 'make vim-ide-integration'.
  13 #
  14 # Design goals:
  15 # - excludelist mechanism, so a warning is either fixed or excluded
  16 # - works in a plugins-enabled clang build
  17 # - no custom configure options required
  18 # - no need to generate a dummy library to build a header
  19
  20 import glob
  21 import json
  22 import multiprocessing
  23 import os
  24 import queue
  25 import re
  26 import subprocess
  27 import sys
  28 import threading
  29 import yaml
  30 import argparse
  31
  32
  33 def ignoreRemoval(include, toAdd, absFileName, moduleRules):
  34     # global rules
  35
  36     # Avoid replacing .hpp with .hdl in the com::sun::star and  ooo::vba namespaces.
  37     if ( include.startswith("com/sun/star") or include.startswith("ooo/vba") ) and include.endswith(".hpp"):
  38         hdl = include.replace(".hpp", ".hdl")
  39         if hdl in toAdd:
  40             return True
  41
  42     # Avoid debug STL.
  43     debugStl = {
  44         "array": ("debug/array", ),
  45         "bitset": ("debug/bitset", ),
  46         "deque": ("debug/deque", ),
  47         "forward_list": ("debug/forward_list", ),
  48         "list": ("debug/list", ),
  49         "map": ("debug/map.h", "debug/multimap.h"),
  50         "set": ("debug/set.h", "debug/multiset.h"),
  51         "unordered_map": ("debug/unordered_map", ),
  52         "unordered_set": ("debug/unordered_set", ),
  53         "vector": ("debug/vector", ),
  54     }
  55     for k, values in debugStl.items():
  56         if include == k:
  57             for value in values:
  58                 if value in toAdd:
  59                     return True
  60
  61     # Avoid proposing to use libstdc++ internal headers.
  62     bits = {
  63         "exception": "bits/exception.h",
  64         "memory": "bits/shared_ptr.h",
  65         "functional": "bits/std_function.h",
  66         "cmath": "bits/std_abs.h",
  67         "ctime": "bits/types/clock_t.h",
  68         "cstdint": "bits/stdint-uintn.h",
  69     }
  70     for k, v in bits.items():
  71         if include == k and v in toAdd:
  72             return True
  73
  74     # Avoid proposing o3tl fw declaration
  75     o3tl = {
  76         "o3tl/typed_flags_set.hxx" : "namespace o3tl { template <typename T> struct typed_flags; }",
  77         "o3tl/deleter.hxx" : "namespace o3tl { template <typename T> struct default_delete; }",
  78         "o3tl/span.hxx" : "namespace o3tl { template <typename T> class span; }",
  79     }
  80     for k, v, in o3tl.items():
  81         if include == k and v in toAdd:
  82             return True
  83
  84     # Follow boost documentation.
  85     if include == "boost/optional.hpp" and "boost/optional/optional.hpp" in toAdd:
  86         return True
  87     if include == "boost/intrusive_ptr.hpp" and "boost/smart_ptr/intrusive_ptr.hpp" in toAdd:
  88         return True
  89     if include == "boost/variant.hpp" and "boost/variant/variant.hpp" in toAdd:
  90         return True
  91     if include == "boost/unordered_map.hpp" and "boost/unordered/unordered_map.hpp" in toAdd:
  92         return True
  93     if include == "boost/functional/hash.hpp" and "boost/container_hash/extensions.hpp" in toAdd:
  94         return True
  95
  96     # Avoid .hxx to .h proposals in basic css/uno/* API
  97     unoapi = {
  98         "com/sun/star/uno/Any.hxx": "com/sun/star/uno/Any.h",
  99         "com/sun/star/uno/Reference.hxx": "com/sun/star/uno/Reference.h",
 100         "com/sun/star/uno/Sequence.hxx": "com/sun/star/uno/Sequence.h",
 101         "com/sun/star/uno/Type.hxx": "com/sun/star/uno/Type.h"
 102     }
 103     for k, v in unoapi.items():
 104         if include == k and v in toAdd:
 105             return True
 106
 107     # 3rd-party, non-self-contained headers.
 108     if include == "libepubgen/libepubgen.h" and "libepubgen/libepubgen-decls.h" in toAdd:
 109         return True
 110     if include == "librevenge/librevenge.h" and "librevenge/RVNGPropertyList.h" in toAdd:
 111         return True
 112     if include == "libetonyek/libetonyek.h" and "libetonyek/EtonyekDocument.h" in toAdd:
 113         return True
 114
 115     noRemove = (
 116         # <https://www.openoffice.org/tools/CodingGuidelines.sxw> insists on not
 117         # removing this.
 118         "sal/config.h",
 119         # Works around a build breakage specific to the broken Android
 120         # toolchain.
 121         "android/compatibility.hxx",
 122         # Removing this would change the meaning of '#if defined OSL_BIGENDIAN'.
 123         "osl/endian.h",
 124     )
 125     if include in noRemove:
 126         return True
 127
 128     # Ignore when <foo> is to be replaced with "foo".
 129     if include in toAdd:
 130         return True
 131
 132     fileName = os.path.relpath(absFileName, os.getcwd())
 133
 134     # Skip headers used only for compile test
 135     if fileName == "cppu/qa/cppumaker/test_cppumaker.cxx":
 136         if include.endswith(".hpp"):
 137             return True
 138
 139     # yaml rules
 140
 141     if "excludelist" in moduleRules.keys():
 142         excludelistRules = moduleRules["excludelist"]
 143         if fileName in excludelistRules.keys():
 144             if include in excludelistRules[fileName]:
 145                 return True
 146
 147     return False
 148
 149
 150 def unwrapInclude(include):
 151     # Drop <> or "" around the include.
 152     return include[1:-1]
 153
 154
 155 def processIWYUOutput(iwyuOutput, moduleRules, fileName):
 156     inAdd = False
 157     toAdd = []
 158     inRemove = False
 159     toRemove = []
 160     currentFileName = None
 161
 162     for line in iwyuOutput:
 163         line = line.strip()
 164
 165         # Bail out if IWYU gave an error due to non self-containedness
 166         if re.match ("(.*): error: (.*)", line):
 167             return -1
 168
 169         if len(line) == 0:
 170             if inRemove:
 171                 inRemove = False
 172                 continue
 173             if inAdd:
 174                 inAdd = False
 175                 continue
 176
 177         shouldAdd = fileName + " should add these lines:"
 178         match = re.match(shouldAdd, line)
 179         if match:
 180             currentFileName = match.group(0).split(' ')[0]
 181             inAdd = True
 182             continue
 183
 184         shouldRemove = fileName + " should remove these lines:"
 185         match = re.match(shouldRemove, line)
 186         if match:
 187             currentFileName = match.group(0).split(' ')[0]
 188             inRemove = True
 189             continue
 190
 191         if inAdd:
 192             match = re.match('#include ([^ ]+)', line)
 193             if match:
 194                 include = unwrapInclude(match.group(1))
 195                 toAdd.append(include)
 196             else:
 197                 # Forward declaration.
 198                 toAdd.append(line)
 199
 200         if inRemove:
 201             match = re.match("- #include (.*)  // lines (.*)-.*", line)
 202             if match:
 203                 # Only suggest removals for now. Removing fwd decls is more complex: they may be
 204                 # indeed unused or they may removed to be replaced with an include. And we want to
 205                 # avoid the later.
 206                 include = unwrapInclude(match.group(1))
 207                 lineno = match.group(2)
 208                 if not ignoreRemoval(include, toAdd, currentFileName, moduleRules):
 209                     toRemove.append("%s:%s: %s" % (currentFileName, lineno, include))
 210
 211     for remove in sorted(toRemove):
 212         print("ERROR: %s: remove not needed include" % remove)
 213     return len(toRemove)
 214
 215
 216 def run_tool(task_queue, failed_files, dontstop):
 217     while True:
 218         invocation, moduleRules = task_queue.get()
 219         if not len(failed_files):
 220             print("[IWYU] " + invocation.split(' ')[-1])
 221             p = subprocess.Popen(invocation, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 222             retcode = processIWYUOutput(p.communicate()[0].decode('utf-8').splitlines(), moduleRules, invocation.split(' ')[-1])
 223             if retcode == -1:
 224                 print("ERROR: A file is probably not self contained, check this commands output:\n" + invocation)
 225             elif retcode > 0:
 226                 print("ERROR: The following command found unused includes:\n" + invocation)
 227                 if not dontstop:
 228                     failed_files.append(invocation)
 229         task_queue.task_done()
 230
 231
 232 def isInUnoIncludeFile(path):
 233     return path.startswith("include/com/") \
 234             or path.startswith("include/cppu/") \
 235             or path.startswith("include/cppuhelper/") \
 236             or path.startswith("include/osl/") \
 237             or path.startswith("include/rtl/") \
 238             or path.startswith("include/sal/") \
 239             or path.startswith("include/salhelper/") \
 240             or path.startswith("include/systools/") \
 241             or path.startswith("include/typelib/") \
 242             or path.startswith("include/uno/")
 243
 244
 245 def tidy(compileCommands, paths, dontstop):
 246     return_code = 0
 247
 248     try:
 249         max_task = multiprocessing.cpu_count()
 250         task_queue = queue.Queue(max_task)
 251         failed_files = []
 252         for _ in range(max_task):
 253             t = threading.Thread(target=run_tool, args=(task_queue, failed_files, dontstop))
 254             t.daemon = True
 255             t.start()
 256
 257         for path in sorted(paths):
 258             if isInUnoIncludeFile(path):
 259                 continue
 260
 261             moduleName = path.split("/")[0]
 262
 263             rulePath = os.path.join(moduleName, "IwyuFilter_" + moduleName + ".yaml")
 264             moduleRules = {}
 265             if os.path.exists(rulePath):
 266                 moduleRules = yaml.full_load(open(rulePath))
 267             assume = None
 268             pathAbs = os.path.abspath(path)
 269             compileFile = pathAbs
 270             matches = [i for i in compileCommands if i["file"] == compileFile]
 271             if not len(matches):
 272                 # Only use assume-filename for headers, so we don't try to analyze e.g. Windows-only
 273                 # code on Linux.
 274                 if "assumeFilename" in moduleRules.keys() and not path.endswith("cxx"):
 275                     assume = moduleRules["assumeFilename"]
 276                 if assume:
 277                     assumeAbs = os.path.abspath(assume)
 278                     compileFile = assumeAbs
 279                     matches = [i for i in compileCommands if i["file"] == compileFile]
 280                     if not len(matches):
 281                         print("WARNING: no compile commands for '" + path + "' (assumed filename: '" + assume + "'")
 282                         continue
 283                 else:
 284                     print("WARNING: no compile commands for '" + path + "'")
 285                     continue
 286
 287             _, _, args = matches[0]["command"].partition(" ")
 288             if assume:
 289                 args = args.replace(assumeAbs, "-x c++ " + pathAbs)
 290
 291             invocation = "include-what-you-use -Xiwyu --no_fwd_decls -Xiwyu --max_line_length=200 " + args
 292             task_queue.put((invocation, moduleRules))
 293
 294         task_queue.join()
 295         if len(failed_files):
 296             return_code = 1
 297
 298     except KeyboardInterrupt:
 299         print('\nCtrl-C detected, goodbye.')
 300         os.kill(0, 9)
 301
 302     sys.exit(return_code)
 303
 304
 305 def main(argv):
 306     parser = argparse.ArgumentParser(description='Check source files for unneeded includes.')
 307     parser.add_argument('--dontstop', action='store_true',
 308                     help='Don\'t stop on errors. Useful for periodic re-check of large amount of files')
 309     parser.add_argument('Files' , nargs='*',
 310                     help='The files to be checked')
 311     parser.add_argument('--recursive', metavar='DIR', nargs=1, type=str,
 312                     help='Recursively search a directory for source files to check')
 313     parser.add_argument('--headers', action='store_true',
 314                     help='Check header files. If omitted, check source files. Use with --recursive.')
 315
 316     args = parser.parse_args()
 317
 318     if not len(argv):
 319         parser.print_help()
 320         return
 321
 322     list_of_files = []
 323     if args.recursive:
 324         for root, dirs, files in os.walk(args.recursive[0]):
 325             for file in files:
 326                 if args.headers:
 327                     if (file.endswith(".hxx") or file.endswith(".hrc") or file.endswith(".h")):
 328                         list_of_files.append(os.path.join(root,file))
 329                 else:
 330                     if (file.endswith(".cxx") or file.endswith(".c")):
 331                         list_of_files.append(os.path.join(root,file))
 332     else:
 333         list_of_files = args.Files
 334
 335     try:
 336         with open("compile_commands.json", 'r') as compileCommandsSock:
 337             compileCommands = json.load(compileCommandsSock)
 338     except FileNotFoundError:
 339         print ("File 'compile_commands.json' does not exist, please run:\nmake vim-ide-integration")
 340         sys.exit(-1)
 341
 342     tidy(compileCommands, paths=list_of_files, dontstop=args.dontstop)
 343
 344 if __name__ == '__main__':
 345     main(sys.argv[1:])
 346
 347 # vim:set shiftwidth=4 softtabstop=4 expandtab: