bin/find-unneeded-includes

   1 #!/usr/bin/env python3
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6 #
   7 # This parses the output of 'include-what-you-use', focusing on just removing
   8 # not needed includes and providing a relatively conservative output by
   9 # filtering out a number of LibreOffice-specific false positives.
  10 #
  11 # It assumes you have a 'compile_commands.json' around (similar to clang-tidy),
  12 # you can generate one with 'make vim-ide-integration'.
  13 #
  14 # Design goals:
  15 # - blacklist mechanism, so a warning is either fixed or blacklisted
  16 # - works in a plugins-enabled clang build
  17 # - no custom configure options required
  18 # - no need to generate a dummy library to build a header
  19
  20 import glob
  21 import json
  22 import multiprocessing
  23 import os
  24 import queue
  25 import re
  26 import subprocess
  27 import sys
  28 import threading
  29 import yaml
  30
  31
  32 def ignoreRemoval(include, toAdd, absFileName, moduleRules):
  33     # global rules
  34
  35     # Avoid replacing .hpp with .hdl in the com::sun::star namespace.
  36     if include.startswith("com/sun/star") and include.endswith(".hpp"):
  37         hdl = include.replace(".hpp", ".hdl")
  38         if hdl in toAdd:
  39             return True
  40
  41     # Avoid debug STL.
  42     debugStl = {
  43         "array": ("debug/array", ),
  44         "bitset": ("debug/bitset", ),
  45         "deque": ("debug/deque", ),
  46         "forward_list": ("debug/forward_list", ),
  47         "list": ("debug/list", ),
  48         "map": ("debug/map.h", "debug/multimap.h"),
  49         "set": ("debug/set.h", "debug/multiset.h"),
  50         "unordered_map": ("debug/unordered_map", ),
  51         "unordered_set": ("debug/unordered_set", ),
  52         "vector": ("debug/vector", ),
  53     }
  54     for k, values in debugStl.items():
  55         if include == k:
  56             for value in values:
  57                 if value in toAdd:
  58                     return True
  59
  60     # Avoid proposing to use libstdc++ internal headers.
  61     bits = {
  62         "exception": "bits/exception.h",
  63         "memory": "bits/shared_ptr.h",
  64         "functional": "bits/std_function.h",
  65         "cmath": "bits/std_abs.h",
  66         "ctime": "bits/types/clock_t.h",
  67         "cstdint": "bits/stdint-uintn.h"
  68     }
  69     for k, v in bits.items():
  70         if include == k and v in toAdd:
  71             return True
  72
  73     # Avoid proposing o3tl fw declaration
  74     o3tl = {
  75         "o3tl/typed_flags_set.hxx" : "namespace o3tl { template <typename T> struct typed_flags; }",
  76         "o3tl/deleter.hxx" : "namespace o3tl { template <typename T> struct default_delete; }",
  77         "o3tl/span.hxx" : "namespace o3tl { template <typename T> class span; }",
  78     }
  79     for k, v, in o3tl.items():
  80         if include == k and v in toAdd:
  81             return True
  82
  83     # Follow boost documentation.
  84     if include == "boost/optional.hpp" and "boost/optional/optional.hpp" in toAdd:
  85         return True
  86     if include == "boost/intrusive_ptr.hpp" and "boost/smart_ptr/intrusive_ptr.hpp" in toAdd:
  87         return True
  88     if include == "boost/variant.hpp" and "boost/variant/variant.hpp" in toAdd:
  89         return True
  90     if include == "boost/unordered_map.hpp" and "boost/unordered/unordered_map.hpp" in toAdd:
  91         return True
  92
  93     # Avoid .hxx to .h proposals in basic css/uno/* API
  94     unoapi = {
  95         "com/sun/star/uno/Any.hxx": "com/sun/star/uno/Any.h",
  96         "com/sun/star/uno/Reference.hxx": "com/sun/star/uno/Reference.h",
  97         "com/sun/star/uno/Sequence.hxx": "com/sun/star/uno/Sequence.h",
  98         "com/sun/star/uno/Type.hxx": "com/sun/star/uno/Type.h"
  99     }
 100     for k, v in unoapi.items():
 101         if include == k and v in toAdd:
 102             return True
 103
 104     # 3rd-party, non-self-contained headers.
 105     if include == "libepubgen/libepubgen.h" and "libepubgen/libepubgen-decls.h" in toAdd:
 106         return True
 107     if include == "librevenge/librevenge.h" and "librevenge/RVNGPropertyList.h" in toAdd:
 108         return True
 109
 110     noRemove = (
 111         # <https://www.openoffice.org/tools/CodingGuidelines.sxw> insists on not
 112         # removing this.
 113         "sal/config.h",
 114         # Works around a build breakage specific to the broken Android
 115         # toolchain.
 116         "android/compatibility.hxx",
 117     )
 118     if include in noRemove:
 119         return True
 120
 121     # Ignore when <foo> is to be replaced with "foo".
 122     if include in toAdd:
 123         return True
 124
 125     fileName = os.path.relpath(absFileName, os.getcwd())
 126
 127     # Skip headers used only for compile test
 128     if fileName == "cppu/qa/cppumaker/test_cppumaker.cxx":
 129         if include.endswith(".hpp"):
 130             return True
 131
 132     # yaml rules
 133
 134     if "blacklist" in moduleRules.keys():
 135         blacklistRules = moduleRules["blacklist"]
 136         if fileName in blacklistRules.keys():
 137             if include in blacklistRules[fileName]:
 138                 return True
 139
 140     return False
 141
 142
 143 def unwrapInclude(include):
 144     # Drop <> or "" around the include.
 145     return include[1:-1]
 146
 147
 148 def processIWYUOutput(iwyuOutput, moduleRules):
 149     inAdd = False
 150     toAdd = []
 151     inRemove = False
 152     toRemove = []
 153     currentFileName = None
 154     for line in iwyuOutput:
 155         line = line.strip()
 156
 157         if len(line) == 0:
 158             if inRemove:
 159                 inRemove = False
 160                 continue
 161             if inAdd:
 162                 inAdd = False
 163                 continue
 164
 165         match = re.match("(.*) should add these lines:$", line)
 166         if match:
 167             currentFileName = match.group(1)
 168             inAdd = True
 169             continue
 170
 171         match = re.match("(.*) should remove these lines:$", line)
 172         if match:
 173             currentFileName = match.group(1)
 174             inRemove = True
 175             continue
 176
 177         if inAdd:
 178             match = re.match('#include ([^ ]+)', line)
 179             if match:
 180                 include = unwrapInclude(match.group(1))
 181                 toAdd.append(include)
 182             else:
 183                 # Forward declaration.
 184                 toAdd.append(line)
 185
 186         if inRemove:
 187             match = re.match("- #include (.*)  // lines (.*)-.*", line)
 188             if match:
 189                 # Only suggest removals for now. Removing fwd decls is more complex: they may be
 190                 # indeed unused or they may removed to be replaced with an include. And we want to
 191                 # avoid the later.
 192                 include = unwrapInclude(match.group(1))
 193                 lineno = match.group(2)
 194                 if not ignoreRemoval(include, toAdd, currentFileName, moduleRules):
 195                     toRemove.append("%s:%s: %s" % (currentFileName, lineno, include))
 196
 197     for remove in sorted(toRemove):
 198         print("ERROR: %s: remove not needed include" % remove)
 199     return len(toRemove)
 200
 201
 202 def run_tool(task_queue, failed_files):
 203     while True:
 204         invocation, moduleRules = task_queue.get()
 205         if not len(failed_files):
 206             print("[IWYU] " + invocation.split(' ')[-1])
 207             p = subprocess.Popen(invocation, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 208             retcode = processIWYUOutput(p.communicate()[0].decode('utf-8').splitlines(), moduleRules)
 209             if retcode != 0:
 210                 print("ERROR: The following command found unused includes:\n" + invocation)
 211                 failed_files.append(invocation)
 212         task_queue.task_done()
 213
 214
 215 def isInUnoIncludeFile(path):
 216     return path.startswith("include/com/") \
 217             or path.startswith("include/cppu/") \
 218             or path.startswith("include/cppuhelper/") \
 219             or path.startswith("include/osl/") \
 220             or path.startswith("include/rtl/") \
 221             or path.startswith("include/sal/") \
 222             or path.startswith("include/salhelper/") \
 223             or path.startswith("include/systools/") \
 224             or path.startswith("include/typelib/") \
 225             or path.startswith("include/uno/")
 226
 227
 228 def tidy(compileCommands, paths):
 229     return_code = 0
 230     try:
 231         max_task = multiprocessing.cpu_count()
 232         task_queue = queue.Queue(max_task)
 233         failed_files = []
 234         for _ in range(max_task):
 235             t = threading.Thread(target=run_tool, args=(task_queue, failed_files))
 236             t.daemon = True
 237             t.start()
 238
 239         for path in sorted(paths):
 240             if isInUnoIncludeFile(path):
 241                 continue
 242
 243             moduleName = path.split("/")[0]
 244
 245             rulePath = os.path.join(moduleName, "IwyuFilter_" + moduleName + ".yaml")
 246             moduleRules = {}
 247             if os.path.exists(rulePath):
 248                 moduleRules = yaml.load(open(rulePath))
 249             assume = None
 250             pathAbs = os.path.abspath(path)
 251             compileFile = pathAbs
 252             matches = [i for i in compileCommands if i["file"] == compileFile]
 253             if not len(matches):
 254                 if "assumeFilename" in moduleRules.keys():
 255                     assume = moduleRules["assumeFilename"]
 256                 if assume:
 257                     assumeAbs = os.path.abspath(assume)
 258                     compileFile = assumeAbs
 259                     matches = [i for i in compileCommands if i["file"] == compileFile]
 260                     if not len(matches):
 261                         print("WARNING: no compile commands for '" + path + "' (assumed filename: '" + assume + "'")
 262                         continue
 263                 else:
 264                     print("WARNING: no compile commands for '" + path + "'")
 265                     continue
 266
 267             _, _, args = matches[0]["command"].partition(" ")
 268             if assume:
 269                 args = args.replace(assumeAbs, "-x c++ " + pathAbs)
 270
 271             invocation = "include-what-you-use -Xiwyu --no_fwd_decls " + args
 272             task_queue.put((invocation, moduleRules))
 273
 274         task_queue.join()
 275         if len(failed_files):
 276             return_code = 1
 277
 278     except KeyboardInterrupt:
 279         print('\nCtrl-C detected, goodbye.')
 280         os.kill(0, 9)
 281
 282     sys.exit(return_code)
 283
 284
 285 def main(argv):
 286     if not len(argv):
 287         print("usage: find-unneeded-includes [FILE]...")
 288         return
 289
 290     with open("compile_commands.json", 'r') as compileCommandsSock:
 291         compileCommands = json.load(compileCommandsSock)
 292
 293     tidy(compileCommands, paths=argv)
 294
 295 if __name__ == '__main__':
 296     main(sys.argv[1:])
 297
 298 # vim:set shiftwidth=4 softtabstop=4 expandtab: