docs/doxygen/includesorter.py

   1 #!/usr/bin/env python3
   2 #
   3 # This file is part of the GROMACS molecular simulation package.
   4 #
   5 # Copyright (c) 2012,2013,2014,2015,2016,2017,2018,2019, by the GROMACS development team, led by
   6 # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   7 # and including many others, as listed in the AUTHORS file in the
   8 # top-level source directory and at http://www.gromacs.org.
   9 #
  10 # GROMACS is free software; you can redistribute it and/or
  11 # modify it under the terms of the GNU Lesser General Public License
  12 # as published by the Free Software Foundation; either version 2.1
  13 # of the License, or (at your option) any later version.
  14 #
  15 # GROMACS is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18 # Lesser General Public License for more details.
  19 #
  20 # You should have received a copy of the GNU Lesser General Public
  21 # License along with GROMACS; if not, see
  22 # http://www.gnu.org/licenses, or write to the Free Software Foundation,
  23 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  24 #
  25 # If you want to redistribute modifications to GROMACS, please
  26 # consider that scientific software is very special. Version
  27 # control is crucial - bugs must be traceable. We will be happy to
  28 # consider code for inclusion in the official distribution, but
  29 # derived work must not be called official GROMACS. Details are found
  30 # in the README & COPYING files - if they are missing, get the
  31 # official version at http://www.gromacs.org.
  32 #
  33 # To help us fund GROMACS development, we humbly ask that you cite
  34 # the research papers on the package. Check out http://www.gromacs.org.
  35
  36 """Include directive sorter for GROMACS.
  37
  38 This module implements an #include directive sorter for GROMACS C/C++ files.
  39 It allows (in most cases) automatically sorting includes and formatting
  40 the paths to use either relative paths or paths relative to src/.
  41 It groups includes in groups of related headers, sorts the headers
  42 alphabetically within each block, and inserts empty lines in between.
  43 It can be run as a standalone script, in which case it requires an up-to-date
  44 Doxygen XML documentation to be present in the
  45 build tree.  It can also be imported as a module to be embedded in other
  46 scripts.  In the latter case, the IncludeSorter provides the main interface.
  47
  48 The sorting assumes some conventions (e.g., that system headers are included
  49 with angle brackets instead of quotes).  Generally, these conventions are
  50 checked by the check-source.py script.
  51
  52 A more detailed description can be found in the developer manual.
  53 """
  54
  55 import os.path
  56 import re
  57 import sys
  58 import functools
  59
  60 @functools.total_ordering
  61 class IncludeGroup(object):
  62
  63     """Enumeration type for grouping includes."""
  64
  65     def __init__(self, value):
  66         """Initialize a IncludeGroup instance.
  67
  68         IncludeGroup.{main,system_c,...} should be used outside the
  69         class instead of calling the constructor.
  70         """
  71         self._value = value
  72
  73     def __eq__(self, other):
  74         """Order include groups in the desired order."""
  75         return self._value == other._value
  76
  77     def __lt__(self, other):
  78         """Order include groups in the desired order."""
  79         return self._value < other._value
  80
  81 # gmxpre.h is always first
  82 IncludeGroup.pre = IncludeGroup(0)
  83 # "main" include file for the source file is next
  84 IncludeGroup.main = IncludeGroup(1)
  85 # config.h is next, if present, to keep its location consistent
  86 IncludeGroup.config = IncludeGroup(2)
  87 # Followed by system headers, with C first and C++ following
  88 IncludeGroup.system_c = IncludeGroup(3)
  89 IncludeGroup.system_c_cpp = IncludeGroup(4)
  90 IncludeGroup.system_cpp = IncludeGroup(5)
  91 # System headers not in standard C/C++ are in a separate block
  92 IncludeGroup.system_other = IncludeGroup(6)
  93 # src/external/ contents that are included with quotes go here
  94 IncludeGroup.nonsystem_other = IncludeGroup(7)
  95 # Other GROMACS headers
  96 IncludeGroup.gmx_general = IncludeGroup(8)
  97 # This group is for shared (unit) testing utilities
  98 IncludeGroup.gmx_test = IncludeGroup(9)
  99 # This group is for headers local to the including file/module
 100 IncludeGroup.gmx_local = IncludeGroup(10)
 101
 102 class GroupedSorter(object):
 103
 104     """Grouping and formatting logic for #include directives.
 105
 106     This class implements the actual logic that decides how includes are
 107     grouped and sorted, and how they are formatted."""
 108
 109     # These variables contain the list of system headers for various blocks
 110     _std_c_headers = ['assert.h', 'ctype.h', 'errno.h', 'float.h',
 111             'inttypes.h', 'limits.h', 'math.h', 'signal.h', 'stdarg.h',
 112             'stddef.h', 'stdint.h', 'stdio.h', 'stdlib.h', 'string.h',
 113             'time.h']
 114     _std_c_cpp_headers = ['c' + x[:-2] for x in _std_c_headers]
 115     _std_cpp_headers = ['algorithm', 'array', 'chrono', 'deque', 'exception', 'fstream',
 116             'functional', 'initializer_list', 'iomanip', 'ios', 'iosfwd',
 117             'iostream', 'istream', 'iterator',
 118             'limits', 'list', 'map', 'memory', 'mutex',
 119             'new', 'numeric', 'ostream', 'random',
 120             'regex', 'set', 'sstream', 'stdexcept', 'streambuf', 'string', 'strstream',
 121             'thread', 'tuple', 'type_traits', 'typeindex', 'typeinfo', 'vector',
 122             'unordered_map', 'utility']
 123
 124     def __init__(self, style='pub-priv', absolute=False):
 125         """Initialize a sorted with the given style."""
 126         if style == 'single-group':
 127             self._local_group = 'none'
 128         elif style == 'pub-priv':
 129             self._local_group = 'private'
 130         else:
 131             self._local_group = 'local'
 132         if absolute:
 133             self._abspath_main = True
 134             self._abspath_local = True
 135         else:
 136             self._abspath_main = False
 137             self._abspath_local = False
 138
 139     def _get_path(self, included_file, group, including_file):
 140         """Compute include path to use for an #include.
 141
 142         The path is made either absolute (i.e., relative to src/), or
 143         relative to the location of the including file, depending on the group
 144         the file is in.
 145         """
 146         use_abspath = including_file is None or group is None
 147         if not use_abspath:
 148             if group in (IncludeGroup.gmx_general, IncludeGroup.gmx_test):
 149                 use_abspath = True
 150             elif group == IncludeGroup.main and self._abspath_main:
 151                 use_abspath = True
 152             elif group == IncludeGroup.gmx_local and self._abspath_local:
 153                 use_abspath = True
 154         if not use_abspath:
 155             fromdir = os.path.dirname(including_file.get_abspath())
 156             relpath = os.path.relpath(included_file.get_abspath(), fromdir)
 157             if not relpath.startswith('..'):
 158                 return relpath
 159         path = included_file.get_relpath()
 160         assert path.startswith('src/')
 161         return path[4:]
 162
 163     def _get_gmx_group(self, including_file, included_file):
 164         """Determine group for GROMACS headers.
 165
 166         Helper function to determine the group for an #include directive
 167         when the #include is in one of the gmx_* groups (or in the main group).
 168         """
 169         main_header = including_file.get_main_header()
 170         if main_header and main_header == included_file:
 171             return IncludeGroup.main
 172         if included_file.get_directory().get_name() == 'testutils':
 173             return IncludeGroup.gmx_test
 174         if including_file.get_directory().contains(included_file):
 175             if self._local_group == 'local':
 176                 return IncludeGroup.gmx_local
 177             if self._local_group == 'private':
 178                 if included_file.api_type_is_reliable() \
 179                         and included_file.is_module_internal():
 180                     return IncludeGroup.gmx_local
 181                 if not included_file.api_type_is_reliable() \
 182                         and including_file.get_relpath().startswith('src/programs'):
 183                     return IncludeGroup.gmx_local
 184         if included_file.is_test_file():
 185             return IncludeGroup.gmx_test
 186         return IncludeGroup.gmx_general
 187
 188     def _split_path(self, path):
 189         """Split include path into sortable compoments.
 190
 191         Plain string on the full path in the #include directive causes some
 192         unintuitive behavior, so this splits the path into a tuple at
 193         points that allow more natural sorting: primary sort criterion is the
 194         directory name, followed by the basename (without extension) of the
 195         included file.
 196         """
 197         path_components = list(os.path.split(path))
 198         path_components[1] = os.path.splitext(path_components[1])
 199         return tuple(path_components)
 200
 201     def _join_path(self, path_components):
 202         """Reconstruct path from the return value of _split_path."""
 203         return os.path.join(path_components[0], ''.join(path_components[1]))
 204
 205     def get_sortable_object(self, include):
 206         """Produce a sortable, opaque object for an include.
 207
 208         Includes are sorted by calling this function for each #include object,
 209         and sorting the list made up of these objects (using the default
 210         comparison operators).  Each element from the sorted list is then
 211         passed to format_include(), which extracts information from the opaque
 212         object and formats the #include directive for output.
 213         """
 214         included_file = include.get_file()
 215         if not included_file:
 216             path = include.get_included_path()
 217             if path in self._std_c_headers:
 218                 group = IncludeGroup.system_c
 219             elif path in self._std_c_cpp_headers:
 220                 group = IncludeGroup.system_c_cpp
 221             elif path in self._std_cpp_headers:
 222                 group = IncludeGroup.system_cpp
 223             else:
 224                 group = IncludeGroup.system_other
 225         elif included_file.is_external():
 226             group = IncludeGroup.nonsystem_other
 227             if 'external/' in include.get_included_path():
 228                 path = self._get_path(included_file, group, None)
 229             else:
 230                 path = include.get_included_path()
 231         elif included_file.get_name() == 'gmxpre.h':
 232             group = IncludeGroup.pre
 233             path = self._get_path(included_file, group, None)
 234         elif included_file.get_name() == 'config.h':
 235             group = IncludeGroup.config
 236             path = self._get_path(included_file, group, None)
 237         else:
 238             including_file = include.get_including_file()
 239             group = self._get_gmx_group(including_file, included_file)
 240             path = self._get_path(included_file, group, including_file)
 241         return (group, self._split_path(path), include)
 242
 243     def format_include(self, obj, prev):
 244         """Format an #include directive after sorting."""
 245         result = []
 246         if prev:
 247             if prev[0] != obj[0]:
 248                 # Print empty line between groups
 249                 result.append('\n')
 250             elif prev[1] == obj[1]:
 251                 # Skip duplicates
 252                 return result
 253         include = obj[2]
 254         line = include.get_full_line()
 255         include_re = r'^(?P<head>\s*#\s*include\s+)["<][^">]*[">](?P<tail>.*)$'
 256         match = re.match(include_re, line)
 257         assert match
 258         if include.is_system():
 259             path = '<{0}>'.format(self._join_path(obj[1]))
 260         else:
 261             path = '"{0}"'.format(self._join_path(obj[1]))
 262         result.append('{0}{1}{2}\n'.format(match.group('head'), path, match.group('tail')))
 263         return result
 264
 265 class IncludeSorter(object):
 266
 267     """High-level logic for sorting includes.
 268
 269     This class contains the high-level logic for sorting include statements.
 270     The actual ordering and formatting the includes is delegated to a sort method
 271     (see GroupedSorter) to keep things separated.
 272     """
 273
 274     def __init__(self, sortmethod=None, quiet=True):
 275         """Initialize the include sorter with the given sorter and options."""
 276         if not sortmethod:
 277             sortmethod = GroupedSorter()
 278         self._sortmethod = sortmethod
 279         self._quiet = quiet
 280         self._changed = False
 281
 282     def _sort_include_block(self, block, lines):
 283         """Sort a single include block.
 284
 285         Returns a new list of lines for the block.
 286         If anything is changed, self._changed is set to True, and the caller
 287         can check that."""
 288         includes = sorted(map(self._sortmethod.get_sortable_object, block.get_includes()))
 289         result = []
 290         prev = None
 291         current_line_number = block.get_first_line()-1
 292         for include in includes:
 293             newlines = self._sortmethod.format_include(include, prev)
 294             result.extend(newlines)
 295             if not self._changed:
 296                 for offset, newline in enumerate(newlines):
 297                     if lines[current_line_number + offset] != newline:
 298                         self._changed = True
 299                         break
 300                 current_line_number += len(newlines)
 301             prev = include
 302         return result
 303
 304     def sort_includes(self, fileobj):
 305         """Sort all includes in a file."""
 306         lines = fileobj.get_contents()
 307         # Format into a list first:
 308         #  - avoid bugs or issues in the script truncating the file
 309         #  - can check whether anything was changed before touching the file
 310         newlines = []
 311         prev = 0
 312         self._changed = False
 313         for block in fileobj.get_include_blocks():
 314             newlines.extend(lines[prev:block.get_first_line()-1])
 315             newlines.extend(self._sort_include_block(block, lines))
 316             # The returned values are 1-based, but indexing here is 0-based,
 317             # so an explicit +1 is not needed.
 318             prev = block.get_last_line()
 319         if self._changed:
 320             if not self._quiet:
 321                 sys.stderr.write('{0}: includes reformatted\n'.format(fileobj.get_relpath()))
 322             newlines.extend(lines[prev:])
 323             with open(fileobj.get_abspath(), 'w') as fp:
 324                 fp.write(''.join(newlines))
 325
 326     def check_sorted(self, fileobj):
 327         """Check that includes within a file are sorted."""
 328         # TODO: Make the checking work without full contents of the file
 329         lines = fileobj.get_contents()
 330         is_sorted = True
 331         details = None
 332         for block in fileobj.get_include_blocks():
 333             self._changed = False
 334             sorted_lines = self._sort_include_block(block, lines)
 335             if self._changed:
 336                 is_sorted = False
 337                 # TODO: Do a proper diff to show the actual changes.
 338                 if details is None:
 339                     details = ["Correct order/style is:"]
 340                 else:
 341                     details.append("    ...")
 342                 details.extend(["    " + x.rstrip() for x in sorted_lines])
 343         return (is_sorted, details)
 344
 345 def main():
 346     """Run the include sorter script."""
 347     import os
 348     import sys
 349
 350     from optparse import OptionParser
 351
 352     from gmxtree import GromacsTree
 353     from reporter import Reporter
 354
 355     parser = OptionParser()
 356     parser.add_option('-S', '--source-root',
 357                       help='Source tree root directory')
 358     parser.add_option('-B', '--build-root',
 359                       help='Build tree root directory')
 360     parser.add_option('-F', '--files',
 361                       help='Specify files to sort')
 362     parser.add_option('-q', '--quiet', action='store_true',
 363                       help='Do not write status messages')
 364     # This is for evaluating different options; can be removed from the final
 365     # version.
 366     parser.add_option('-s', '--style', type='choice', default='pub-priv',
 367                       choices=('single-group', 'pub-priv', 'pub-local'),
 368                       help='Style for GROMACS includes')
 369     parser.add_option('--absolute', action='store_true',
 370                       help='Write all include paths relative to src/')
 371     options, args = parser.parse_args()
 372
 373     filelist = args
 374     if options.files:
 375         if options.files == '-':
 376             lines = sys.stdin.readlines()
 377         else:
 378             with open(options.files, 'r') as fp:
 379                 lines = fp.readlines()
 380         filelist.extend([x.strip() for x in lines])
 381
 382     reporter = Reporter(quiet=True)
 383
 384     if not options.quiet:
 385         sys.stderr.write('Scanning source tree...\n')
 386     if not options.source_root:
 387         sys.stderr.write('Source root required not specified.\n')
 388         sys.exit(2)
 389     if not options.build_root:
 390         sys.stderr.write('Build root required not specified.\n')
 391         sys.exit(2)
 392     tree = GromacsTree(options.source_root, options.build_root, reporter)
 393     files = []
 394     for filename in filelist:
 395         fileobj = tree.get_file(os.path.abspath(filename))
 396         if not fileobj:
 397             sys.stderr.write('warning: ignoring unknown file {0}\n'.format(filename))
 398             continue
 399         files.append(fileobj)
 400     if not options.quiet:
 401         sys.stderr.write('Reading source files...\n')
 402     tree.scan_files(only_files=files, keep_contents=True)
 403     extfiles = set(files)
 404     for fileobj in files:
 405         for included_file in fileobj.get_includes():
 406             other_file = included_file.get_file()
 407             if other_file:
 408                 extfiles.add(other_file)
 409     if not options.quiet:
 410         sys.stderr.write('Reading Doxygen XML files...\n')
 411     tree.load_xml(only_files=extfiles)
 412
 413     if not options.quiet:
 414         sys.stderr.write('Sorting includes...\n')
 415
 416     sorter = IncludeSorter(GroupedSorter(options.style, options.absolute), options.quiet)
 417
 418     for fileobj in files:
 419         sorter.sort_includes(fileobj)
 420
 421 if __name__ == '__main__':
 422     main()