docs/doxygen/gmxtree.py

   1 #!/usr/bin/env python2
   2 #
   3 # This file is part of the GROMACS molecular simulation package.
   4 #
   5 # Copyright (c) 2014,2015,2016,2017,2018, by the GROMACS development team, led by
   6 # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   7 # and including many others, as listed in the AUTHORS file in the
   8 # top-level source directory and at http://www.gromacs.org.
   9 #
  10 # GROMACS is free software; you can redistribute it and/or
  11 # modify it under the terms of the GNU Lesser General Public License
  12 # as published by the Free Software Foundation; either version 2.1
  13 # of the License, or (at your option) any later version.
  14 #
  15 # GROMACS is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18 # Lesser General Public License for more details.
  19 #
  20 # You should have received a copy of the GNU Lesser General Public
  21 # License along with GROMACS; if not, see
  22 # http://www.gnu.org/licenses, or write to the Free Software Foundation,
  23 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  24 #
  25 # If you want to redistribute modifications to GROMACS, please
  26 # consider that scientific software is very special. Version
  27 # control is crucial - bugs must be traceable. We will be happy to
  28 # consider code for inclusion in the official distribution, but
  29 # derived work must not be called official GROMACS. Details are found
  30 # in the README & COPYING files - if they are missing, get the
  31 # official version at http://www.gromacs.org.
  32 #
  33 # To help us fund GROMACS development, we humbly ask that you cite
  34 # the research papers on the package. Check out http://www.gromacs.org.
  35
  36 """GROMACS-specific representation for source tree and documentation.
  37
  38 This module provides classes that construct a GROMACS-specific representation
  39 of the source tree and associate the Doxygen XML output with it.  It constructs
  40 an initial representation by walking the source tree in the file system, and
  41 then associates information from the Doxygen XML output into this.
  42 It also adds some additional knowledge from how the GROMACS source tree is
  43 organized to construct a representation that is easy to process and check as
  44 the top-level scripts expect.
  45
  46 The object model is rooted at a GromacsTree object.  Currently, it constructs a
  47 representation of the source tree from the file system, but is otherwise mostly
  48 a thin wrapper around the Doxygen XML tree.  It already adds some relations and
  49 rules that come from GROMACS-specific knowledge.  In the future, more such
  50 customizations will be added.
  51 """
  52
  53 import collections
  54 import os
  55 import os.path
  56 import re
  57 import subprocess
  58
  59 import doxygenxml as xml
  60 import reporter
  61 # We import DocType directly so that it is exposed from this module as well.
  62 from doxygenxml import DocType
  63
  64 def _get_api_type_for_compound(grouplist):
  65     """Helper function to deduce API type from Doxygen group membership."""
  66     result = DocType.internal
  67     for group in grouplist:
  68         if isinstance(group, xml.Group):
  69             if group.get_name() == 'group_publicapi':
  70                 result = DocType.public
  71             elif group.get_name() == 'group_libraryapi':
  72                 result = DocType.library
  73             # TODO: Check for multiple group membership
  74     return result
  75
  76 class IncludedFile(object):
  77
  78     """Information about an #include directive in a file."""
  79
  80     def __init__(self, including_file, lineno, included_file, included_path, is_relative, is_system, line):
  81         self._including_file = including_file
  82         self._line_number = lineno
  83         self._included_file = included_file
  84         self._included_path = included_path
  85         #self._used_include_path = used_include_path
  86         self._is_relative = is_relative
  87         self._is_system = is_system
  88         self._line = line
  89
  90     def __str__(self):
  91         if self._is_system:
  92             return '<{0}>'.format(self._included_path)
  93         else:
  94             return '"{0}"'.format(self._included_path)
  95
  96     def is_system(self):
  97         return self._is_system
  98
  99     def is_relative(self):
 100         return self._is_relative
 101
 102     def get_included_path(self):
 103         return self._included_path
 104
 105     def get_including_file(self):
 106         return self._including_file
 107
 108     def get_file(self):
 109         return self._included_file
 110
 111     def get_line_number(self):
 112         return self._line_number
 113
 114     def get_full_line(self):
 115         """Return the full source line on which this include appears.
 116
 117         Trailing newline is included."""
 118         return self._line
 119
 120     def get_reporter_location(self):
 121         return reporter.Location(self._including_file.get_abspath(), self._line_number)
 122
 123 class IncludeBlock(object):
 124
 125     """Block of consequent #include directives in a file."""
 126
 127     def __init__(self, first_included_file):
 128         self._first_line = first_included_file.get_line_number()
 129         self._last_line = self._first_line
 130         self._files = []
 131         self.add_file(first_included_file)
 132
 133     def add_file(self, included_file):
 134         self._files.append(included_file)
 135         self._last_line = included_file.get_line_number()
 136
 137     def get_includes(self):
 138         return self._files
 139
 140     def get_first_line(self):
 141         return self._first_line
 142
 143     def get_last_line(self):
 144         return self._last_line
 145
 146 class File(object):
 147
 148     """Source/header file in the GROMACS tree."""
 149
 150     def __init__(self, abspath, relpath, directory):
 151         """Initialize a file representation with basic information."""
 152         self._abspath = abspath
 153         self._relpath = relpath
 154         self._dir = directory
 155         self._rawdoc = None
 156         self._installed = False
 157         extension = os.path.splitext(abspath)[1]
 158         self._sourcefile = (extension in ('.c', '.cc', '.cpp', '.cu'))
 159         self._apitype = DocType.none
 160         self._modules = set()
 161         self._includes = []
 162         self._include_blocks = []
 163         self._main_header = None
 164         self._lines = None
 165         self._filter = None
 166         self._declared_defines = None
 167         self._used_defines = dict()
 168         directory.add_file(self)
 169
 170     def set_doc_xml(self, rawdoc, sourcetree):
 171         """Assiociate Doxygen documentation entity with the file."""
 172         assert self._rawdoc is None
 173         assert rawdoc.is_source_file() == self._sourcefile
 174         self._rawdoc = rawdoc
 175         if self._rawdoc.is_documented():
 176             grouplist = self._rawdoc.get_groups()
 177             self._apitype = _get_api_type_for_compound(grouplist)
 178             for group in grouplist:
 179                 module = sourcetree.get_object(group)
 180                 if module:
 181                     self._modules.add(module)
 182
 183     def set_installed(self):
 184         """Mark the file installed."""
 185         self._installed = True
 186
 187     def set_git_filter_attribute(self, filtername):
 188         """Set the git filter attribute associated with the file."""
 189         self._filter = filtername
 190
 191     def set_main_header(self, included_file):
 192         """Set the main header file for a source file."""
 193         assert self.is_source_file()
 194         self._main_header = included_file
 195
 196     def _process_include(self, lineno, is_system, includedpath, line, sourcetree):
 197         """Process #include directive during scan()."""
 198         is_relative = False
 199         if is_system:
 200             fileobj = sourcetree.find_include_file(includedpath)
 201         else:
 202             fullpath = os.path.join(self._dir.get_abspath(), includedpath)
 203             fullpath = os.path.abspath(fullpath)
 204             if os.path.exists(fullpath):
 205                 is_relative = True
 206                 fileobj = sourcetree.get_file(fullpath)
 207             else:
 208                 fileobj = sourcetree.find_include_file(includedpath)
 209         included_file = IncludedFile(self, lineno, fileobj, includedpath,
 210             is_relative, is_system, line)
 211         self._includes.append(included_file)
 212         return included_file
 213
 214     def scan_contents(self, sourcetree, keep_contents, detect_defines):
 215         """Scan the file contents and initialize information based on it."""
 216         # TODO: Consider a more robust regex.
 217         include_re = r'^\s*#\s*include\s+(?P<quote>["<])(?P<path>[^">]*)[">]'
 218         define_re = r'^\s*#.*define(?:01)?\s+(\w*)'
 219         current_block = None
 220         with open(self._abspath, 'r') as scanfile:
 221             contents = scanfile.read()
 222         lines = contents.splitlines(True)
 223         for lineno, line in enumerate(lines, 1):
 224             match = re.match(include_re, line)
 225             if match:
 226                 is_system = (match.group('quote') == '<')
 227                 includedpath = match.group('path')
 228                 included_file = self._process_include(lineno, is_system,
 229                         includedpath, line, sourcetree)
 230                 if current_block is None:
 231                     current_block = IncludeBlock(included_file)
 232                     self._include_blocks.append(current_block)
 233                 else:
 234                     current_block.add_file(included_file)
 235             elif line and not line.isspace():
 236                 current_block = None
 237         if detect_defines:
 238             self._declared_defines = []
 239             for line in lines:
 240                 match = re.match(define_re, line)
 241                 if match:
 242                     self._declared_defines.append(match.group(1))
 243         if keep_contents:
 244             self._lines = lines
 245
 246     def add_used_defines(self, define_file, defines):
 247         """Add defines used in this file.
 248
 249         Used internally by find_define_file_uses()."""
 250         if define_file not in self._used_defines:
 251             self._used_defines[define_file] = set()
 252         self._used_defines[define_file].update(defines)
 253
 254     def get_reporter_location(self):
 255         return reporter.Location(self._abspath, None)
 256
 257     def is_installed(self):
 258         return self._installed
 259
 260     def is_external(self):
 261         return self._dir.is_external()
 262
 263     def is_source_file(self):
 264         return self._sourcefile
 265
 266     def is_test_file(self):
 267         return self._dir.is_test_directory()
 268
 269     def should_includes_be_sorted(self):
 270         """Return whether the include directives in the file should be sorted."""
 271         return self._filter in ('includesort', 'uncrustify')
 272
 273     def is_documented(self):
 274         return self._rawdoc and self._rawdoc.is_documented()
 275
 276     def has_brief_description(self):
 277         return self._rawdoc and self._rawdoc.has_brief_description()
 278
 279     def get_abspath(self):
 280         return self._abspath
 281
 282     def get_relpath(self):
 283         return self._relpath
 284
 285     def get_name(self):
 286         return os.path.basename(self._abspath)
 287
 288     def get_directory(self):
 289         return self._dir
 290
 291     def get_doc_type(self):
 292         if not self._rawdoc:
 293             return DocType.none
 294         return self._rawdoc.get_visibility()
 295
 296     def get_api_type(self):
 297         return self._apitype
 298
 299     def api_type_is_reliable(self):
 300         if self._apitype in (DocType.internal, DocType.library):
 301             return True
 302         module = self.get_module()
 303         return module and module.is_documented()
 304
 305     def is_public(self):
 306         if self.api_type_is_reliable():
 307             return self.get_api_type() == DocType.public
 308         return self.get_api_type() == DocType.public or self.is_installed()
 309
 310     def is_module_internal(self):
 311         if self.is_source_file():
 312             return True
 313         return not self.is_installed() and self.get_api_type() <= DocType.internal
 314
 315     def get_expected_module(self):
 316         return self._dir.get_module()
 317
 318     def get_doc_modules(self):
 319         return self._modules
 320
 321     def get_module(self):
 322         module = self.get_expected_module()
 323         if not module and len(self._modules) == 1:
 324             module = list(self._modules)[0]
 325         return module
 326
 327     def get_includes(self):
 328         return self._includes
 329
 330     def get_include_blocks(self):
 331         return self._include_blocks
 332
 333     def _get_included_files_recurse(self, result):
 334         for include in self._includes:
 335             included_file = include.get_file()
 336             if included_file is not None and not included_file in result:
 337                 result.add(included_file)
 338                 included_file._get_included_files_recurse(result)
 339
 340     def get_included_files(self, recursive=False):
 341         if recursive:
 342             result = set()
 343             self._get_included_files_recurse(result)
 344             return result
 345         return set([x.get_file() for x in self._includes])
 346
 347     def get_main_header(self):
 348         return self._main_header
 349
 350     def get_contents(self):
 351         return self._lines
 352
 353     def get_declared_defines(self):
 354         """Return set of defines declared in this file.
 355
 356         The information is only populated for selected files."""
 357         return self._declared_defines
 358
 359     def get_used_define_files(self):
 360         """Return files like config.h whose defines are used in this file.
 361
 362         The return value is empty if find_define_file_uses() has not been called,
 363         as well as for headers that declare these defines."""
 364         return set(self._used_defines.iterkeys())
 365
 366     def get_used_defines(self, define_file):
 367         """Return set of defines used in this file for a given file like config.h.
 368         """
 369         return self._used_defines.get(define_file, set())
 370
 371 class GeneratedFile(File):
 372     def __init__(self, abspath, relpath, directory):
 373         File.__init__(self, abspath, relpath, directory)
 374         self._generator_source_file = None
 375
 376     def scan_contents(self, sourcetree, keep_contents, detect_defines):
 377         if os.path.exists(self.get_abspath()):
 378             File.scan_contents(self, sourcetree, keep_contents, False)
 379
 380     def set_generator_source(self, sourcefile):
 381         self._generator_source_file = sourcefile
 382
 383     def get_generator_source(self):
 384         return self._generator_source_file
 385
 386     def get_reporter_location(self):
 387         if self._generator_source_file:
 388             return self._generator_source_file.get_reporter_location()
 389         return File.get_reporter_location(self)
 390
 391     def get_declared_defines(self):
 392         if self._generator_source_file:
 393             return self._generator_source_file.get_declared_defines()
 394         return File.get_declared_defines(self)
 395
 396 class GeneratorSourceFile(File):
 397     pass
 398
 399 class Directory(object):
 400
 401     """(Sub)directory in the GROMACS tree."""
 402
 403     def __init__(self, abspath, relpath, parent):
 404         """Initialize a file representation with basic information."""
 405         self._abspath = abspath
 406         self._relpath = relpath
 407         self._name = os.path.basename(abspath)
 408         self._parent = parent
 409         self._rawdoc = None
 410         self._module = None
 411         self._is_test_dir = False
 412         if parent and parent.is_test_directory() or \
 413                 self._name == 'tests':
 414             self._is_test_dir = True
 415         self._is_external = False
 416         if parent and parent.is_external() or self._name == 'external':
 417             self._is_external = True
 418         self._subdirs = set()
 419         if parent:
 420             parent._subdirs.add(self)
 421         self._files = set()
 422         self._has_installed_files = None
 423
 424     def set_doc_xml(self, rawdoc, sourcetree):
 425         """Assiociate Doxygen documentation entity with the directory."""
 426         assert self._rawdoc is None
 427         assert rawdoc.get_path().rstrip('/') in (self._abspath, self._relpath)
 428         self._rawdoc = rawdoc
 429
 430     def set_module(self, module):
 431         assert self._module is None
 432         self._module = module
 433
 434     def add_file(self, fileobj):
 435         self._files.add(fileobj)
 436
 437     def get_name(self):
 438         return self._name
 439
 440     def get_reporter_location(self):
 441         return reporter.Location(self._abspath, None)
 442
 443     def get_abspath(self):
 444         return self._abspath
 445
 446     def get_relpath(self):
 447         return self._relpath
 448
 449     def is_test_directory(self):
 450         return self._is_test_dir
 451
 452     def is_external(self):
 453         return self._is_external
 454
 455     def has_installed_files(self):
 456         if self._has_installed_files is None:
 457             self._has_installed_files = False
 458             for subdir in self._subdirs:
 459                 if subdir.has_installed_files():
 460                     self._has_installed_files = True
 461                     return True
 462             for fileobj in self._files:
 463                 if fileobj.is_installed():
 464                     self._has_installed_files = True
 465                     return True
 466         return self._has_installed_files
 467
 468     def get_module(self):
 469         if self._module:
 470             return self._module
 471         if self._parent:
 472             return self._parent.get_module()
 473         return None
 474
 475     def get_subdirectories(self):
 476         return self._subdirs
 477
 478     def get_files(self):
 479         for subdir in self._subdirs:
 480             for fileobj in subdir.get_files():
 481                 yield fileobj
 482         for fileobj in self._files:
 483             yield fileobj
 484
 485     def contains(self, fileobj):
 486         """Check whether file is within the directory or its subdirectories."""
 487         dirobj = fileobj.get_directory()
 488         while dirobj:
 489             if dirobj == self:
 490                 return True
 491             dirobj = dirobj._parent
 492         return False
 493
 494 class ModuleDependency(object):
 495
 496     """Dependency between modules."""
 497
 498     def __init__(self, othermodule):
 499         """Initialize empty dependency object with given module as dependency."""
 500         self._othermodule = othermodule
 501         self._includedfiles = []
 502         self._cyclesuppression = None
 503         self._is_test_only_dependency = True
 504         self.suppression_used = True
 505
 506     def add_included_file(self, includedfile):
 507         """Add IncludedFile that is part of this dependency."""
 508         assert includedfile.get_file().get_module() == self._othermodule
 509         if not includedfile.get_including_file().is_test_file():
 510             self._is_test_only_dependency = False
 511         self._includedfiles.append(includedfile)
 512
 513     def set_cycle_suppression(self):
 514         """Set suppression on cycles containing this dependency."""
 515         self._cyclesuppression = True
 516         self.suppression_used = False
 517
 518     def is_cycle_suppressed(self):
 519         """Return whether cycles containing this dependency are suppressed."""
 520         self.suppression_used = True
 521         return self._cyclesuppression is not None
 522
 523     def is_test_only_dependency(self):
 524         """Return whether this dependency is only from test code."""
 525         return self._is_test_only_dependency
 526
 527     def get_other_module(self):
 528         """Get module that this dependency is to."""
 529         return self._othermodule
 530
 531     def get_included_files(self):
 532         """Get IncludedFile objects for the individual include dependencies."""
 533         return self._includedfiles
 534
 535 class Module(object):
 536
 537     """Code module in the GROMACS source tree.
 538
 539     Modules are specific subdirectories that host a more or less coherent
 540     set of routines.  Simplified, every subdirectory under src/gromacs/ is
 541     a different module.  This object provides that abstraction and also links
 542     the subdirectory to the module documentation (documented as a group in
 543     Doxygen) if that exists.
 544     """
 545
 546     def __init__(self, name, rootdir):
 547         self._name = name
 548         self._rawdoc = None
 549         self._rootdir = rootdir
 550         self._group = None
 551         self._dependencies = dict()
 552
 553     def set_doc_xml(self, rawdoc, sourcetree):
 554         """Assiociate Doxygen documentation entity with the module."""
 555         assert self._rawdoc is None
 556         self._rawdoc = rawdoc
 557         if self._rawdoc.is_documented():
 558             groups = list(self._rawdoc.get_groups())
 559             if len(groups) == 1:
 560                 groupname = groups[0].get_name()
 561                 if groupname.startswith('group_'):
 562                     self._group = groupname[6:]
 563
 564     def add_dependency(self, othermodule, includedfile):
 565         """Add #include dependency from a file in this module."""
 566         assert includedfile.get_file().get_module() == othermodule
 567         if othermodule not in self._dependencies:
 568             self._dependencies[othermodule] = ModuleDependency(othermodule)
 569         self._dependencies[othermodule].add_included_file(includedfile)
 570
 571     def is_documented(self):
 572         return self._rawdoc is not None
 573
 574     def get_name(self):
 575         return self._name
 576
 577     def get_root_dir(self):
 578         return self._rootdir
 579
 580     def get_files(self):
 581         # TODO: Include public API convenience headers?
 582         return self._rootdir.get_files()
 583
 584     def get_group(self):
 585         return self._group
 586
 587     def get_dependencies(self):
 588         return self._dependencies.itervalues()
 589
 590 class Namespace(object):
 591
 592     """Namespace in the GROMACS source code."""
 593
 594     def __init__(self, rawdoc):
 595         self._rawdoc = rawdoc
 596
 597     def is_anonymous(self):
 598         return self._rawdoc.is_anonymous()
 599
 600 class Class(object):
 601
 602     """Class/struct/union in the GROMACS source code."""
 603
 604     def __init__(self, rawdoc, files):
 605         self._rawdoc = rawdoc
 606         self._files = set(files)
 607
 608     def get_name(self):
 609         return self._rawdoc.get_name()
 610
 611     def get_reporter_location(self):
 612         return self._rawdoc.get_reporter_location()
 613
 614     def get_files(self):
 615         return self._files
 616
 617     def is_documented(self):
 618         return self._rawdoc.is_documented()
 619
 620     def has_brief_description(self):
 621         return self._rawdoc.has_brief_description()
 622
 623     def get_doc_type(self):
 624         """Return documentation type (visibility) for the class.
 625
 626         In addition to the actual code, this encodes GROMACS-specific logic
 627         of setting EXTRACT_LOCAL_CLASSES=YES only for the full documentation.
 628         Local classes never appear outside the full documentation, no matter
 629         what is their visibility.
 630         """
 631         if not self.is_documented():
 632             return DocType.none
 633         if self._rawdoc.is_local():
 634             return DocType.internal
 635         return self._rawdoc.get_visibility()
 636
 637     def get_file_doc_type(self):
 638         return max([fileobj.get_doc_type() for fileobj in self._files])
 639
 640     def is_in_installed_file(self):
 641         return any([fileobj.is_installed() for fileobj in self._files])
 642
 643 class Member(object):
 644
 645     """Member (in Doxygen terminology) in the GROMACS source tree.
 646
 647     Currently, modeling is limited to the minimal set of properties that the
 648     checker uses.
 649     """
 650
 651     def __init__(self, rawdoc, namespace):
 652         self._rawdoc = rawdoc
 653         self._namespace = namespace
 654
 655     def get_name(self):
 656         return self._rawdoc.get_name()
 657
 658     def get_reporter_location(self):
 659         return self._rawdoc.get_reporter_location()
 660
 661     def is_documented(self):
 662         return self._rawdoc.is_documented()
 663
 664     def has_brief_description(self):
 665         return self._rawdoc.has_brief_description()
 666
 667     def has_inbody_description(self):
 668         return self._rawdoc.has_inbody_description()
 669
 670     def is_visible(self):
 671         """Return whether the member is visible in Doxygen documentation.
 672
 673         Doxygen ignores members whose parent compounds are not documented.
 674         However, when EXTRACT_ANON_NPACES=ON (which is set for our full
 675         documentation), members of anonymous namespaces are extracted even if
 676         the namespace is the only parent and is not documented.
 677         """
 678         if self._namespace and self._namespace.is_anonymous():
 679             return True
 680         return self._rawdoc.get_inherited_visibility() != DocType.none
 681
 682
 683 class GromacsTree(object):
 684
 685     """Root object for navigating the GROMACS source tree.
 686
 687     On initialization, the list of files and directories is initialized by
 688     walking the source tree, and modules are created for top-level
 689     subdirectories.  At this point, only information that is accessible from
 690     file names and paths only is available.
 691
 692     load_git_attributes() can be called to load attribute information from
 693     .gitattributes for all the files.
 694
 695     load_installed_file_list() can be called to load the list of installed
 696     files from the build tree (generated by CMake).
 697
 698     scan_files() can be called to read all the files and initialize #include
 699     dependencies between the files based on the information.  This is done like
 700     this instead of relying on Doxygen-extracted include files to make the
 701     dependency graph independent from preprocessor macro definitions
 702     (Doxygen only sees those #includes that the preprocessor sees, which
 703     depends on what #defines it has seen).
 704
 705     find_define_file_uses() can be called to find all uses of defines
 706     declared in config.h and some other macro headers. In the current
 707     implementation, scan_files() must have been called earlier.
 708
 709     load_xml() can be called to load information from Doxygen XML data in
 710     the build tree (the Doxygen XML data must have been built separately).
 711     """
 712
 713     def __init__(self, source_root, build_root, reporter):
 714         """Initialize the tree object by walking the source tree."""
 715         self._source_root = os.path.abspath(source_root)
 716         self._build_root = os.path.abspath(build_root)
 717         self._reporter = reporter
 718         self._docset = None
 719         self._docmap = dict()
 720         self._dirs = dict()
 721         self._files = dict()
 722         self._modules = dict()
 723         self._classes = set()
 724         self._namespaces = set()
 725         self._members = set()
 726         self._walk_dir(os.path.join(self._source_root, 'src'))
 727         for fileobj in self.get_files():
 728             if fileobj and fileobj.is_source_file() and not fileobj.is_external():
 729                 (basedir, name) = os.path.split(fileobj.get_abspath())
 730                 (basename, ext) = os.path.splitext(name)
 731                 header = self.get_file(os.path.join(basedir, basename + '.h'))
 732                 if not header and ext == '.cu':
 733                     header = self.get_file(os.path.join(basedir, basename + '.cuh'))
 734                 if not header and fileobj.is_test_file():
 735                     basedir = os.path.dirname(basedir)
 736                     header = self.get_file(os.path.join(basedir, basename + '.h'))
 737                     if not header:
 738                         # Somewhat of a hack; currently, the tests for
 739                         # analysisdata/modules/ and trajectoryanalysis/modules/
 740                         # is at the top-level tests directory.
 741                         # TODO: It could be clearer to split the tests so that
 742                         # there would be a separate modules/tests/.
 743                         header = self.get_file(os.path.join(basedir, 'modules', basename + '.h'))
 744                     if not header and basename.endswith('_tests'):
 745                         header = self.get_file(os.path.join(basedir, basename[:-6] + '.h'))
 746                 if header:
 747                     fileobj.set_main_header(header)
 748         rootdir = self._get_dir(os.path.join('src', 'gromacs'))
 749         for subdir in rootdir.get_subdirectories():
 750             self._create_module(subdir)
 751         rootdir = self._get_dir(os.path.join('src', 'testutils'))
 752         self._create_module(rootdir)
 753
 754     def _get_rel_path(self, path):
 755         assert os.path.isabs(path)
 756         if path.startswith(self._build_root):
 757             return os.path.relpath(path, self._build_root)
 758         if path.startswith(self._source_root):
 759             return os.path.relpath(path, self._source_root)
 760         raise ValueError("path not under build nor source tree: {0}".format(path))
 761
 762     def _walk_dir(self, rootpath):
 763         """Construct representation of the source tree by walking the file system."""
 764         assert os.path.isabs(rootpath)
 765         assert rootpath not in self._dirs
 766         relpath = self._get_rel_path(rootpath)
 767         self._dirs[relpath] = Directory(rootpath, relpath, None)
 768         for dirpath, dirnames, filenames in os.walk(rootpath):
 769             if 'contrib' in dirnames:
 770                 dirnames.remove('contrib')
 771             if 'refdata' in dirnames:
 772                 dirnames.remove('refdata')
 773             currentdir = self._dirs[self._get_rel_path(dirpath)]
 774             # Loop through a copy so that we can modify dirnames.
 775             for dirname in list(dirnames):
 776                 fullpath = os.path.join(dirpath, dirname)
 777                 if fullpath == self._build_root:
 778                     dirnames.remove(dirname)
 779                     continue
 780                 relpath = self._get_rel_path(fullpath)
 781                 self._dirs[relpath] = Directory(fullpath, relpath, currentdir)
 782             extensions = ('.h', '.cuh', '.hpp', '.c', '.cc', '.cpp', '.cu', '.bm')
 783             for filename in filenames:
 784                 basename, extension = os.path.splitext(filename)
 785                 if extension in extensions:
 786                     fullpath = os.path.join(dirpath, filename)
 787                     relpath = self._get_rel_path(fullpath)
 788                     self._files[relpath] = File(fullpath, relpath, currentdir)
 789                 elif extension == '.cmakein':
 790                     extension = os.path.splitext(basename)[1]
 791                     if extension in extensions:
 792                         fullpath = os.path.join(dirpath, filename)
 793                         relpath = self._get_rel_path(fullpath)
 794                         sourcefile = GeneratorSourceFile(fullpath, relpath, currentdir)
 795                         self._files[relpath] = sourcefile
 796                         fullpath = os.path.join(dirpath, basename)
 797                         relpath = self._get_rel_path(fullpath)
 798                         fullpath = os.path.join(self._build_root, relpath)
 799                         generatedfile = GeneratedFile(fullpath, relpath, currentdir)
 800                         self._files[relpath] = generatedfile
 801                         generatedfile.set_generator_source(sourcefile)
 802                 elif extension in ('.l', '.y', '.pre'):
 803                     fullpath = os.path.join(dirpath, filename)
 804                     relpath = self._get_rel_path(fullpath)
 805                     self._files[relpath] = GeneratorSourceFile(fullpath, relpath, currentdir)
 806
 807     def _create_module(self, rootdir):
 808         """Create module for a subdirectory."""
 809         name = 'module_' + rootdir.get_name()
 810         moduleobj = Module(name, rootdir)
 811         rootdir.set_module(moduleobj)
 812         self._modules[name] = moduleobj
 813
 814     def scan_files(self, only_files=None, keep_contents=False):
 815         """Read source files to initialize #include dependencies."""
 816         if only_files:
 817             filelist = only_files
 818         else:
 819             filelist = self._files.itervalues()
 820         define_files = list(self.get_checked_define_files())
 821         for define_file in list(define_files):
 822             if isinstance(define_file, GeneratedFile) and \
 823                     define_file.get_generator_source() is not None:
 824                 define_files.append(define_file.get_generator_source())
 825         for fileobj in filelist:
 826             if not fileobj.is_external():
 827                 detect_defines = fileobj in define_files
 828                 fileobj.scan_contents(self, keep_contents, detect_defines)
 829                 module = fileobj.get_module()
 830                 if module:
 831                     for includedfile in fileobj.get_includes():
 832                         otherfile = includedfile.get_file()
 833                         if otherfile:
 834                             othermodule = otherfile.get_module()
 835                             if othermodule and othermodule != module:
 836                                 module.add_dependency(othermodule, includedfile)
 837
 838     def load_xml(self, only_files=None):
 839         """Load Doxygen XML information.
 840
 841         If only_files is True, XML data is not loaded for code constructs, but
 842         only for files, directories, and their potential parents.
 843         """
 844         xmldir = os.path.join(self._build_root, 'docs', 'html', 'doxygen', 'xml')
 845         self._docset = xml.DocumentationSet(xmldir, self._reporter)
 846         if only_files:
 847             if isinstance(only_files, collections.Iterable):
 848                 filelist = [x.get_relpath() for x in only_files]
 849                 self._docset.load_file_details(filelist)
 850             else:
 851                 self._docset.load_file_details()
 852         else:
 853             self._docset.load_details()
 854             self._docset.merge_duplicates()
 855         self._load_dirs()
 856         self._load_modules()
 857         self._load_files()
 858         if not only_files:
 859             self._load_namespaces()
 860             self._load_classes()
 861             self._load_members()
 862
 863     def _load_dirs(self):
 864         """Load Doxygen XML directory information."""
 865         rootdirs = self._docset.get_compounds(xml.Directory,
 866                 lambda x: x.get_parent() is None)
 867         for dirdoc in rootdirs:
 868             self._load_dir(dirdoc, None)
 869
 870     def _load_dir(self, dirdoc, parent):
 871         """Load Doxygen XML directory information for a single directory."""
 872         path = dirdoc.get_path().rstrip('/')
 873         if not os.path.isabs(path):
 874             path = os.path.join(self._source_root, path)
 875         relpath = self._get_rel_path(path)
 876         dirobj = self._dirs.get(relpath)
 877         if not dirobj:
 878             dirobj = Directory(path, relpath, parent)
 879             self._dirs[relpath] = dirobj
 880         dirobj.set_doc_xml(dirdoc, self)
 881         self._docmap[dirdoc] = dirobj
 882         for subdirdoc in dirdoc.get_subdirectories():
 883             self._load_dir(subdirdoc, dirobj)
 884
 885     def _load_modules(self):
 886         """Load Doxygen XML module (group) information."""
 887         moduledocs = self._docset.get_compounds(xml.Group,
 888                 lambda x: x.get_name().startswith('module_'))
 889         for moduledoc in moduledocs:
 890             moduleobj = self._modules.get(moduledoc.get_name())
 891             if not moduleobj:
 892                 self._reporter.input_error(
 893                         "no matching directory for module: {0}".format(moduledoc))
 894                 continue
 895             moduleobj.set_doc_xml(moduledoc, self)
 896             self._docmap[moduledoc] = moduleobj
 897
 898     def _load_files(self):
 899         """Load Doxygen XML file information."""
 900         for filedoc in self._docset.get_files():
 901             path = filedoc.get_path()
 902             if not path:
 903                 # In case of only partially loaded file information,
 904                 # the path information is not set for unloaded files.
 905                 continue
 906             if not os.path.isabs(path):
 907                 path = os.path.join(self._source_root, path)
 908             extension = os.path.splitext(path)[1]
 909             # We don't care about Markdown files that only produce pages
 910             # (and fail the directory check below).
 911             if extension == '.md':
 912                 continue
 913             dirdoc = filedoc.get_directory()
 914             if not dirdoc:
 915                 self._reporter.xml_assert(filedoc.get_xml_path(),
 916                         "file is not in any directory in Doxygen")
 917                 continue
 918             relpath = self._get_rel_path(path)
 919             fileobj = self._files.get(relpath)
 920             if not fileobj:
 921                 fileobj = File(path, relpath, self._docmap[dirdoc])
 922                 self._files[relpath] = fileobj
 923             fileobj.set_doc_xml(filedoc, self)
 924             self._docmap[filedoc] = fileobj
 925
 926     def _load_namespaces(self):
 927         """Load Doxygen XML namespace information."""
 928         nsdocs = self._docset.get_namespaces()
 929         for nsdoc in nsdocs:
 930             nsobj = Namespace(nsdoc)
 931             self._docmap[nsdoc] = nsobj
 932             self._namespaces.add(nsobj)
 933
 934     def _load_classes(self):
 935         """Load Doxygen XML class information."""
 936         classdocs = self._docset.get_classes()
 937         for classdoc in classdocs:
 938             files = [self._docmap[filedoc] for filedoc in classdoc.get_files()]
 939             classobj = Class(classdoc, files)
 940             self._docmap[classdoc] = classobj
 941             self._classes.add(classobj)
 942
 943     def _load_members(self):
 944         """Load Doxygen XML member information."""
 945         memberdocs = self._docset.get_members()
 946         for memberdoc in memberdocs:
 947             nsdoc = memberdoc.get_namespace()
 948             nsobj = self.get_object(nsdoc)
 949             memberobj = Member(memberdoc, nsobj)
 950             self._docmap[memberdoc] = memberobj
 951             self._members.add(memberobj)
 952
 953     def _get_dir(self, relpath):
 954         """Get directory object for a path relative to source tree root."""
 955         return self._dirs.get(relpath)
 956
 957     def get_file(self, path):
 958         """Get file object for a path relative to source tree root."""
 959         return self._files.get(self._get_rel_path(path))
 960
 961     def find_include_file(self, includedpath):
 962         """Find a file object corresponding to an include path."""
 963         for testdir in ('src', 'src/external/thread_mpi/include',
 964                 'src/external/tng_io/include'):
 965             testpath = os.path.join(testdir, includedpath)
 966             if testpath in self._files:
 967                 return self._files[testpath]
 968
 969     def load_git_attributes(self):
 970         """Load git attribute information for files."""
 971         args = ['git', 'check-attr', '--stdin', 'filter']
 972         git_check_attr = subprocess.Popen(args, stdin=subprocess.PIPE,
 973                 stdout=subprocess.PIPE, cwd=self._source_root)
 974         filelist = '\n'.join(map(File.get_relpath, self._files.itervalues()))
 975         filters = git_check_attr.communicate(filelist)[0]
 976         for fileinfo in filters.splitlines():
 977             path, dummy, value = fileinfo.split(': ')
 978             fileobj = self._files.get(path)
 979             assert fileobj is not None
 980             fileobj.set_git_filter_attribute(value)
 981
 982     def find_define_file_uses(self):
 983         """Find files that use defines from config.h."""
 984         # Executing git grep is substantially faster than using the define_re
 985         # directly on the contents of the file in Python.
 986         for define_file in self.get_checked_define_files():
 987             excluded_files = set([define_file])
 988             excluded_files.update(define_file.get_included_files(recursive=True))
 989             all_defines = define_file.get_declared_defines()
 990             args = ['git', 'grep', '-zwIF']
 991             for define in all_defines:
 992                 args.extend(['-e', define])
 993             args.extend(['--', '*.cpp', '*.c', '*.cu', '*.h', '*.cuh'])
 994             define_re = r'\b(?:' + '|'.join(all_defines)+ r')\b'
 995             output = subprocess.check_output(args, cwd=self._source_root)
 996             for line in output.splitlines():
 997                 (filename, text) = line.split('\0')
 998                 fileobj = self._files.get(filename)
 999                 if fileobj is not None and fileobj not in excluded_files:
1000                     defines = re.findall(define_re, text)
1001                     fileobj.add_used_defines(define_file, defines)
1002
1003     def load_installed_file_list(self):
1004         """Load list of installed files from the build tree."""
1005         listpath = os.path.join(self._build_root, 'src', 'gromacs', 'installed-headers.txt')
1006         with open(listpath, 'r') as installedfp:
1007             for line in installedfp:
1008                 path = line.strip()
1009                 if not os.path.isabs(path):
1010                     self._reporter.input_error(
1011                             "installed file not specified with absolute path: {0}"
1012                             .format(path))
1013                     continue
1014                 relpath = self._get_rel_path(path)
1015                 if relpath not in self._files:
1016                     self._reporter.input_error(
1017                             "installed file not in source tree: {0}".format(path))
1018                     continue
1019                 self._files[relpath].set_installed()
1020
1021     def load_cycle_suppression_list(self, filename):
1022         """Load a list of edges to suppress in cycles.
1023
1024         These edges between modules, if present, will be marked in the
1025         corresponding ModuleDependency objects.
1026         """
1027         with open(filename, 'r') as fp:
1028             for line in fp:
1029                 line = line.strip()
1030                 if not line or line.startswith('#'):
1031                     continue
1032                 modulenames = ['module_' + x.strip() for x in line.split('->')]
1033                 if len(modulenames) != 2:
1034                     self._reporter.input_error(
1035                             "invalid cycle suppression line: {0}".format(line))
1036                     continue
1037                 firstmodule = self._modules.get(modulenames[0])
1038                 secondmodule = self._modules.get(modulenames[1])
1039                 if not firstmodule or not secondmodule:
1040                     self._reporter.input_error(
1041                             "unknown modules mentioned on cycle suppression line: {0}".format(line))
1042                     continue
1043                 for dep in firstmodule.get_dependencies():
1044                     if dep.get_other_module() == secondmodule:
1045                         dep.set_cycle_suppression()
1046                         break
1047                 else:
1048                     self._reporter.cyclic_issue("unused cycle suppression: {0}".format(line))
1049
1050     def report_unused_cycle_suppressions(self, reporter):
1051         """Reports unused cycle suppressions."""
1052         for module in self.get_modules():
1053             for dep in module.get_dependencies():
1054                 if not dep.suppression_used:
1055                     reporter.cyclic_issue("unused cycle suppression: {0} -> {1}".format(module.get_name()[7:], dep.get_other_module().get_name()[7:]))
1056
1057     def get_object(self, docobj):
1058         """Get tree object for a Doxygen XML object."""
1059         if docobj is None:
1060             return None
1061         return self._docmap.get(docobj)
1062
1063     def get_files(self):
1064         """Get iterable for all files in the source tree."""
1065         return self._files.itervalues()
1066
1067     def get_modules(self):
1068         """Get iterable for all modules in the source tree."""
1069         return self._modules.itervalues()
1070
1071     def get_classes(self):
1072         """Get iterable for all classes in the source tree."""
1073         return self._classes
1074
1075     def get_members(self):
1076         """Get iterable for all members (in Doxygen terms) in the source tree."""
1077         return self._members
1078
1079     def get_checked_define_files(self):
1080         """Get list of files that contain #define macros whose usage needs to
1081         be checked."""
1082         return (self._files['src/config.h'],
1083                 self._files['src/gromacs/simd/simd.h'],
1084                 self._files['src/gromacs/ewald/pme-simd.h'],
1085                 self._files['src/gromacs/mdlib/nbnxn_simd.h'])