docs/doxygen/gmxtree.py

   1 #!/usr/bin/python
   2 #
   3 # This file is part of the GROMACS molecular simulation package.
   4 #
   5 # Copyright (c) 2014,2015, by the GROMACS development team, led by
   6 # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   7 # and including many others, as listed in the AUTHORS file in the
   8 # top-level source directory and at http://www.gromacs.org.
   9 #
  10 # GROMACS is free software; you can redistribute it and/or
  11 # modify it under the terms of the GNU Lesser General Public License
  12 # as published by the Free Software Foundation; either version 2.1
  13 # of the License, or (at your option) any later version.
  14 #
  15 # GROMACS is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18 # Lesser General Public License for more details.
  19 #
  20 # You should have received a copy of the GNU Lesser General Public
  21 # License along with GROMACS; if not, see
  22 # http://www.gnu.org/licenses, or write to the Free Software Foundation,
  23 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  24 #
  25 # If you want to redistribute modifications to GROMACS, please
  26 # consider that scientific software is very special. Version
  27 # control is crucial - bugs must be traceable. We will be happy to
  28 # consider code for inclusion in the official distribution, but
  29 # derived work must not be called official GROMACS. Details are found
  30 # in the README & COPYING files - if they are missing, get the
  31 # official version at http://www.gromacs.org.
  32 #
  33 # To help us fund GROMACS development, we humbly ask that you cite
  34 # the research papers on the package. Check out http://www.gromacs.org.
  35
  36 """GROMACS-specific representation for source tree and documentation.
  37
  38 This module provides classes that construct a GROMACS-specific representation
  39 of the source tree and associate the Doxygen XML output with it.  It constructs
  40 an initial representation by walking the source tree in the file system, and
  41 then associates information from the Doxygen XML output into this.
  42 It also adds some additional knowledge from how the GROMACS source tree is
  43 organized to construct a representation that is easy to process and check as
  44 the top-level scripts expect.
  45
  46 The object model is rooted at a GromacsTree object.  Currently, it constructs a
  47 representation of the source tree from the file system, but is otherwise mostly
  48 a thin wrapper around the Doxygen XML tree.  It already adds some relations and
  49 rules that come from GROMACS-specific knowledge.  In the future, more such
  50 customizations will be added.
  51 """
  52
  53 import collections
  54 import os
  55 import os.path
  56 import re
  57 import subprocess
  58
  59 import doxygenxml as xml
  60 import reporter
  61 # We import DocType directly so that it is exposed from this module as well.
  62 from doxygenxml import DocType
  63
  64 def _get_api_type_for_compound(grouplist):
  65     """Helper function to deduce API type from Doxygen group membership."""
  66     result = DocType.internal
  67     for group in grouplist:
  68         if isinstance(group, xml.Group):
  69             if group.get_name() == 'group_publicapi':
  70                 result = DocType.public
  71             elif group.get_name() == 'group_libraryapi':
  72                 result = DocType.library
  73             # TODO: Check for multiple group membership
  74     return result
  75
  76 class IncludedFile(object):
  77
  78     """Information about an #include directive in a file."""
  79
  80     def __init__(self, including_file, lineno, included_file, included_path, is_relative, is_system, line):
  81         self._including_file = including_file
  82         self._line_number = lineno
  83         self._included_file = included_file
  84         self._included_path = included_path
  85         #self._used_include_path = used_include_path
  86         self._is_relative = is_relative
  87         self._is_system = is_system
  88         self._line = line
  89
  90     def __str__(self):
  91         if self._is_system:
  92             return '<{0}>'.format(self._included_path)
  93         else:
  94             return '"{0}"'.format(self._included_path)
  95
  96     def is_system(self):
  97         return self._is_system
  98
  99     def is_relative(self):
 100         return self._is_relative
 101
 102     def get_included_path(self):
 103         return self._included_path
 104
 105     def get_including_file(self):
 106         return self._including_file
 107
 108     def get_file(self):
 109         return self._included_file
 110
 111     def get_line_number(self):
 112         return self._line_number
 113
 114     def get_full_line(self):
 115         """Return the full source line on which this include appears.
 116
 117         Trailing newline is included."""
 118         return self._line
 119
 120     def get_reporter_location(self):
 121         return reporter.Location(self._including_file.get_abspath(), self._line_number)
 122
 123 class IncludeBlock(object):
 124
 125     """Block of consequent #include directives in a file."""
 126
 127     def __init__(self, first_included_file):
 128         self._first_line = first_included_file.get_line_number()
 129         self._last_line = self._first_line
 130         self._files = []
 131         self.add_file(first_included_file)
 132
 133     def add_file(self, included_file):
 134         self._files.append(included_file)
 135         self._last_line = included_file.get_line_number()
 136
 137     def get_includes(self):
 138         return self._files
 139
 140     def get_first_line(self):
 141         return self._first_line
 142
 143     def get_last_line(self):
 144         return self._last_line
 145
 146 class File(object):
 147
 148     """Source/header file in the GROMACS tree."""
 149
 150     def __init__(self, abspath, relpath, directory):
 151         """Initialize a file representation with basic information."""
 152         self._abspath = abspath
 153         self._relpath = relpath
 154         self._dir = directory
 155         self._rawdoc = None
 156         self._installed = False
 157         extension = os.path.splitext(abspath)[1]
 158         self._sourcefile = (extension in ('.c', '.cc', '.cpp', '.cu'))
 159         self._apitype = DocType.none
 160         self._modules = set()
 161         self._includes = []
 162         self._include_blocks = []
 163         self._main_header = None
 164         self._lines = None
 165         self._filter = None
 166         self._declared_defines = None
 167         self._used_define_files = set()
 168         directory.add_file(self)
 169
 170     def set_doc_xml(self, rawdoc, sourcetree):
 171         """Assiociate Doxygen documentation entity with the file."""
 172         assert self._rawdoc is None
 173         assert rawdoc.is_source_file() == self._sourcefile
 174         self._rawdoc = rawdoc
 175         if self._rawdoc.is_documented():
 176             grouplist = self._rawdoc.get_groups()
 177             self._apitype = _get_api_type_for_compound(grouplist)
 178             for group in grouplist:
 179                 module = sourcetree.get_object(group)
 180                 if module:
 181                     self._modules.add(module)
 182
 183     def set_installed(self):
 184         """Mark the file installed."""
 185         self._installed = True
 186
 187     def set_git_filter_attribute(self, filtername):
 188         """Set the git filter attribute associated with the file."""
 189         self._filter = filtername
 190
 191     def set_main_header(self, included_file):
 192         """Set the main header file for a source file."""
 193         assert self.is_source_file()
 194         self._main_header = included_file
 195
 196     def _process_include(self, lineno, is_system, includedpath, line, sourcetree):
 197         """Process #include directive during scan()."""
 198         is_relative = False
 199         if is_system:
 200             fileobj = sourcetree.find_include_file(includedpath)
 201         else:
 202             fullpath = os.path.join(self._dir.get_abspath(), includedpath)
 203             fullpath = os.path.abspath(fullpath)
 204             if os.path.exists(fullpath):
 205                 is_relative = True
 206                 fileobj = sourcetree.get_file(fullpath)
 207             else:
 208                 fileobj = sourcetree.find_include_file(includedpath)
 209         included_file = IncludedFile(self, lineno, fileobj, includedpath,
 210             is_relative, is_system, line)
 211         self._includes.append(included_file)
 212         return included_file
 213
 214     def scan_contents(self, sourcetree, keep_contents, detect_defines):
 215         """Scan the file contents and initialize information based on it."""
 216         # TODO: Consider a more robust regex.
 217         include_re = r'^\s*#\s*include\s+(?P<quote>["<])(?P<path>[^">]*)[">]'
 218         define_re = r'^\s*#.*define(?:01)?\s+(\w*)'
 219         current_block = None
 220         with open(self._abspath, 'r') as scanfile:
 221             contents = scanfile.read()
 222         lines = contents.splitlines(True)
 223         for lineno, line in enumerate(lines, 1):
 224             match = re.match(include_re, line)
 225             if match:
 226                 is_system = (match.group('quote') == '<')
 227                 includedpath = match.group('path')
 228                 included_file = self._process_include(lineno, is_system,
 229                         includedpath, line, sourcetree)
 230                 if current_block is None:
 231                     current_block = IncludeBlock(included_file)
 232                     self._include_blocks.append(current_block)
 233                 else:
 234                     current_block.add_file(included_file)
 235             elif line and not line.isspace():
 236                 current_block = None
 237         if detect_defines:
 238             self._declared_defines = []
 239             for line in lines:
 240                 match = re.match(define_re, line)
 241                 if match:
 242                     self._declared_defines.append(match.group(1))
 243         if keep_contents:
 244             self._lines = lines
 245
 246     def add_used_defines(self, define_file, defines):
 247         """Add defines used in this file.
 248
 249         Used internally by find_define_file_uses()."""
 250         self._used_define_files.add(define_file)
 251
 252     def get_reporter_location(self):
 253         return reporter.Location(self._abspath, None)
 254
 255     def is_installed(self):
 256         return self._installed
 257
 258     def is_external(self):
 259         return self._dir.is_external()
 260
 261     def is_source_file(self):
 262         return self._sourcefile
 263
 264     def is_test_file(self):
 265         return self._dir.is_test_directory()
 266
 267     def should_includes_be_sorted(self):
 268         """Return whether the include directives in the file should be sorted."""
 269         return self._filter in ('includesort', 'uncrustify')
 270
 271     def is_documented(self):
 272         return self._rawdoc and self._rawdoc.is_documented()
 273
 274     def has_brief_description(self):
 275         return self._rawdoc and self._rawdoc.has_brief_description()
 276
 277     def get_abspath(self):
 278         return self._abspath
 279
 280     def get_relpath(self):
 281         return self._relpath
 282
 283     def get_name(self):
 284         return os.path.basename(self._abspath)
 285
 286     def get_directory(self):
 287         return self._dir
 288
 289     def get_doc_type(self):
 290         if not self._rawdoc:
 291             return DocType.none
 292         return self._rawdoc.get_visibility()
 293
 294     def get_api_type(self):
 295         return self._apitype
 296
 297     def api_type_is_reliable(self):
 298         if self._apitype in (DocType.internal, DocType.library):
 299             return True
 300         module = self.get_module()
 301         return module and module.is_documented()
 302
 303     def is_public(self):
 304         if self.api_type_is_reliable():
 305             return self.get_api_type() == DocType.public
 306         return self.get_api_type() == DocType.public or self.is_installed()
 307
 308     def is_module_internal(self):
 309         if self.is_source_file():
 310             return True
 311         return not self.is_installed() and self.get_api_type() <= DocType.internal
 312
 313     def get_expected_module(self):
 314         return self._dir.get_module()
 315
 316     def get_doc_modules(self):
 317         return self._modules
 318
 319     def get_module(self):
 320         module = self.get_expected_module()
 321         if not module and len(self._modules) == 1:
 322             module = list(self._modules)[0]
 323         return module
 324
 325     def get_includes(self):
 326         return self._includes
 327
 328     def get_include_blocks(self):
 329         return self._include_blocks
 330
 331     def _get_included_files_recurse(self, result):
 332         for include in self._includes:
 333             included_file = include.get_file()
 334             if included_file is not None and not included_file in result:
 335                 result.add(included_file)
 336                 included_file._get_included_files_recurse(result)
 337
 338     def get_included_files(self, recursive=False):
 339         if recursive:
 340             result = set()
 341             self._get_included_files_recurse(result)
 342             return result
 343         return set([x.get_file() for x in self._includes])
 344
 345     def get_main_header(self):
 346         return self._main_header
 347
 348     def get_contents(self):
 349         return self._lines
 350
 351     def get_declared_defines(self):
 352         """Return set of defines declared in this file.
 353
 354         The information is only populated for selected files."""
 355         return self._declared_defines
 356
 357     def get_used_define_files(self):
 358         """Return set of defines from config.h that are used in this file.
 359
 360         The return value is empty if find_define_file_uses() has not been called,
 361         as well as for headers that declare these defines."""
 362         return self._used_define_files
 363
 364 class GeneratedFile(File):
 365     def __init__(self, abspath, relpath, directory):
 366         File.__init__(self, abspath, relpath, directory)
 367         self._generator_source_file = None
 368
 369     def scan_contents(self, sourcetree, keep_contents, detect_defines):
 370         if os.path.exists(self.get_abspath()):
 371             File.scan_contents(self, sourcetree, keep_contents, False)
 372
 373     def set_generator_source(self, sourcefile):
 374         self._generator_source_file = sourcefile
 375
 376     def get_generator_source(self):
 377         return self._generator_source_file
 378
 379     def get_reporter_location(self):
 380         if self._generator_source_file:
 381             return self._generator_source_file.get_reporter_location()
 382         return File.get_reporter_location(self)
 383
 384     def get_declared_defines(self):
 385         if self._generator_source_file:
 386             return self._generator_source_file.get_declared_defines()
 387         return File.get_declared_defines(self)
 388
 389 class GeneratorSourceFile(File):
 390     pass
 391
 392 class Directory(object):
 393
 394     """(Sub)directory in the GROMACS tree."""
 395
 396     def __init__(self, abspath, relpath, parent):
 397         """Initialize a file representation with basic information."""
 398         self._abspath = abspath
 399         self._relpath = relpath
 400         self._name = os.path.basename(abspath)
 401         self._parent = parent
 402         self._rawdoc = None
 403         self._module = None
 404         self._is_test_dir = False
 405         if parent and parent.is_test_directory() or \
 406                 self._name in ('tests', 'legacytests'):
 407             self._is_test_dir = True
 408         self._is_external = False
 409         if parent and parent.is_external() or self._name == 'external':
 410             self._is_external = True
 411         self._subdirs = set()
 412         if parent:
 413             parent._subdirs.add(self)
 414         self._files = set()
 415         self._has_installed_files = None
 416
 417     def set_doc_xml(self, rawdoc, sourcetree):
 418         """Assiociate Doxygen documentation entity with the directory."""
 419         assert self._rawdoc is None
 420         assert self._abspath == rawdoc.get_path().rstrip('/')
 421         self._rawdoc = rawdoc
 422
 423     def set_module(self, module):
 424         assert self._module is None
 425         self._module = module
 426
 427     def add_file(self, fileobj):
 428         self._files.add(fileobj)
 429
 430     def get_name(self):
 431         return self._name
 432
 433     def get_reporter_location(self):
 434         return reporter.Location(self._abspath, None)
 435
 436     def get_abspath(self):
 437         return self._abspath
 438
 439     def get_relpath(self):
 440         return self._relpath
 441
 442     def is_test_directory(self):
 443         return self._is_test_dir
 444
 445     def is_external(self):
 446         return self._is_external
 447
 448     def has_installed_files(self):
 449         if self._has_installed_files is None:
 450             self._has_installed_files = False
 451             for subdir in self._subdirs:
 452                 if subdir.has_installed_files():
 453                     self._has_installed_files = True
 454                     return True
 455             for fileobj in self._files:
 456                 if fileobj.is_installed():
 457                     self._has_installed_files = True
 458                     return True
 459         return self._has_installed_files
 460
 461     def get_module(self):
 462         if self._module:
 463             return self._module
 464         if self._parent:
 465             return self._parent.get_module()
 466         return None
 467
 468     def get_subdirectories(self):
 469         return self._subdirs
 470
 471     def get_files(self):
 472         for subdir in self._subdirs:
 473             for fileobj in subdir.get_files():
 474                 yield fileobj
 475         for fileobj in self._files:
 476             yield fileobj
 477
 478     def contains(self, fileobj):
 479         """Check whether file is within the directory or its subdirectories."""
 480         dirobj = fileobj.get_directory()
 481         while dirobj:
 482             if dirobj == self:
 483                 return True
 484             dirobj = dirobj._parent
 485         return False
 486
 487 class ModuleDependency(object):
 488
 489     """Dependency between modules."""
 490
 491     def __init__(self, othermodule):
 492         """Initialize empty dependency object with given module as dependency."""
 493         self._othermodule = othermodule
 494         self._includedfiles = []
 495         self._cyclesuppression = None
 496         self._is_test_only_dependency = True
 497         self.suppression_used = True
 498
 499     def add_included_file(self, includedfile):
 500         """Add IncludedFile that is part of this dependency."""
 501         assert includedfile.get_file().get_module() == self._othermodule
 502         if not includedfile.get_including_file().is_test_file():
 503             self._is_test_only_dependency = False
 504         self._includedfiles.append(includedfile)
 505
 506     def set_cycle_suppression(self):
 507         """Set suppression on cycles containing this dependency."""
 508         self._cyclesuppression = True
 509         self.suppression_used = False
 510
 511     def is_cycle_suppressed(self):
 512         """Return whether cycles containing this dependency are suppressed."""
 513         self.suppression_used = True
 514         return self._cyclesuppression is not None
 515
 516     def is_test_only_dependency(self):
 517         """Return whether this dependency is only from test code."""
 518         return self._is_test_only_dependency
 519
 520     def get_other_module(self):
 521         """Get module that this dependency is to."""
 522         return self._othermodule
 523
 524     def get_included_files(self):
 525         """Get IncludedFile objects for the individual include dependencies."""
 526         return self._includedfiles
 527
 528 class Module(object):
 529
 530     """Code module in the GROMACS source tree.
 531
 532     Modules are specific subdirectories that host a more or less coherent
 533     set of routines.  Simplified, every subdirectory under src/gromacs/ is
 534     a different module.  This object provides that abstraction and also links
 535     the subdirectory to the module documentation (documented as a group in
 536     Doxygen) if that exists.
 537     """
 538
 539     def __init__(self, name, rootdir):
 540         self._name = name
 541         self._rawdoc = None
 542         self._rootdir = rootdir
 543         self._group = None
 544         self._dependencies = dict()
 545
 546     def set_doc_xml(self, rawdoc, sourcetree):
 547         """Assiociate Doxygen documentation entity with the module."""
 548         assert self._rawdoc is None
 549         self._rawdoc = rawdoc
 550         if self._rawdoc.is_documented():
 551             groups = list(self._rawdoc.get_groups())
 552             if len(groups) == 1:
 553                 groupname = groups[0].get_name()
 554                 if groupname.startswith('group_'):
 555                     self._group = groupname[6:]
 556
 557     def add_dependency(self, othermodule, includedfile):
 558         """Add #include dependency from a file in this module."""
 559         assert includedfile.get_file().get_module() == othermodule
 560         if othermodule not in self._dependencies:
 561             self._dependencies[othermodule] = ModuleDependency(othermodule)
 562         self._dependencies[othermodule].add_included_file(includedfile)
 563
 564     def is_documented(self):
 565         return self._rawdoc is not None
 566
 567     def get_name(self):
 568         return self._name
 569
 570     def get_root_dir(self):
 571         return self._rootdir
 572
 573     def get_files(self):
 574         # TODO: Include public API convenience headers?
 575         return self._rootdir.get_files()
 576
 577     def get_group(self):
 578         return self._group
 579
 580     def get_dependencies(self):
 581         return self._dependencies.itervalues()
 582
 583 class Namespace(object):
 584
 585     """Namespace in the GROMACS source code."""
 586
 587     def __init__(self, rawdoc):
 588         self._rawdoc = rawdoc
 589
 590     def is_anonymous(self):
 591         return self._rawdoc.is_anonymous()
 592
 593 class Class(object):
 594
 595     """Class/struct/union in the GROMACS source code."""
 596
 597     def __init__(self, rawdoc, files):
 598         self._rawdoc = rawdoc
 599         self._files = set(files)
 600
 601     def get_name(self):
 602         return self._rawdoc.get_name()
 603
 604     def get_reporter_location(self):
 605         return self._rawdoc.get_reporter_location()
 606
 607     def get_files(self):
 608         return self._files
 609
 610     def is_documented(self):
 611         return self._rawdoc.is_documented()
 612
 613     def has_brief_description(self):
 614         return self._rawdoc.has_brief_description()
 615
 616     def get_doc_type(self):
 617         """Return documentation type (visibility) for the class.
 618
 619         In addition to the actual code, this encodes GROMACS-specific logic
 620         of setting EXTRACT_LOCAL_CLASSES=YES only for the full documentation.
 621         Local classes never appear outside the full documentation, no matter
 622         what is their visibility.
 623         """
 624         if not self.is_documented():
 625             return DocType.none
 626         if self._rawdoc.is_local():
 627             return DocType.internal
 628         return self._rawdoc.get_visibility()
 629
 630     def get_file_doc_type(self):
 631         return max([fileobj.get_doc_type() for fileobj in self._files])
 632
 633     def is_in_installed_file(self):
 634         return any([fileobj.is_installed() for fileobj in self._files])
 635
 636 class Member(object):
 637
 638     """Member (in Doxygen terminology) in the GROMACS source tree.
 639
 640     Currently, modeling is limited to the minimal set of properties that the
 641     checker uses.
 642     """
 643
 644     def __init__(self, rawdoc, namespace):
 645         self._rawdoc = rawdoc
 646         self._namespace = namespace
 647
 648     def get_name(self):
 649         return self._rawdoc.get_name()
 650
 651     def get_reporter_location(self):
 652         return self._rawdoc.get_reporter_location()
 653
 654     def is_documented(self):
 655         return self._rawdoc.is_documented()
 656
 657     def has_brief_description(self):
 658         return self._rawdoc.has_brief_description()
 659
 660     def has_inbody_description(self):
 661         return self._rawdoc.has_inbody_description()
 662
 663     def is_visible(self):
 664         """Return whether the member is visible in Doxygen documentation.
 665
 666         Doxygen ignores members whose parent compounds are not documented.
 667         However, when EXTRACT_ANON_NPACES=ON (which is set for our full
 668         documentation), members of anonymous namespaces are extracted even if
 669         the namespace is the only parent and is not documented.
 670         """
 671         if self._namespace and self._namespace.is_anonymous():
 672             return True
 673         return self._rawdoc.get_inherited_visibility() != DocType.none
 674
 675
 676 class GromacsTree(object):
 677
 678     """Root object for navigating the GROMACS source tree.
 679
 680     On initialization, the list of files and directories is initialized by
 681     walking the source tree, and modules are created for top-level
 682     subdirectories.  At this point, only information that is accessible from
 683     file names and paths only is available.
 684
 685     load_git_attributes() can be called to load attribute information from
 686     .gitattributes for all the files.
 687
 688     load_installed_file_list() can be called to load the list of installed
 689     files from the build tree (generated by CMake).
 690
 691     scan_files() can be called to read all the files and initialize #include
 692     dependencies between the files based on the information.  This is done like
 693     this instead of relying on Doxygen-extracted include files to make the
 694     dependency graph independent from preprocessor macro definitions
 695     (Doxygen only sees those #includes that the preprocessor sees, which
 696     depends on what #defines it has seen).
 697
 698     find_define_file_uses() can be called to find all uses of defines
 699     declared in config.h and some other macro headers. In the current
 700     implementation, scan_files() must have been called earlier.
 701
 702     load_xml() can be called to load information from Doxygen XML data in
 703     the build tree (the Doxygen XML data must have been built separately).
 704     """
 705
 706     def __init__(self, source_root, build_root, reporter):
 707         """Initialize the tree object by walking the source tree."""
 708         self._source_root = os.path.abspath(source_root)
 709         self._build_root = os.path.abspath(build_root)
 710         self._reporter = reporter
 711         self._docset = None
 712         self._docmap = dict()
 713         self._dirs = dict()
 714         self._files = dict()
 715         self._modules = dict()
 716         self._classes = set()
 717         self._namespaces = set()
 718         self._members = set()
 719         self._walk_dir(os.path.join(self._source_root, 'src'))
 720         for fileobj in self.get_files():
 721             if fileobj and fileobj.is_source_file() and not fileobj.is_external():
 722                 (basedir, name) = os.path.split(fileobj.get_abspath())
 723                 (basename, ext) = os.path.splitext(name)
 724                 header = self.get_file(os.path.join(basedir, basename + '.h'))
 725                 if not header and ext == '.cu':
 726                     header = self.get_file(os.path.join(basedir, basename + '.cuh'))
 727                 if not header and fileobj.is_test_file():
 728                     basedir = os.path.dirname(basedir)
 729                     header = self.get_file(os.path.join(basedir, basename + '.h'))
 730                     if not header:
 731                         # Somewhat of a hack; currently, the tests for
 732                         # analysisdata/modules/ and trajectoryanalysis/modules/
 733                         # is at the top-level tests directory.
 734                         # TODO: It could be clearer to split the tests so that
 735                         # there would be a separate modules/tests/.
 736                         header = self.get_file(os.path.join(basedir, 'modules', basename + '.h'))
 737                     if not header and basename.endswith('_tests'):
 738                         header = self.get_file(os.path.join(basedir, basename[:-6] + '.h'))
 739                 if not header and fileobj.get_relpath().startswith('src/gromacs'):
 740                     header = self._files.get(os.path.join('src/gromacs/legacyheaders', basename + '.h'))
 741                 if header:
 742                     fileobj.set_main_header(header)
 743         rootdir = self._get_dir(os.path.join('src', 'gromacs'))
 744         for subdir in rootdir.get_subdirectories():
 745             self._create_module(subdir)
 746         rootdir = self._get_dir(os.path.join('src', 'testutils'))
 747         self._create_module(rootdir)
 748
 749     def _get_rel_path(self, path):
 750         assert os.path.isabs(path)
 751         if path.startswith(self._build_root):
 752             return os.path.relpath(path, self._build_root)
 753         if path.startswith(self._source_root):
 754             return os.path.relpath(path, self._source_root)
 755         raise ValueError("path not under build nor source tree: {0}".format(path))
 756
 757     def _walk_dir(self, rootpath):
 758         """Construct representation of the source tree by walking the file system."""
 759         assert os.path.isabs(rootpath)
 760         assert rootpath not in self._dirs
 761         relpath = self._get_rel_path(rootpath)
 762         self._dirs[relpath] = Directory(rootpath, relpath, None)
 763         for dirpath, dirnames, filenames in os.walk(rootpath):
 764             if 'contrib' in dirnames:
 765                 dirnames.remove('contrib')
 766             if 'refdata' in dirnames:
 767                 dirnames.remove('refdata')
 768             currentdir = self._dirs[self._get_rel_path(dirpath)]
 769             # Loop through a copy so that we can modify dirnames.
 770             for dirname in list(dirnames):
 771                 fullpath = os.path.join(dirpath, dirname)
 772                 if fullpath == self._build_root:
 773                     dirnames.remove(dirname)
 774                     continue
 775                 relpath = self._get_rel_path(fullpath)
 776                 self._dirs[relpath] = Directory(fullpath, relpath, currentdir)
 777             extensions = ('.h', '.cuh', '.hpp', '.c', '.cc', '.cpp', '.cu', '.bm')
 778             for filename in filenames:
 779                 basename, extension = os.path.splitext(filename)
 780                 if extension in extensions:
 781                     fullpath = os.path.join(dirpath, filename)
 782                     relpath = self._get_rel_path(fullpath)
 783                     self._files[relpath] = File(fullpath, relpath, currentdir)
 784                 elif extension == '.cmakein':
 785                     extension = os.path.splitext(basename)[1]
 786                     if extension in extensions:
 787                         fullpath = os.path.join(dirpath, filename)
 788                         relpath = self._get_rel_path(fullpath)
 789                         sourcefile = GeneratorSourceFile(fullpath, relpath, currentdir)
 790                         self._files[relpath] = sourcefile
 791                         fullpath = os.path.join(dirpath, basename)
 792                         relpath = self._get_rel_path(fullpath)
 793                         fullpath = os.path.join(self._build_root, relpath)
 794                         generatedfile = GeneratedFile(fullpath, relpath, currentdir)
 795                         self._files[relpath] = generatedfile
 796                         generatedfile.set_generator_source(sourcefile)
 797                 elif extension in ('.l', '.y', '.pre'):
 798                     fullpath = os.path.join(dirpath, filename)
 799                     relpath = self._get_rel_path(fullpath)
 800                     self._files[relpath] = GeneratorSourceFile(fullpath, relpath, currentdir)
 801
 802     def _create_module(self, rootdir):
 803         """Create module for a subdirectory."""
 804         name = 'module_' + rootdir.get_name()
 805         moduleobj = Module(name, rootdir)
 806         rootdir.set_module(moduleobj)
 807         self._modules[name] = moduleobj
 808
 809     def scan_files(self, only_files=None, keep_contents=False):
 810         """Read source files to initialize #include dependencies."""
 811         if only_files:
 812             filelist = only_files
 813         else:
 814             filelist = self._files.itervalues()
 815         define_files = list(self.get_checked_define_files())
 816         for define_file in list(define_files):
 817             if isinstance(define_file, GeneratedFile) and \
 818                     define_file.get_generator_source() is not None:
 819                 define_files.append(define_file.get_generator_source())
 820         for fileobj in filelist:
 821             if not fileobj.is_external():
 822                 detect_defines = fileobj in define_files
 823                 fileobj.scan_contents(self, keep_contents, detect_defines)
 824                 module = fileobj.get_module()
 825                 if module:
 826                     for includedfile in fileobj.get_includes():
 827                         otherfile = includedfile.get_file()
 828                         if otherfile:
 829                             othermodule = otherfile.get_module()
 830                             if othermodule and othermodule != module:
 831                                 module.add_dependency(othermodule, includedfile)
 832
 833     def load_xml(self, only_files=None):
 834         """Load Doxygen XML information.
 835
 836         If only_files is True, XML data is not loaded for code constructs, but
 837         only for files, directories, and their potential parents.
 838         """
 839         xmldir = os.path.join(self._build_root, 'docs', 'html', 'doxygen', 'xml')
 840         self._docset = xml.DocumentationSet(xmldir, self._reporter)
 841         if only_files:
 842             if isinstance(only_files, collections.Iterable):
 843                 filelist = [x.get_abspath() for x in only_files]
 844                 self._docset.load_file_details(filelist)
 845             else:
 846                 self._docset.load_file_details()
 847         else:
 848             self._docset.load_details()
 849             self._docset.merge_duplicates()
 850         self._load_dirs()
 851         self._load_modules()
 852         self._load_files()
 853         if not only_files:
 854             self._load_namespaces()
 855             self._load_classes()
 856             self._load_members()
 857
 858     def _load_dirs(self):
 859         """Load Doxygen XML directory information."""
 860         rootdirs = self._docset.get_compounds(xml.Directory,
 861                 lambda x: x.get_parent() is None)
 862         for dirdoc in rootdirs:
 863             self._load_dir(dirdoc, None)
 864
 865     def _load_dir(self, dirdoc, parent):
 866         """Load Doxygen XML directory information for a single directory."""
 867         path = dirdoc.get_path().rstrip('/')
 868         if not os.path.isabs(path):
 869             self._reporter.xml_assert(dirdoc.get_xml_path(),
 870                     "expected absolute path in Doxygen-produced XML file")
 871             return
 872         relpath = self._get_rel_path(path)
 873         dirobj = self._dirs.get(relpath)
 874         if not dirobj:
 875             dirobj = Directory(path, relpath, parent)
 876             self._dirs[relpath] = dirobj
 877         dirobj.set_doc_xml(dirdoc, self)
 878         self._docmap[dirdoc] = dirobj
 879         for subdirdoc in dirdoc.get_subdirectories():
 880             self._load_dir(subdirdoc, dirobj)
 881
 882     def _load_modules(self):
 883         """Load Doxygen XML module (group) information."""
 884         moduledocs = self._docset.get_compounds(xml.Group,
 885                 lambda x: x.get_name().startswith('module_'))
 886         for moduledoc in moduledocs:
 887             moduleobj = self._modules.get(moduledoc.get_name())
 888             if not moduleobj:
 889                 self._reporter.input_error(
 890                         "no matching directory for module: {0}".format(moduledoc))
 891                 continue
 892             moduleobj.set_doc_xml(moduledoc, self)
 893             self._docmap[moduledoc] = moduleobj
 894
 895     def _load_files(self):
 896         """Load Doxygen XML file information."""
 897         for filedoc in self._docset.get_files():
 898             path = filedoc.get_path()
 899             if not path:
 900                 # In case of only partially loaded file information,
 901                 # the path information is not set for unloaded files.
 902                 continue
 903             if not os.path.isabs(path):
 904                 self._reporter.xml_assert(filedoc.get_xml_path(),
 905                         "expected absolute path in Doxygen-produced XML file")
 906                 continue
 907             extension = os.path.splitext(path)[1]
 908             # We don't care about Markdown files that only produce pages
 909             # (and fail the directory check below).
 910             if extension == '.md':
 911                 continue
 912             dirdoc = filedoc.get_directory()
 913             if not dirdoc:
 914                 self._reporter.xml_assert(filedoc.get_xml_path(),
 915                         "file is not in any directory in Doxygen")
 916                 continue
 917             relpath = self._get_rel_path(path)
 918             fileobj = self._files.get(relpath)
 919             if not fileobj:
 920                 fileobj = File(path, relpath, self._docmap[dirdoc])
 921                 self._files[relpath] = fileobj
 922             fileobj.set_doc_xml(filedoc, self)
 923             self._docmap[filedoc] = fileobj
 924
 925     def _load_namespaces(self):
 926         """Load Doxygen XML namespace information."""
 927         nsdocs = self._docset.get_namespaces()
 928         for nsdoc in nsdocs:
 929             nsobj = Namespace(nsdoc)
 930             self._docmap[nsdoc] = nsobj
 931             self._namespaces.add(nsobj)
 932
 933     def _load_classes(self):
 934         """Load Doxygen XML class information."""
 935         classdocs = self._docset.get_classes()
 936         for classdoc in classdocs:
 937             files = [self._docmap[filedoc] for filedoc in classdoc.get_files()]
 938             classobj = Class(classdoc, files)
 939             self._docmap[classdoc] = classobj
 940             self._classes.add(classobj)
 941
 942     def _load_members(self):
 943         """Load Doxygen XML member information."""
 944         memberdocs = self._docset.get_members()
 945         for memberdoc in memberdocs:
 946             nsdoc = memberdoc.get_namespace()
 947             nsobj = self.get_object(nsdoc)
 948             memberobj = Member(memberdoc, nsobj)
 949             self._docmap[memberdoc] = memberobj
 950             self._members.add(memberobj)
 951
 952     def _get_dir(self, relpath):
 953         """Get directory object for a path relative to source tree root."""
 954         return self._dirs.get(relpath)
 955
 956     def get_file(self, path):
 957         """Get file object for a path relative to source tree root."""
 958         return self._files.get(self._get_rel_path(path))
 959
 960     def find_include_file(self, includedpath):
 961         """Find a file object corresponding to an include path."""
 962         for testdir in ('src', 'src/external/thread_mpi/include',
 963                 'src/external/tng_io/include'):
 964             testpath = os.path.join(testdir, includedpath)
 965             if testpath in self._files:
 966                 return self._files[testpath]
 967
 968     def load_git_attributes(self):
 969         """Load git attribute information for files."""
 970         args = ['git', 'check-attr', '--stdin', 'filter']
 971         git_check_attr = subprocess.Popen(args, stdin=subprocess.PIPE,
 972                 stdout=subprocess.PIPE, cwd=self._source_root)
 973         filelist = '\n'.join(map(File.get_relpath, self._files.itervalues()))
 974         filters = git_check_attr.communicate(filelist)[0]
 975         for fileinfo in filters.splitlines():
 976             path, dummy, value = fileinfo.split(': ')
 977             fileobj = self._files.get(path)
 978             assert fileobj is not None
 979             fileobj.set_git_filter_attribute(value)
 980
 981     def find_define_file_uses(self):
 982         """Find files that use defines from config.h."""
 983         # Executing git grep is substantially faster than using the define_re
 984         # directly on the contents of the file in Python.
 985         for define_file in self.get_checked_define_files():
 986             excluded_files = set([define_file])
 987             excluded_files.update(define_file.get_included_files(recursive=True))
 988             all_defines = define_file.get_declared_defines()
 989             args = ['git', 'grep', '-zwIF']
 990             for define in all_defines:
 991                 args.extend(['-e', define])
 992             args.extend(['--', '*.cpp', '*.c', '*.cu', '*.h', '*.cuh'])
 993             define_re = r'\b(?:' + '|'.join(all_defines)+ r')\b'
 994             output = subprocess.check_output(args, cwd=self._source_root)
 995             for line in output.splitlines():
 996                 (filename, text) = line.split('\0')
 997                 fileobj = self._files.get(filename)
 998                 if fileobj is not None and fileobj not in excluded_files:
 999                     defines = re.findall(define_re, text)
1000                     fileobj.add_used_defines(define_file, defines)
1001
1002     def load_installed_file_list(self):
1003         """Load list of installed files from the build tree."""
1004         listpath = os.path.join(self._build_root, 'src', 'gromacs', 'installed-headers.txt')
1005         with open(listpath, 'r') as installedfp:
1006             for line in installedfp:
1007                 path = line.strip()
1008                 if not os.path.isabs(path):
1009                     self._reporter.input_error(
1010                             "installed file not specified with absolute path: {0}"
1011                             .format(path))
1012                     continue
1013                 relpath = self._get_rel_path(path)
1014                 if relpath not in self._files:
1015                     self._reporter.input_error(
1016                             "installed file not in source tree: {0}".format(path))
1017                     continue
1018                 self._files[relpath].set_installed()
1019
1020     def load_cycle_suppression_list(self, filename):
1021         """Load a list of edges to suppress in cycles.
1022
1023         These edges between modules, if present, will be marked in the
1024         corresponding ModuleDependency objects.
1025         """
1026         with open(filename, 'r') as fp:
1027             for line in fp:
1028                 line = line.strip()
1029                 if not line or line.startswith('#'):
1030                     continue
1031                 modulenames = ['module_' + x.strip() for x in line.split('->')]
1032                 if len(modulenames) != 2:
1033                     self._reporter.input_error(
1034                             "invalid cycle suppression line: {0}".format(line))
1035                     continue
1036                 firstmodule = self._modules.get(modulenames[0])
1037                 secondmodule = self._modules.get(modulenames[1])
1038                 if not firstmodule or not secondmodule:
1039                     self._reporter.input_error(
1040                             "unknown modules mentioned on cycle suppression line: {0}".format(line))
1041                     continue
1042                 for dep in firstmodule.get_dependencies():
1043                     if dep.get_other_module() == secondmodule:
1044                         dep.set_cycle_suppression()
1045                         break
1046                 else:
1047                     self._reporter.cyclic_issue("unused cycle suppression: {0}".format(line))
1048
1049     def report_unused_cycle_suppressions(self, reporter):
1050         """Reports unused cycle suppressions."""
1051         for module in self.get_modules():
1052             for dep in module.get_dependencies():
1053                 if not dep.suppression_used:
1054                     reporter.cyclic_issue("unused cycle suppression: {0} -> {1}".format(module.get_name()[7:], dep.get_other_module().get_name()[7:]))
1055
1056     def get_object(self, docobj):
1057         """Get tree object for a Doxygen XML object."""
1058         if docobj is None:
1059             return None
1060         return self._docmap.get(docobj)
1061
1062     def get_files(self):
1063         """Get iterable for all files in the source tree."""
1064         return self._files.itervalues()
1065
1066     def get_modules(self):
1067         """Get iterable for all modules in the source tree."""
1068         return self._modules.itervalues()
1069
1070     def get_classes(self):
1071         """Get iterable for all classes in the source tree."""
1072         return self._classes
1073
1074     def get_members(self):
1075         """Get iterable for all members (in Doxygen terms) in the source tree."""
1076         return self._members
1077
1078     def get_checked_define_files(self):
1079         """Get list of files that contain #define macros whose usage needs to
1080         be checked."""
1081         return (self._files['src/config.h'],
1082                 self._files['src/gromacs/simd/simd.h'],
1083                 self._files['src/gromacs/ewald/pme-simd.h'],
1084                 self._files['src/gromacs/mdlib/nbnxn_simd.h'])