doxygen/gmxtree.py

   1 #!/usr/bin/python
   2 #
   3 # This file is part of the GROMACS molecular simulation package.
   4 #
   5 # Copyright (c) 2014, by the GROMACS development team, led by
   6 # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
   7 # and including many others, as listed in the AUTHORS file in the
   8 # top-level source directory and at http://www.gromacs.org.
   9 #
  10 # GROMACS is free software; you can redistribute it and/or
  11 # modify it under the terms of the GNU Lesser General Public License
  12 # as published by the Free Software Foundation; either version 2.1
  13 # of the License, or (at your option) any later version.
  14 #
  15 # GROMACS is distributed in the hope that it will be useful,
  16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18 # Lesser General Public License for more details.
  19 #
  20 # You should have received a copy of the GNU Lesser General Public
  21 # License along with GROMACS; if not, see
  22 # http://www.gnu.org/licenses, or write to the Free Software Foundation,
  23 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  24 #
  25 # If you want to redistribute modifications to GROMACS, please
  26 # consider that scientific software is very special. Version
  27 # control is crucial - bugs must be traceable. We will be happy to
  28 # consider code for inclusion in the official distribution, but
  29 # derived work must not be called official GROMACS. Details are found
  30 # in the README & COPYING files - if they are missing, get the
  31 # official version at http://www.gromacs.org.
  32 #
  33 # To help us fund GROMACS development, we humbly ask that you cite
  34 # the research papers on the package. Check out http://www.gromacs.org.
  35
  36 """GROMACS-specific representation for source tree and documentation.
  37
  38 This module provides classes that construct a GROMACS-specific representation
  39 of the source tree and associate the Doxygen XML output with it.  It constructs
  40 an initial representation by walking the source tree in the file system, and
  41 then associates information from the Doxygen XML output into this.
  42 It also adds some additional knowledge from how the GROMACS source tree is
  43 organized to construct a representation that is easy to process and check as
  44 the top-level scripts expect.
  45
  46 The object model is rooted at a GromacsTree object.  Currently, it constructs a
  47 representation of the source tree from the file system, but is otherwise mostly
  48 a thin wrapper around the Doxygen XML tree.  It already adds some relations and
  49 rules that come from GROMACS-specific knowledge.  In the future, more such
  50 customizations will be added.
  51 """
  52
  53 import os
  54 import os.path
  55 import re
  56
  57 import doxygenxml as xml
  58 import reporter
  59 # We import DocType directly so that it is exposed from this module as well.
  60 from doxygenxml import DocType
  61
  62 def _get_api_type_for_compound(grouplist):
  63     """Helper function to deduce API type from Doxygen group membership."""
  64     result = DocType.internal
  65     for group in grouplist:
  66         if isinstance(group, xml.Group):
  67             if group.get_name() == 'group_publicapi':
  68                 result = DocType.public
  69             elif group.get_name() == 'group_libraryapi':
  70                 result = DocType.library
  71             # TODO: Check for multiple group membership
  72     return result
  73
  74 class IncludedFile(object):
  75
  76     """Information about an #include directive in a file."""
  77
  78     def __init__(self, abspath, lineno, included_file, included_path, is_relative, is_system):
  79         self._abspath = abspath
  80         self._line_number = lineno
  81         self._included_file = included_file
  82         self._included_path = included_path
  83         #self._used_include_path = used_include_path
  84         self._is_relative = is_relative
  85         self._is_system = is_system
  86
  87     def __str__(self):
  88         if self._is_system:
  89             return '<{0}>'.format(self._included_path)
  90         else:
  91             return '"{0}"'.format(self._included_path)
  92
  93     def is_system(self):
  94         return self._is_system
  95
  96     def is_relative(self):
  97         return self._is_relative
  98
  99     def get_file(self):
 100         return self._included_file
 101
 102     def get_reporter_location(self):
 103         return reporter.Location(self._abspath, self._line_number)
 104
 105 class File(object):
 106
 107     """Source/header file in the GROMACS tree."""
 108
 109     def __init__(self, abspath, relpath, directory):
 110         """Initialize a file representation with basic information."""
 111         self._abspath = abspath
 112         self._relpath = relpath
 113         self._dir = directory
 114         self._rawdoc = None
 115         self._installed = False
 116         extension = os.path.splitext(abspath)[1]
 117         self._sourcefile = (extension in ('.c', '.cc', '.cpp', '.cu'))
 118         self._apitype = DocType.none
 119         self._modules = set()
 120         self._includes = []
 121         directory.add_file(self)
 122
 123     def set_doc_xml(self, rawdoc, sourcetree):
 124         """Assiociate Doxygen documentation entity with the file."""
 125         assert self._rawdoc is None
 126         assert rawdoc.is_source_file() == self._sourcefile
 127         self._rawdoc = rawdoc
 128         if self._rawdoc.is_documented():
 129             grouplist = self._rawdoc.get_groups()
 130             self._apitype = _get_api_type_for_compound(grouplist)
 131             for group in grouplist:
 132                 module = sourcetree.get_object(group)
 133                 if module:
 134                     self._modules.add(module)
 135
 136     def set_installed(self):
 137         """Mark the file installed."""
 138         self._installed = True
 139
 140     def _process_include(self, lineno, is_system, includedpath, sourcetree):
 141         """Process #include directive during scan()."""
 142         is_relative = False
 143         if is_system:
 144             fileobj = sourcetree.find_include_file(includedpath)
 145         else:
 146             fullpath = os.path.join(self._dir.get_abspath(), includedpath)
 147             fullpath = os.path.abspath(fullpath)
 148             if os.path.exists(fullpath):
 149                 is_relative = True
 150                 fileobj = sourcetree.get_file(fullpath)
 151             else:
 152                 fileobj = sourcetree.find_include_file(includedpath)
 153         self._includes.append(IncludedFile(self.get_abspath(), lineno, fileobj, includedpath,
 154                 is_relative, is_system))
 155
 156     def scan_contents(self, sourcetree):
 157         """Scan the file contents and initialize information based on it."""
 158         # TODO: Consider a more robust regex.
 159         include_re = r'^#\s*include\s+(?P<quote>["<])(?P<path>[^">]*)[">]'
 160         with open(self._abspath, 'r') as scanfile:
 161             for lineno, line in enumerate(scanfile, 1):
 162                 match = re.match(include_re, line)
 163                 if match:
 164                     is_system = (match.group('quote') == '<')
 165                     includedpath = match.group('path')
 166                     self._process_include(lineno, is_system, includedpath,
 167                             sourcetree)
 168
 169     def get_reporter_location(self):
 170         return reporter.Location(self._abspath, None)
 171
 172     def is_installed(self):
 173         return self._installed
 174
 175     def is_external(self):
 176         return self._dir.is_external()
 177
 178     def is_source_file(self):
 179         return self._sourcefile
 180
 181     def is_test_file(self):
 182         return self._dir.is_test_directory()
 183
 184     def is_documented(self):
 185         return self._rawdoc and self._rawdoc.is_documented()
 186
 187     def has_brief_description(self):
 188         return self._rawdoc and self._rawdoc.has_brief_description()
 189
 190     def get_abspath(self):
 191         return self._abspath
 192
 193     def get_relpath(self):
 194         return self._relpath
 195
 196     def get_name(self):
 197         return os.path.basename(self._abspath)
 198
 199     def get_documentation_type(self):
 200         if not self._rawdoc:
 201             return DocType.none
 202         return self._rawdoc.get_visibility()
 203
 204     def get_api_type(self):
 205         return self._apitype
 206
 207     def get_expected_module(self):
 208         return self._dir.get_module()
 209
 210     def get_doc_modules(self):
 211         return self._modules
 212
 213     def get_module(self):
 214         module = self.get_expected_module()
 215         if not module and len(self._modules) == 1:
 216             module = list(self._modules)[0]
 217         return module
 218
 219     def get_includes(self):
 220         return self._includes
 221
 222 class GeneratedFile(File):
 223     pass
 224
 225 class Directory(object):
 226
 227     """(Sub)directory in the GROMACS tree."""
 228
 229     def __init__(self, abspath, relpath, parent):
 230         """Initialize a file representation with basic information."""
 231         self._abspath = abspath
 232         self._relpath = relpath
 233         self._name = os.path.basename(abspath)
 234         self._parent = parent
 235         self._rawdoc = None
 236         self._module = None
 237         self._is_test_dir = False
 238         if parent and parent.is_test_directory() or \
 239                 self._name in ('tests', 'legacytests'):
 240             self._is_test_dir = True
 241         self._is_external = False
 242         if parent and parent.is_external() or self._name == 'external':
 243             self._is_external = True
 244         self._subdirs = set()
 245         if parent:
 246             parent._subdirs.add(self)
 247         self._files = set()
 248         self._has_installed_files = None
 249
 250     def set_doc_xml(self, rawdoc, sourcetree):
 251         """Assiociate Doxygen documentation entity with the directory."""
 252         assert self._rawdoc is None
 253         assert self._abspath == rawdoc.get_path().rstrip('/')
 254         self._rawdoc = rawdoc
 255
 256     def set_module(self, module):
 257         assert self._module is None
 258         self._module = module
 259
 260     def add_file(self, fileobj):
 261         self._files.add(fileobj)
 262
 263     def get_name(self):
 264         return self._name
 265
 266     def get_reporter_location(self):
 267         return reporter.Location(self._abspath, None)
 268
 269     def get_abspath(self):
 270         return self._abspath
 271
 272     def get_relpath(self):
 273         return self._relpath
 274
 275     def is_test_directory(self):
 276         return self._is_test_dir
 277
 278     def is_external(self):
 279         return self._is_external
 280
 281     def has_installed_files(self):
 282         if self._has_installed_files is None:
 283             self._has_installed_files = False
 284             for subdir in self._subdirs:
 285                 if subdir.has_installed_files():
 286                     self._has_installed_files = True
 287                     return True
 288             for fileobj in self._files:
 289                 if fileobj.is_installed():
 290                     self._has_installed_files = True
 291                     return True
 292         return self._has_installed_files
 293
 294     def get_module(self):
 295         if self._module:
 296             return self._module
 297         if self._parent:
 298             return self._parent.get_module()
 299         return None
 300
 301     def get_subdirectories(self):
 302         return self._subdirs
 303
 304     def get_files(self):
 305         for subdir in self._subdirs:
 306             for fileobj in subdir.get_files():
 307                 yield fileobj
 308         for fileobj in self._files:
 309             yield fileobj
 310
 311 class Module(object):
 312
 313     """Code module in the GROMACS source tree.
 314
 315     Modules are specific subdirectories that host a more or less coherent
 316     set of routines.  Simplified, every subdirectory under src/gromacs/ is
 317     a different module.  This object provides that abstraction and also links
 318     the subdirectory to the module documentation (documented as a group in
 319     Doxygen) if that exists.
 320     """
 321
 322     def __init__(self, name, rootdir):
 323         self._name = name
 324         self._rawdoc = None
 325         self._rootdir = rootdir
 326         self._group = None
 327
 328     def set_doc_xml(self, rawdoc, sourcetree):
 329         """Assiociate Doxygen documentation entity with the module."""
 330         assert self._rawdoc is None
 331         self._rawdoc = rawdoc
 332         if self._rawdoc.is_documented():
 333             groups = list(self._rawdoc.get_groups())
 334             if len(groups) == 1:
 335                 groupname = groups[0].get_name()
 336                 if groupname.startswith('group_'):
 337                     self._group = groupname[6:]
 338
 339     def is_documented(self):
 340         return self._rawdoc is not None
 341
 342     def get_name(self):
 343         return self._name
 344
 345     def get_root_dir(self):
 346         return self._rootdir
 347
 348     def get_files(self):
 349         # TODO: Include public API convenience headers?
 350         return self._rootdir.get_files()
 351
 352     def get_group(self):
 353         return self._group
 354
 355
 356 class Class(object):
 357
 358     """Class/struct/union in the GROMACS source code."""
 359
 360     def __init__(self, rawdoc, files):
 361         self._rawdoc = rawdoc
 362         self._files = set(files)
 363
 364     def get_name(self):
 365         return self._rawdoc.get_name()
 366
 367     def get_reporter_location(self):
 368         return self._rawdoc.get_reporter_location()
 369
 370     def get_files(self):
 371         return self._files
 372
 373     def is_documented(self):
 374         return self._rawdoc.is_documented()
 375
 376     def has_brief_description(self):
 377         return self._rawdoc.has_brief_description()
 378
 379     def get_documentation_type(self):
 380         if not self.is_documented():
 381             return DocType.none
 382         if self._rawdoc.is_local():
 383             return DocType.internal
 384         return self._rawdoc.get_visibility()
 385
 386     def get_file_documentation_type(self):
 387         return max([fileobj.get_documentation_type() for fileobj in self._files])
 388
 389     def is_in_installed_file(self):
 390         return any([fileobj.is_installed() for fileobj in self._files])
 391
 392 class GromacsTree(object):
 393
 394     """Root object for navigating the GROMACS source tree.
 395
 396     On initialization, the list of files and directories is initialized by
 397     walking the source tree, and modules are created for top-level
 398     subdirectories.  At this point, only information that is accessible from
 399     file names and paths only is available.
 400
 401     set_installed_file_list() can be called to set the list of installed
 402     files.
 403
 404     scan_files() can be called to read all the files and initialize #include
 405     dependencies between the files based on the information.  This is done like
 406     this instead of relying on Doxygen-extracted include files to make the
 407     dependency graph independent from preprocessor macro definitions
 408     (Doxygen only sees those #includes that the preprocessor sees, which
 409     depends on what #defines it has seen).
 410
 411     load_xml() can be called to load information from Doxygen XML data in
 412     the build tree (the Doxygen XML data must have been built separately).
 413     """
 414
 415     def __init__(self, source_root, build_root, reporter):
 416         """Initialize the tree object by walking the source tree."""
 417         self._source_root = os.path.abspath(source_root)
 418         self._build_root = os.path.abspath(build_root)
 419         self._reporter = reporter
 420         self._docset = None
 421         self._docmap = dict()
 422         self._dirs = dict()
 423         self._files = dict()
 424         self._modules = dict()
 425         self._classes = set()
 426         self._walk_dir(os.path.join(self._source_root, 'src'))
 427         rootdir = self._get_dir(os.path.join('src', 'gromacs'))
 428         for subdir in rootdir.get_subdirectories():
 429             self._create_module(subdir)
 430         rootdir = self._get_dir(os.path.join('src', 'testutils'))
 431         self._create_module(rootdir)
 432
 433     def _get_rel_path(self, path):
 434         assert os.path.isabs(path)
 435         if path.startswith(self._build_root):
 436             return os.path.relpath(path, self._build_root)
 437         if path.startswith(self._source_root):
 438             return os.path.relpath(path, self._source_root)
 439         raise ValueError("path not under build nor source tree: {0}".format(path))
 440
 441     def _walk_dir(self, rootpath):
 442         """Construct representation of the source tree by walking the file system."""
 443         assert os.path.isabs(rootpath)
 444         assert rootpath not in self._dirs
 445         relpath = self._get_rel_path(rootpath)
 446         self._dirs[relpath] = Directory(rootpath, relpath, None)
 447         for dirpath, dirnames, filenames in os.walk(rootpath):
 448             if 'contrib' in dirnames:
 449                 dirnames.remove('contrib')
 450             if 'refdata' in dirnames:
 451                 dirnames.remove('refdata')
 452             currentdir = self._dirs[self._get_rel_path(dirpath)]
 453             # Loop through a copy so that we can modify dirnames.
 454             for dirname in list(dirnames):
 455                 fullpath = os.path.join(dirpath, dirname)
 456                 if fullpath == self._build_root:
 457                     dirnames.remove(dirname)
 458                     continue
 459                 relpath = self._get_rel_path(fullpath)
 460                 self._dirs[relpath] = Directory(fullpath, relpath, currentdir)
 461             extensions = ('.h', '.cuh', '.hpp', '.c', '.cc', '.cpp', '.cu', '.bm')
 462             for filename in filenames:
 463                 basename, extension = os.path.splitext(filename)
 464                 if extension in extensions:
 465                     fullpath = os.path.join(dirpath, filename)
 466                     relpath = self._get_rel_path(fullpath)
 467                     self._files[relpath] = File(fullpath, relpath, currentdir)
 468                 elif extension == '.cmakein':
 469                     extension = os.path.splitext(basename)[1]
 470                     if extension in extensions:
 471                         fullpath = os.path.join(dirpath, basename)
 472                         relpath = self._get_rel_path(fullpath)
 473                         fullpath = os.path.join(dirpath, filename)
 474                         self._files[relpath] = GeneratedFile(fullpath, relpath, currentdir)
 475
 476     def _create_module(self, rootdir):
 477         """Create module for a subdirectory."""
 478         name = 'module_' + rootdir.get_name()
 479         moduleobj = Module(name, rootdir)
 480         rootdir.set_module(moduleobj)
 481         self._modules[name] = moduleobj
 482
 483     def scan_files(self):
 484         """Read source files to initialize #include dependencies."""
 485         for fileobj in self._files.itervalues():
 486             if not fileobj.is_external():
 487                 fileobj.scan_contents(self)
 488
 489     def load_xml(self, only_files=False):
 490         """Load Doxygen XML information.
 491
 492         If only_files is True, XML data is not loaded for code constructs, but
 493         only for files, directories, and their potential parents.
 494         """
 495         xmldir = os.path.join(self._build_root, 'doxygen', 'xml')
 496         self._docset = xml.DocumentationSet(xmldir, self._reporter)
 497         if only_files:
 498             self._docset.load_file_details()
 499         else:
 500             self._docset.load_details()
 501             self._docset.merge_duplicates()
 502         self._load_dirs()
 503         self._load_modules()
 504         self._load_files()
 505         if not only_files:
 506             self._load_classes()
 507
 508     def _load_dirs(self):
 509         """Load Doxygen XML directory information."""
 510         rootdirs = self._docset.get_compounds(xml.Directory,
 511                 lambda x: x.get_parent() is None)
 512         for dirdoc in rootdirs:
 513             self._load_dir(dirdoc, None)
 514
 515     def _load_dir(self, dirdoc, parent):
 516         """Load Doxygen XML directory information for a single directory."""
 517         path = dirdoc.get_path().rstrip('/')
 518         if not os.path.isabs(path):
 519             self._reporter.xml_assert(dirdoc.get_xml_path(),
 520                     "expected absolute path in Doxygen-produced XML file")
 521             return
 522         relpath = self._get_rel_path(path)
 523         dirobj = self._dirs.get(relpath)
 524         if not dirobj:
 525             dirobj = Directory(path, relpath, parent)
 526             self._dirs[relpath] = dirobj
 527         dirobj.set_doc_xml(dirdoc, self)
 528         self._docmap[dirdoc] = dirobj
 529         for subdirdoc in dirdoc.get_subdirectories():
 530             self._load_dir(subdirdoc, dirobj)
 531
 532     def _load_modules(self):
 533         """Load Doxygen XML module (group) information."""
 534         moduledocs = self._docset.get_compounds(xml.Group,
 535                 lambda x: x.get_name().startswith('module_'))
 536         for moduledoc in moduledocs:
 537             moduleobj = self._modules.get(moduledoc.get_name())
 538             if not moduleobj:
 539                 self._reporter.input_error(
 540                         "no matching directory for module: {0}".format(moduledoc))
 541                 continue
 542             moduleobj.set_doc_xml(moduledoc, self)
 543             self._docmap[moduledoc] = moduleobj
 544
 545     def _load_files(self):
 546         """Load Doxygen XML file information."""
 547         for filedoc in self._docset.get_files():
 548             path = filedoc.get_path()
 549             if not os.path.isabs(path):
 550                 self._reporter.xml_assert(filedoc.get_xml_path(),
 551                         "expected absolute path in Doxygen-produced XML file")
 552                 continue
 553             extension = os.path.splitext(filedoc.get_path())[1]
 554             # We don't care about Markdown files that only produce pages
 555             # (and fail the directory check below).
 556             if extension == '.md':
 557                 continue
 558             dirdoc = filedoc.get_directory()
 559             if not dirdoc:
 560                 self._reporter.xml_assert(filedoc.get_xml_path(),
 561                         "file is not in any directory in Doxygen")
 562                 continue
 563             relpath = self._get_rel_path(path)
 564             fileobj = self._files.get(relpath)
 565             if not fileobj:
 566                 fileobj = File(path, relpath, self._docmap[dirdoc])
 567                 self._files[relpath] = fileobj
 568             fileobj.set_doc_xml(filedoc, self)
 569             self._docmap[filedoc] = fileobj
 570
 571     def _load_classes(self):
 572         """Load Doxygen XML class information."""
 573         classdocs = self._docset.get_classes()
 574         for classdoc in classdocs:
 575             files = [self._docmap[filedoc] for filedoc in classdoc.get_files()]
 576             classobj = Class(classdoc, files)
 577             self._docmap[classdoc] = classobj
 578             self._classes.add(classobj)
 579
 580     def _get_dir(self, relpath):
 581         """Get directory object for a path relative to source tree root."""
 582         return self._dirs.get(relpath)
 583
 584     def get_file(self, path):
 585         """Get file object for a path relative to source tree root."""
 586         return self._files.get(self._get_rel_path(path))
 587
 588     def find_include_file(self, includedpath):
 589         """Find a file object corresponding to an include path."""
 590         for testdir in ('src', 'src/gromacs/legacyheaders', 'src/external/thread_mpi/include'):
 591             testpath = os.path.join(testdir, includedpath)
 592             if testpath in self._files:
 593                 return self._files[testpath]
 594
 595     def set_installed_file_list(self, installedfiles):
 596         """Set list of installed files."""
 597         for path in installedfiles:
 598             if not os.path.isabs(path):
 599                 self._reporter.input_error(
 600                         "installed file not specified with absolute path: {0}"
 601                         .format(path))
 602                 continue
 603             relpath = self._get_rel_path(path)
 604             if relpath not in self._files:
 605                 self._reporter.input_error(
 606                         "installed file not in source tree: {0}".format(path))
 607                 continue
 608             self._files[relpath].set_installed()
 609
 610     def get_object(self, docobj):
 611         """Get tree object for a Doxygen XML object."""
 612         return self._docmap.get(docobj)
 613
 614     def get_files(self):
 615         """Get iterable for all files in the source tree."""
 616         return self._files.itervalues()
 617
 618     def get_modules(self):
 619         """Get iterable for all modules in the source tree."""
 620         return self._modules.itervalues()
 621
 622     def get_classes(self):
 623         """Get iterable for all classes in the source tree."""
 624         return self._classes
 625
 626     def get_members(self):
 627         """Get iterable for all members (in Doxygen terms) in the source tree."""
 628         # TODO: Add wrappers to solve some issues.
 629         return self._docset.get_members()