Test gcc-6 and cuda-9.0
[gromacs.git] / docs / doxygen / graphbuilder.py
blob3fef8a81c111b2b9901d9f8117892aff98e5b354
1 #!/usr/bin/env python2
3 # This file is part of the GROMACS molecular simulation package.
5 # Copyright (c) 2012,2013,2014,2015,2018, by the GROMACS development team, led by
6 # Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
7 # and including many others, as listed in the AUTHORS file in the
8 # top-level source directory and at http://www.gromacs.org.
10 # GROMACS is free software; you can redistribute it and/or
11 # modify it under the terms of the GNU Lesser General Public License
12 # as published by the Free Software Foundation; either version 2.1
13 # of the License, or (at your option) any later version.
15 # GROMACS is distributed in the hope that it will be useful,
16 # but WITHOUT ANY WARRANTY; without even the implied warranty of
17 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 # Lesser General Public License for more details.
20 # You should have received a copy of the GNU Lesser General Public
21 # License along with GROMACS; if not, see
22 # http://www.gnu.org/licenses, or write to the Free Software Foundation,
23 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
25 # If you want to redistribute modifications to GROMACS, please
26 # consider that scientific software is very special. Version
27 # control is crucial - bugs must be traceable. We will be happy to
28 # consider code for inclusion in the official distribution, but
29 # derived work must not be called official GROMACS. Details are found
30 # in the README & COPYING files - if they are missing, get the
31 # official version at http://www.gromacs.org.
33 # To help us fund GROMACS development, we humbly ask that you cite
34 # the research papers on the package. Check out http://www.gromacs.org.
36 """Generate include dependency graphs.
38 This script generates include dependency graphs from the GROMACS source tree.
39 One graph is generated to show inter-module dependencies, and separate graphs
40 for each module to show file-level dependencies within the module.
42 Output format for the graphs is suitable for processing with 'dot' in graphviz.
44 The graphs are built from the source tree representation constructed in
45 gmxtree.py.
47 Classes Graph, Node, Edge, and EdgeType provide a relatively general
48 implementation for constructing 'dot' graphs. GraphBuilder is used to
49 create Graph instances from a gmxtree.GromacsTree object; the actual graph
50 objects will not contain any references to the gmxtree objects.
52 When run in script mode, the GromacsTree object is first constructed, and then
53 GraphBuilder is used to construct the necessary graphs, which are then written
54 out.
56 The produced graphs are documented in doxygen.md.
57 """
59 import os.path
60 import re
62 from gmxtree import DocType
64 class EdgeType(object):
66 """Enumeration type for edge types in include dependency graphs."""
68 # Mapping to string representation for the internal integer values
69 _names = ['test', 'pubimpl', 'libimpl', 'library', 'public',
70 'intramodule', 'legacy', 'undocumented']
72 def __init__(self, value):
73 """Initialize a EdgeType instance.
75 EdgeType.{test,pubimpl,...,undocumented} should be used outside the
76 class instead of calling the constructor.
77 """
78 self._value = value
80 def __str__(self):
81 """Return string representation for the edge type (for debugging)."""
82 return self._names[self._value]
84 def __cmp__(self, other):
85 """Order edge types in the order of increasing coupling."""
86 return cmp(self._value, other._value)
88 # Tests depend on test
89 EdgeType.test = EdgeType(0)
90 # Implementation depends on public/library headers
91 EdgeType.pubimpl = EdgeType(1)
92 EdgeType.libimpl = EdgeType(2)
93 # Library header depends on other module
94 EdgeType.library = EdgeType(3)
95 # Public header depends on other module
96 EdgeType.public = EdgeType(4)
97 # Intramodule dependency
98 EdgeType.intramodule = EdgeType(5)
99 EdgeType.legacy = EdgeType(6)
100 EdgeType.cyclic = EdgeType(7)
101 # Invalid dependency
102 EdgeType.undocumented = EdgeType(8)
104 class Edge(object):
106 """Graph edge between two Node objects in 'dot' graph.
108 Signifies an include dependency between the two nodes, and manages types
109 associated with the dependencies.
112 def __init__(self, fromnode, tonode, edgetype):
113 """Create edge between given Nodes with given type."""
114 self._fromnode = fromnode
115 self._tonode = tonode
116 self._edgetype = edgetype
118 def merge_edge(self, other):
119 """Merge another edge into this one and choose an appropriate type.
121 Updates the type of this edge based on the types of the merged edges.
123 self._edgetype = max(self._edgetype, other._edgetype)
125 def format(self):
126 """Format this edge for 'dot'."""
127 # If you change these styles, update also the legend in modulegraph.md
128 if self._fromnode.is_file_node() and self._tonode.is_file_node():
129 properties = ''
130 elif self._edgetype == EdgeType.intramodule:
131 properties = ''
132 elif self._edgetype == EdgeType.test:
133 # TODO: Consider if only some test edges should be made non-constraints
134 properties = 'color=".33 .8 .8", style=dashed, constraint=no'
135 elif self._edgetype == EdgeType.libimpl:
136 properties = 'color=".66 .8 .8", style=dashed'
137 elif self._edgetype == EdgeType.pubimpl:
138 properties = 'color=black, style=dashed'
139 elif self._edgetype == EdgeType.library:
140 properties = 'color=".66 .8 .8"'
141 elif self._edgetype == EdgeType.public:
142 properties = 'color=black'
143 elif self._edgetype == EdgeType.legacy:
144 properties = 'color=grey75'
145 elif self._edgetype == EdgeType.cyclic:
146 properties = 'color=red, constraint=no'
147 else: # undocumented
148 properties = 'color=red'
149 return '{0} -> {1} [{2}]'.format(self._fromnode.get_nodename(),
150 self._tonode.get_nodename(),
151 properties)
153 class Node(object):
155 """Node in 'dot' graph."""
157 def __init__(self, nodename, label, style=None, properties=None, is_file=False):
158 """Create node with given attributes.
160 is_file does not affect the appearance of the node, but is used for
161 formatting edges between two files differently from other edges.
162 style and properties should be iterables with graphviz attributes for
163 the node.
165 Node can have child nodes. Such nodes are rendered as cluster
166 subgraphs for 'dot'.
168 self._nodename = nodename
169 self._label = label
170 if style:
171 self._style = ','.join(style)
172 else:
173 self._style = None
174 if properties:
175 self._properties = ', '.join(properties)
176 else:
177 self._properties = None
178 self._is_file = is_file
179 self._children = []
181 def add_child(self, child):
182 """Add a child node."""
183 self._children.append(child)
185 def clear_children(self):
186 """Remove all children from the node."""
187 self._children = []
189 def is_file_node(self):
190 """Return True if the node was created with is_file=True."""
191 return self._is_file
193 def get_nodename(self):
194 """Get internal name of the node in 'dot'."""
195 return self._nodename
197 def get_children(self, recursive=False):
198 """Get list of child nodes."""
199 if recursive:
200 result = list(self._children)
201 for child in self._children:
202 result.extend(child.get_children(recursive=True))
203 return result
204 else:
205 return self._children
207 def format(self):
208 """Format this node for 'dot'."""
209 # TODO: Take indent as a parameter to make output marginally nicer.
210 result = ''
211 if self._children:
212 result += ' subgraph cluster_{0} {{\n' \
213 .format(self._nodename)
214 result += ' label = "{0}"\n'.format(self._label)
215 for child in self._children:
216 result += child.format()
217 result += ' }\n'
218 else:
219 properties = 'label="{0}"'.format(self._label)
220 if self._properties:
221 properties += ', ' + self._properties
222 if self._style:
223 properties += ', style="{0}"'.format(self._style)
224 result += ' {0} [{1}]\n'.format(self._nodename, properties)
225 return result
228 class Graph(object):
230 """Graph for 'dot'."""
232 def __init__(self, nodes, edges):
233 """Create graph with given nodes and edges."""
234 self._nodes = set(nodes)
235 self._edges = edges
236 self._left_to_right = False
237 self._concentrate = True
239 def set_options(self, left_to_right=None, concentrate=None):
240 """Set output options for the graph."""
241 if left_to_right != None:
242 self._left_to_right = left_to_right
243 if concentrate != None:
244 self._concentrate = concentrate
246 def merge_nodes(self, nodes, target):
247 """Merge a set of nodes into a single node.
249 All nodes from the list nodes are merged into the target node.
250 All edges to or from the merged nodes are rerouted to/from target
251 instead. Duplicate edges are not created. Instead, if an edge already
252 exists, the edge types are merged. All nodes from the list nodes are
253 removed from the graph after the merge is done.
255 nodes = set(nodes)
256 nodes.add(target)
257 newedges = []
258 edgesto = dict()
259 edgesfrom = dict()
260 for edge in self._edges:
261 isfrom = (edge._fromnode in nodes)
262 isto = (edge._tonode in nodes)
263 if isfrom and isto:
264 pass
265 elif isfrom:
266 if not edge._tonode in edgesfrom:
267 edgesfrom[edge._tonode] = \
268 Edge(target, edge._tonode, edge._edgetype)
269 else:
270 edgesfrom[edge._tonode].merge_edge(edge)
271 elif isto:
272 if not edge._fromnode in edgesto:
273 edgesto[edge._fromnode] = \
274 Edge(edge._fromnode, target, edge._edgetype)
275 else:
276 edgesto[edge._fromnode].merge_edge(edge)
277 else:
278 newedges.append(edge)
279 newedges.extend(edgesfrom.values())
280 newedges.extend(edgesto.values())
281 self._edges = newedges
283 def collapse_node(self, node):
284 """Merge all children of a node into the node.
286 All child nodes are removed after the merge is done.
288 nodes = node.get_children(recursive=True)
289 self.merge_nodes(nodes, node)
290 node.clear_children()
292 def write(self, outfile):
293 """Write the graph in 'dot' format."""
294 outfile.write('digraph includedeps {\n')
295 if self._left_to_right:
296 outfile.write(' rankdir = LR\n')
297 if self._concentrate:
298 outfile.write(' concentrate = true\n')
299 outfile.write(' node [fontname="FreeSans",fontsize=10,height=.2,'
300 'shape=box]\n')
301 for node in self._nodes:
302 outfile.write(node.format())
303 for edge in self._edges:
304 outfile.write(' ' + edge.format() + '\n')
305 outfile.write('}\n')
307 class GraphBuilder(object):
309 """Builder for Graph objects from gmxtree.GromacsTree representation."""
311 def __init__(self, tree):
312 """Initialize builder for a given tree representation."""
313 self._tree = tree
315 def _create_file_node(self, fileobj, filenodes):
316 """Create graph node for a file object.
318 filenodes is a dict() that maps file objects to their nodes, and is
319 updated by this call.
321 nodename = re.subn(r'[-./]', '_', fileobj.get_relpath())[0]
322 style = []
323 properties = []
324 properties.append('URL="\\ref {0}"'.format(fileobj.get_name()))
325 if not fileobj.get_module():
326 style.append('bold')
327 properties.append('color=red')
328 if fileobj.is_test_file():
329 style.append('filled')
330 properties.append('fillcolor=".33 .2 1"')
331 elif fileobj.is_source_file():
332 style.append('filled')
333 properties.append('fillcolor=grey75')
334 elif fileobj.get_api_type() == DocType.public:
335 style.append('filled')
336 properties.append('fillcolor=".66 .2 1"')
337 elif fileobj.get_api_type() == DocType.library:
338 style.append('filled')
339 properties.append('fillcolor=".66 .5 1"')
340 node = Node(nodename, fileobj.get_name(), style, properties, is_file=True)
341 filenodes[fileobj] = node
342 return node
344 def _get_file_edge_type(self, fromfile, tofile):
345 """Get EdgeType for an edge between two file objects.
347 Determines the type for the edge from the information provided by
348 gmxtree.
350 intramodule = (fromfile.get_module() == tofile.get_module())
351 is_legacy = not tofile.api_type_is_reliable()
352 if fromfile.get_module() == tofile.get_module():
353 return EdgeType.intramodule
354 elif tofile.get_api_type() == DocType.internal and not tofile.is_public():
355 if is_legacy:
356 return EdgeType.legacy
357 else:
358 return EdgeType.undocumented
359 elif fromfile.is_test_file():
360 return EdgeType.test
361 elif tofile.is_test_file():
362 return EdgeType.undocumented
363 elif fromfile.is_module_internal():
364 if tofile.is_public():
365 return EdgeType.pubimpl
366 elif tofile.get_api_type() == DocType.library:
367 return EdgeType.libimpl
368 elif is_legacy:
369 return EdgeType.legacy
370 elif not tofile.is_documented():
371 return EdgeType.undocumented
372 else:
373 raise ValueError('Unknown edge type between {0} and {1}'
374 .format(fromfile.get_relpath(), tofile.get_relpath()))
375 elif fromfile.get_api_type() == DocType.library:
376 return EdgeType.library
377 elif fromfile.is_public() or fromfile.is_installed():
378 if tofile.is_public() or tofile.is_installed():
379 return EdgeType.public
380 else:
381 return EdgeType.undocumented
382 elif is_legacy:
383 return EdgeType.legacy
384 else:
385 raise ValueError('Unknown edge type between {0} and {1}'
386 .format(fromfile.get_relpath(), tofile.get_relpath()))
388 def _create_file_edge(self, fromfile, tofile, filenodes):
389 """Create edge between two file objects.
391 Determines the type for the edge from the information provided by
392 gmxtree.
394 edgetype = self._get_file_edge_type(fromfile, tofile)
395 return Edge(filenodes[fromfile], filenodes[tofile], edgetype)
397 def _create_file_edges(self, filenodes):
398 """Create edges between all file nodes.
400 Create edges between file nodes specified in filenodes from all include
401 dependencies. An edge is created only if both ends of the dependency
402 are in the list of nodes.
404 edges = []
405 for fileobj in filenodes.iterkeys():
406 for includedfile in fileobj.get_includes():
407 otherfile = includedfile.get_file()
408 if otherfile and otherfile in filenodes:
409 edge = self._create_file_edge(fileobj, otherfile, filenodes)
410 edges.append(edge)
411 return edges
413 def _get_module_color(self, modulegroup):
414 # If you change these styles, update also the legend in modulegraph.md
415 if modulegroup == 'legacy':
416 return 'fillcolor=grey75'
417 elif modulegroup == 'analysismodules':
418 return 'fillcolor="0 .2 1"'
419 elif modulegroup == 'utilitymodules':
420 return 'fillcolor=".08 .2 1"'
421 elif modulegroup == 'mdrun':
422 return 'fillcolor=".75 .2 1"'
423 return None
425 def _create_module_node(self, module):
426 """Create node for a module."""
427 style = []
428 properties = []
429 properties.append('shape=ellipse')
430 if module.is_documented():
431 properties.append('URL="\\ref {0}"'.format(module.get_name()))
432 if not module.is_documented():
433 fillcolor = self._get_module_color('legacy')
434 else:
435 fillcolor = self._get_module_color(module.get_group())
436 if fillcolor:
437 style.append('filled')
438 properties.append(fillcolor)
439 rootdir = module.get_root_dir()
440 if rootdir.has_installed_files():
441 properties.append('color=".66 .5 1"')
442 properties.append('penwidth=3')
443 nodename = 'module_' + re.subn(r'[-./]', '_', rootdir.get_relpath())[0]
444 label = module.get_name()[7:]
445 node = Node(nodename, label, style, properties)
446 return node
448 def _create_module_edges(self, modulenodes):
449 """Create edges between all module nodes.
451 Create edges between module nodes specified in modulenodes from all
452 include dependencies. An edge is created only if both ends of the
453 dependency are in the list of nodes.
455 edges = []
456 for moduleobj in modulenodes.iterkeys():
457 for dep in moduleobj.get_dependencies():
458 othermodule = dep.get_other_module()
459 if othermodule and othermodule in modulenodes:
460 if dep.is_cycle_suppressed():
461 edgetype = EdgeType.cyclic
462 else:
463 edgetype = max([
464 self._get_file_edge_type(x.get_including_file(), x.get_file())
465 for x in dep.get_included_files()])
466 edge = Edge(modulenodes[moduleobj], modulenodes[othermodule], edgetype)
467 edges.append(edge)
468 return edges
470 def create_modules_graph(self):
471 """Create module dependency graph."""
472 nodes = []
473 modulenodes = dict()
474 libgromacsnode = Node('libgromacs', 'libgromacs')
475 nodes.append(libgromacsnode)
476 for moduleobj in self._tree.get_modules():
477 node = self._create_module_node(moduleobj)
478 if moduleobj.get_root_dir().get_relpath().startswith('src/gromacs'):
479 libgromacsnode.add_child(node)
480 else:
481 nodes.append(node)
482 modulenodes[moduleobj] = node
483 edges = self._create_module_edges(modulenodes)
484 graph = Graph(nodes, edges)
485 graph.set_options(concentrate=False)
486 return graph
488 def create_module_file_graph(self, module):
489 """Create file dependency graph for files within a module."""
490 filenodes = dict()
491 nodes = []
492 for fileobj in module.get_files():
493 nodes.append(self._create_file_node(fileobj, filenodes))
494 edges = self._create_file_edges(filenodes)
495 graph = Graph(nodes, edges)
496 graph.set_options(left_to_right=True)
497 return graph
499 def main():
500 """Run the graph generation script."""
501 import os
502 import sys
504 from optparse import OptionParser
506 from gmxtree import GromacsTree
507 from reporter import Reporter
509 parser = OptionParser()
510 parser.add_option('-S', '--source-root',
511 help='Source tree root directory')
512 parser.add_option('-B', '--build-root',
513 help='Build tree root directory')
514 parser.add_option('--ignore-cycles',
515 help='Set file with module dependencies to ignore in cycles')
516 parser.add_option('-o', '--outdir', default='.',
517 help='Specify output directory for graphs')
518 parser.add_option('-q', '--quiet', action='store_true',
519 help='Do not write status messages')
520 options, args = parser.parse_args()
522 reporter = Reporter(quiet=True)
524 if not options.quiet:
525 sys.stderr.write('Scanning source tree...\n')
526 tree = GromacsTree(options.source_root, options.build_root, reporter)
527 tree.load_installed_file_list()
528 if not options.quiet:
529 sys.stderr.write('Reading source files...\n')
530 tree.scan_files()
531 if options.ignore_cycles:
532 tree.load_cycle_suppression_list(options.ignore_cycles)
533 if not options.quiet:
534 sys.stderr.write('Reading Doxygen XML files...\n')
535 tree.load_xml(only_files=True)
537 if not options.quiet:
538 sys.stderr.write('Writing graphs...\n')
539 graphbuilder = GraphBuilder(tree)
540 if not os.path.exists(options.outdir):
541 os.mkdir(options.outdir)
543 filename = os.path.join(options.outdir, 'module-deps.dot')
544 graph = graphbuilder.create_modules_graph()
545 with open(filename, 'w') as outfile:
546 graph.write(outfile)
548 # Skip some modules that are too big to make any sense
549 skippedmodules = ('gmxlib', 'mdlib', 'gmxana', 'gmxpreprocess')
550 for module in tree.get_modules():
551 if not module.get_name()[7:] in skippedmodules:
552 filename = '{0}-deps.dot'.format(module.get_name())
553 filename = os.path.join(options.outdir, filename)
554 graph = graphbuilder.create_module_file_graph(module)
555 with open(filename, 'w') as outfile:
556 graph.write(outfile)
558 if __name__ == '__main__':
559 main()