scripts/maint/redox.py

   1 #!/usr/bin/env python
   2 #
   3 #  Copyright (c) 2008-2019, The Tor Project, Inc.
   4 #  See LICENSE for licensing information.
   5 #
   6 # Hi!
   7 # I'm redox.py, the Tor redocumentation tool!
   8 # I am a horrible hack!
   9 # I read the output of doxygen from stderr, and add missing DOCDOC comments
  10 #   to tell you where documentation should go!
  11 # To use me, edit the stuff below...
  12 #  ...and run 'make doxygen 2>doxygen.stderr' ...
  13 #  ...and run ./scripts/maint/redox.py < doxygen.stderr !
  14 # I'll make a bunch of new files by adding missing DOCDOC comments to your
  15 #    source.  Those files will have names like ./src/common/util.c.newdoc.
  16 # You will want to look over the changes by hand before checking them in.
  17 #
  18 # So, here's your workflow:
  19 #
  20 # 0. Make sure you're running a bourne shell for the redirects below.
  21 # 1. make doxygen 1>doxygen.stdout 2>doxygen.stderr.
  22 # 2. grep Warning doxygen.stderr | grep -v 'is not documented' | less
  23 #      [This will tell you about all the bogus doxygen output you have]
  24 # 3. python ./scripts/maint/redox.py <doxygen.stderr
  25 #      [This will make lots of .newdoc files with DOCDOC comments for
  26 #       whatever was missing documentation.]
  27 # 4. Look over those .newdoc files, and see which docdoc comments you
  28 #     want to merge into the main file.  If it's all good, just run
  29 #     "mv fname.c.newdoc fname.c".  Otherwise, you'll need to merge
  30 #     the parts you like by hand.
  31
  32 # Future imports for Python 2.7, mandatory in 3.0
  33 from __future__ import division
  34 from __future__ import print_function
  35 from __future__ import unicode_literals
  36
  37 import re
  38 import sys
  39
  40 try:
  41     xrange  # Python 2
  42 except NameError:
  43     xrange = range  # Python 3
  44
  45 # Which files should we ignore warning from?  Mostly, these are external
  46 # files that we've snarfed in from somebody else, whose C we do no intend
  47 # to document for them.
  48 SKIP_FILES = [ "OpenBSD_malloc_Linux.c",
  49                "strlcat.c",
  50                "strlcpy.c",
  51                "sha256.c",
  52                "sha256.h",
  53                "aes.c",
  54                "aes.h" ]
  55
  56 # What names of things never need javadoc
  57 SKIP_NAME_PATTERNS = [ r'^.*_c_id$',
  58                        r'^.*_H_ID$' ]
  59
  60 # Which types of things should get DOCDOC comments added if they are
  61 # missing documentation?  Recognized types are in KINDS below.
  62 ADD_DOCDOCS_TO_TYPES = [ 'function', 'type', 'typedef' ]
  63 ADD_DOCDOCS_TO_TYPES += [ 'variable', ]
  64
  65 # ====================
  66 # The rest of this should not need hacking.
  67
  68 KINDS = [ "type", "field", "typedef", "define", "function", "variable",
  69           "enumeration" ]
  70
  71 NODOC_LINE_RE = re.compile(r'^([^:]+):(\d+): (\w+): (.*) is not documented\.$')
  72
  73 THING_RE = re.compile(r'^Member ([a-zA-Z0-9_]+).*\((typedef|define|function|variable|enumeration|macro definition)\) of (file|class) ')
  74
  75 SKIP_NAMES = [re.compile(s) for s in SKIP_NAME_PATTERNS]
  76
  77 def parsething(thing):
  78     """I figure out what 'foobar baz in quux quum is not documented' means,
  79        and return: the name of the foobar, and the kind of the foobar.
  80     """
  81     if thing.startswith("Compound "):
  82         tp, name = "type", thing.split()[1]
  83     else:
  84         m = THING_RE.match(thing)
  85         if not m:
  86             print(thing, "???? Format didn't match.")
  87             return None, None
  88         else:
  89             name, tp, parent = m.groups()
  90             if parent == 'class':
  91                 if tp == 'variable' or tp == 'function':
  92                     tp = 'field'
  93
  94     return name, tp
  95
  96 def read():
  97     """I snarf doxygen stderr from stdin, and parse all the "foo has no
  98        documentation messages.  I return a map from filename to lists
  99        of tuples of (alleged line number, name of thing, kind of thing)
 100     """
 101     errs = {}
 102     for line in sys.stdin:
 103         m = NODOC_LINE_RE.match(line)
 104         if m:
 105             file, line, tp, thing = m.groups()
 106             assert tp.lower() == 'warning'
 107             name, kind = parsething(thing)
 108             errs.setdefault(file, []).append((int(line), name, kind))
 109
 110     return errs
 111
 112 def findline(lines, lineno, ident):
 113     """Given a list of all the lines in the file (adjusted so 1-indexing works),
 114        a line number that ident is allegedly on, and ident, I figure out
 115        the line where ident was really declared."""
 116     lno = lineno
 117     for lineno in xrange(lineno, 0, -1):
 118         try:
 119             if ident in lines[lineno]:
 120                 return lineno
 121         except IndexError:
 122             continue
 123
 124     return None
 125
 126 FUNC_PAT = re.compile(r"^[A-Za-z0-9_]+\(")
 127
 128 def hascomment(lines, lineno, kind):
 129     """I return true if it looks like there's already a good comment about
 130        the thing on lineno of lines of type kind. """
 131     if "*/" in lines[lineno-1]:
 132         return True
 133     if kind == 'function' and FUNC_PAT.match(lines[lineno]):
 134         if "*/" in lines[lineno-2]:
 135             return True
 136     return False
 137
 138 def hasdocdoc(lines, lineno, kind):
 139     """I return true if it looks like there's already a docdoc comment about
 140        the thing on lineno of lines of type kind."""
 141     try:
 142         if "DOCDOC" in lines[lineno]:
 143             return True
 144     except IndexError:
 145         pass
 146     try:
 147         if "DOCDOC" in lines[lineno-1]:
 148             return True
 149     except IndexError:
 150         pass
 151     if kind == 'function' and FUNC_PAT.match(lines[lineno]):
 152         if "DOCDOC" in lines[lineno-2]:
 153             return True
 154     return False
 155
 156 def checkf(fn, errs):
 157     """I go through the output of read() for a single file, and build a list
 158        of tuples of things that want DOCDOC comments.  Each tuple has:
 159        the line number where the comment goes; the kind of thing; its name.
 160     """
 161     for skip in SKIP_FILES:
 162         if fn.endswith(skip):
 163             print("Skipping",fn)
 164             return
 165
 166     comments = []
 167     lines = [ None ]
 168     try:
 169         lines.extend( open(fn, 'r').readlines() )
 170     except IOError:
 171         return
 172
 173     for line, name, kind in errs:
 174         if any(pat.match(name) for pat in SKIP_NAMES):
 175             continue
 176
 177         if kind not in ADD_DOCDOCS_TO_TYPES:
 178             continue
 179
 180         ln = findline(lines, line, name)
 181         if ln == None:
 182             print("Couldn't find the definition of %s allegedly on %s of %s"%(
 183                 name, line, fn))
 184         else:
 185             if hasdocdoc(lines, line, kind):
 186 #                print "Has a DOCDOC"
 187 #                print fn, line, name, kind
 188 #                print "\t",lines[line-2],
 189 #                print "\t",lines[line-1],
 190 #                print "\t",lines[line],
 191 #                print "-------"
 192                 pass
 193             else:
 194                 if kind == 'function' and FUNC_PAT.match(lines[ln]):
 195                     ln = ln - 1
 196
 197                 comments.append((ln, kind, name))
 198
 199     return comments
 200
 201 def applyComments(fn, entries):
 202     """I apply lots of comments to the file in fn, making a new .newdoc file.
 203     """
 204     N = 0
 205
 206     lines = [ None ]
 207     try:
 208         lines.extend( open(fn, 'r').readlines() )
 209     except IOError:
 210         return
 211
 212     # Process the comments in reverse order by line number, so that
 213     # the line numbers for the ones we haven't added yet remain valid
 214     # until we add them.  Standard trick.
 215     entries.sort()
 216     entries.reverse()
 217
 218     for ln, kind, name in entries:
 219
 220         lines.insert(ln, "/* DOCDOC %s */\n"%name)
 221         N += 1
 222
 223     outf = open(fn+".newdoc", 'w')
 224     for line in lines[1:]:
 225         outf.write(line)
 226     outf.close()
 227
 228     print("Added %s DOCDOCs to %s" %(N, fn))
 229
 230 e = read()
 231
 232 for fn, errs in e.iteritems():
 233     print(repr((fn, errs)))
 234     comments = checkf(fn, errs)
 235     if comments:
 236         applyComments(fn, comments)