scripts/gen-api-gtkdoc.py

   1 #!/usr/bin/env python
   2 #
   3 #  Copyright 2015-2016 Thomas Martitz <kugel@rockbox.org>
   4 #
   5 #  This program is free software; you can redistribute it and/or modify
   6 #  it under the terms of the GNU General Public License as published by
   7 #  the Free Software Foundation; either version 2 of the License, or
   8 #  (at your option) any later version.
   9 #
  10 #  This program is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #  GNU General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU General Public License
  16 #  along with this program; if not, write to the Free Software
  17 #  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
  18 #  MA 02110-1301, USA.
  19
  20 import os
  21 import sys
  22 import re
  23 from lxml import etree
  24 from optparse import OptionParser
  25
  26
  27 def normalize_text(s):
  28     r"""
  29     Normalizes whitespace in text.
  30
  31     >>> normalize_text("asd xxx")
  32     'asd xxx'
  33     >>> normalize_text(" asd\nxxx  ")
  34     'asd xxx'
  35     """
  36     return s.replace("\n", " ").strip()
  37
  38
  39 CXX_NAMESPACE_RE = re.compile(r'[_a-zA-Z][_0-9a-zA-Z]*::')
  40 def fix_definition(s):
  41     """
  42     Removes C++ name qualifications from some definitions.
  43
  44     For example:
  45
  46     >>> fix_definition("bool flag")
  47     'bool flag'
  48     >>> fix_definition("bool FooBar::flag")
  49     'bool flag'
  50     >>> fix_definition("void(* _GeanyObjectClass::project_open) (GKeyFile *keyfile)")
  51     'void(* project_open) (GKeyFile *keyfile)'
  52
  53     """
  54     return CXX_NAMESPACE_RE.sub(r"", s)
  55
  56
  57 class AtDoc(object):
  58     def __init__(self):
  59         self.retval = None
  60         self.since = ""
  61         self.annot = []
  62
  63     def cb(self, type, str):
  64         if (type == "param"):
  65             words = str.split(" ", 2)
  66             self.annot = []
  67         elif (type == "return"):
  68             self.annot = []
  69         elif (type == "since"):
  70             self.since = str.rstrip()
  71         elif type in ("geany:nullable",
  72                       "geany:optional",
  73                       "geany:out",
  74                       "geany:skip",
  75                       "geany:closure",
  76                       "geany:destroy"):
  77             self.annot.append(type.split(":")[1])
  78         elif type in ("geany:transfer",
  79                       "geany:element-type",
  80                       "geany:scope"):
  81             type = type.split(":")[1]
  82             self.annot.append("%s %s" % (type, str))
  83         elif (type == "see"):
  84             return "See " + str
  85         elif type in ("a", "c") and str in ("NULL", "TRUE", "FALSE"):
  86             # FIXME: some of Geany does @a NULL instead of @c NULL
  87             return "%" + str
  88         elif (type == "a"):
  89             return "@" + str
  90         else:
  91             return str
  92
  93         return ""
  94
  95
  96 class DoxygenProcess(object):
  97     def __init__(self):
  98         self.at = None
  99
 100     # http://stackoverflow.com/questions/4624062/get-all-text-inside-a-tag-in-lxml
 101     @staticmethod
 102     def stringify_children(node):
 103         from lxml.etree import tostring
 104         from itertools import chain
 105         parts = ([node.text] +
 106                  list(chain(*([c.text, tostring(c).decode("utf-8"), c.tail] for c in node.getchildren()))) +
 107                  [node.tail])
 108         # filter removes possible Nones in texts and tails
 109         return "".join(filter(None, parts))
 110
 111     def get_program_listing(self, xml):
 112         from lxml.etree import tostring
 113         arr = ["", "|[<!-- language=\"C\" -->"]
 114         for l in xml.getchildren():
 115             if (l.tag == "codeline"):
 116                 # a codeline is of the form
 117                 # <highlight class="normal">GeanyDocument<sp/>*doc<sp/>=<sp/>...;</highlight>
 118                 # <sp/> tags must be replaced with spaces, then just use the text
 119                 h = l.find("highlight")
 120                 if h is not None:
 121                     html = tostring(h).decode("utf-8")
 122                     html = html.replace("<sp/>", " ")
 123                     arr.append("  " + tostring(etree.HTML(html), method="text").decode("utf-8"))
 124         arr.append("]|")
 125         return "\n".join(arr)
 126
 127     def join_annot(self):
 128         s = " ".join(map(lambda x: "(%s)" % x, self.at.annot))
 129         return s + ": " if s else ""
 130
 131     def process_element(self, xml):
 132         self.at = AtDoc()
 133         s = self.__process_element(xml)
 134         return s
 135
 136     def get_extra(self):
 137         return self.join_annot()
 138
 139     def get_return(self):
 140         return self.at.retval
 141
 142     def get_since(self):
 143         return self.at.since
 144
 145     def __process_element(self, xml):
 146         s = ""
 147
 148         if xml.text:
 149             s += xml.text
 150         for n in xml.getchildren():
 151             if n.tag == "emphasis":
 152                 s += self.at.cb("a", self.__process_element(n))
 153             if n.tag == "computeroutput":
 154                 s += self.at.cb("c", self.__process_element(n))
 155             if n.tag == "itemizedlist":
 156                 s += "\n" + self.__process_element(n)
 157             if n.tag == "listitem":
 158                 s += " - " + self.__process_element(n)
 159             if n.tag == "para":
 160                 s += self.__process_element(n) + "\n"
 161             if n.tag == "ref":
 162                 s += n.text if n.text else ""
 163             if n.tag == "simplesect":
 164                 ss = self.at.cb(n.get("kind"), self.__process_element(n))
 165                 s += ss + "\n" if ss else ""
 166             if n.tag == "programlisting":
 167                 s += self.get_program_listing(n)
 168             if n.tag == "xrefsect":
 169                 s += self.__process_element(n)
 170             if n.tag == "xreftitle":
 171                 s += self.__process_element(n) + ": "
 172             if n.tag == "xrefdescription":
 173                 s += self.__process_element(n)
 174             if n.tag == "ulink":
 175                 s += self.__process_element(n)
 176             if n.tag == "linebreak":
 177                 s += "\n"
 178             if n.tag == "ndash":
 179                 s += "--"
 180                 # workaround for doxygen bug #646002
 181             if n.tag == "htmlonly":
 182                 s += ""
 183             if n.tail:
 184                 s += n.tail
 185             if n.tag.startswith("param"):
 186                 pass  # parameters are handled separately in DoxyFunction::from_memberdef()
 187         return s
 188
 189
 190 class DoxyMember(object):
 191     def __init__(self, name, brief, extra=""):
 192         self.name       = name
 193         self.brief      = brief
 194         self.extra      = extra
 195
 196
 197 class DoxyElement(object):
 198
 199     def __init__(self, name, definition, **kwargs):
 200         self.name       = name
 201         self.definition = definition
 202         self.brief      = kwargs.get('brief', "")
 203         self.detail     = kwargs.get('detail', "")
 204         self.members    = kwargs.get('members', [])
 205         self.since      = kwargs.get('since', "")
 206         self.extra      = kwargs.get('extra', "")
 207         self.retval     = kwargs.get('retval', None)
 208
 209     def is_documented(self):
 210         if (normalize_text(self.brief)) != "":
 211             return True
 212         return False
 213
 214     def add_brief(self, xml):
 215         proc = DoxygenProcess()
 216         self.brief = proc.process_element(xml)
 217         self.extra += proc.get_extra()
 218
 219     def add_detail(self, xml):
 220         proc = DoxygenProcess()
 221         self.detail = proc.process_element(xml)
 222         self.extra += proc.get_extra()
 223         self.since = proc.get_since()
 224
 225     def add_member(self, xml):
 226         name = xml.find("name").text
 227         proc = DoxygenProcess()
 228         brief = proc.process_element(xml.find("briefdescription"))
 229         # optional doxygen command output appears within <detaileddescription />
 230         proc.process_element(xml.find("detaileddescription"))
 231         self.members.append(DoxyMember(name, normalize_text(brief), proc.get_extra()))
 232
 233     def add_param(self, xml):
 234         name = xml.find("parameternamelist").find("parametername").text
 235         proc = DoxygenProcess()
 236         brief = proc.process_element(xml.find("parameterdescription"))
 237         self.members.append(DoxyMember(name, normalize_text(brief), proc.get_extra()))
 238
 239     def add_return(self, xml):
 240         proc = DoxygenProcess()
 241         brief = proc.process_element(xml)
 242         self.retval = DoxyMember("ret", normalize_text(brief), proc.get_extra())
 243
 244     def to_gtkdoc(self):
 245         s = []
 246         s.append("/**")
 247         s.append(" * %s: %s" % (self.name, self.extra))
 248         for p in self.members:
 249             s.append(" * @%s: %s %s" % (p.name, p.extra, p.brief))
 250         s.append(" *")
 251         s.append(" * %s" % self.brief.replace("\n", "\n * "))
 252         s.append(" *")
 253         s.append(" * %s" % self.detail.replace("\n", "\n * "))
 254         s.append(" *")
 255         if self.retval:
 256             s.append(" * Returns: %s %s" % (self.retval.extra, self.retval.brief))
 257         if self.since:
 258             s.append(" *")
 259             s.append(" * Since: %s" % self.since)
 260         s.append(" */")
 261         s.append("")
 262         return "\n".join(s)
 263
 264
 265 class DoxyTypedef(DoxyElement):
 266     @staticmethod
 267     def from_memberdef(xml):
 268         name = xml.find("name").text
 269         d = normalize_text(xml.find("definition").text)
 270         d += ";"
 271         return DoxyTypedef(name, d)
 272
 273
 274 class DoxyEnum(DoxyElement):
 275     @staticmethod
 276     def from_memberdef(xml):
 277         name = xml.find("name").text
 278         d = "typedef enum {\n"
 279         for member in xml.findall("enumvalue"):
 280             v = member.find("initializer")
 281             d += "\t%s%s,\n" % (member.find("name").text, " "+v.text if v is not None else "")
 282         d += "} %s;\n" % name
 283
 284         e = DoxyEnum(name, d)
 285         e.add_brief(xml.find("briefdescription"))
 286         for p in xml.findall("enumvalue"):
 287             e.add_member(p)
 288         return e
 289
 290
 291 class DoxyStruct(DoxyElement):
 292     @staticmethod
 293     def from_compounddef(xml, typedefs=[]):
 294         name = xml.find("compoundname").text
 295         d = "struct %s {\n" % name
 296         memberdefs = xml.xpath(".//sectiondef[@kind='public-attrib']/memberdef")
 297         for p in memberdefs:
 298             # workaround for struct members. g-ir-scanner can't properly map struct members
 299             # (beginning with struct GeanyFoo) to the typedef and assigns a generic type for them
 300             # thus we fix that up here and enforce usage of the typedef. These are written
 301             # out first, before any struct definition, for this reason
 302             # Exception: there are no typedefs for GeanyFooPrivate so skip those. Their exact
 303             # type isn't needed anyway
 304             s = fix_definition(p.find("definition").text).lstrip()
 305             proc = DoxygenProcess()
 306             brief = proc.process_element(p.find("briefdescription"))
 307             private = (normalize_text(brief) == "")
 308             words = s.split()
 309             if (words[0] == "struct"):
 310                 if not (words[1].endswith("Private") or words[1].endswith("Private*")):
 311                     s = " ".join(words[1:])
 312             d += "\t/*< %s >*/\n\t%s;\n" % ("private" if private else "public", s)
 313
 314         d += "};\n"
 315         e = DoxyStruct(name, d)
 316         e.add_brief(xml.find("briefdescription"))
 317         for p in memberdefs:
 318             e.add_member(p)
 319         return e
 320
 321
 322 class DoxyFunction(DoxyElement):
 323     @staticmethod
 324     def from_memberdef(xml):
 325         name = xml.find("name").text
 326         d = normalize_text(xml.find("definition").text)
 327         d += " " + xml.find("argsstring").text + ";"
 328         d = normalize_text(d)
 329
 330         e = DoxyFunction(name, d)
 331         e.add_brief(xml.find("briefdescription"))
 332         e.add_detail(xml.find("detaileddescription"))
 333         for p in xml.xpath(".//detaileddescription/*/parameterlist[@kind='param']/parameteritem"):
 334             e.add_param(p)
 335         x = xml.xpath(".//detaileddescription/*/simplesect[@kind='return']")
 336         if (len(x) > 0):
 337             e.add_return(x[0])
 338         return e
 339
 340
 341 def main(args):
 342     xml_dir = None
 343     outfile = None
 344     scioutfile = None
 345
 346     parser = OptionParser(usage="usage: %prog [options] XML_DIR")
 347     parser.add_option("--xmldir", metavar="DIRECTORY", help="Path to Doxygen-generated XML files",
 348                       action="store", dest="xml_dir")
 349     parser.add_option("-d", "--outdir", metavar="DIRECTORY", help="Path to Doxygen-generated XML files",
 350                       action="store", dest="outdir", default=".")
 351     parser.add_option("-o", "--output", metavar="FILE", help="Write output to FILE",
 352                       action="store", dest="outfile")
 353     parser.add_option("--sci-output", metavar="FILE", help="Write output to FILE (only sciwrappers)",
 354                       action="store", dest="scioutfile")
 355     opts, args = parser.parse_args(args[1:])
 356
 357     xml_dir = args[0]
 358
 359     if not (os.path.exists(xml_dir)):
 360         sys.stderr.write("invalid xml directory\n")
 361         return 1
 362
 363     transform = etree.XSLT(etree.parse(os.path.join(xml_dir, "combine.xslt")))
 364     doc = etree.parse(os.path.join(xml_dir, "index.xml"))
 365     root = transform(doc)
 366
 367     other = []
 368     enums = []
 369     typedefs = []
 370
 371     c_files = root.xpath(".//compounddef[@kind='file']/compoundname[substring(.,string-length(.)-1)='.c']/..")
 372     h_files = root.xpath(".//compounddef[@kind='file']/compoundname[substring(.,string-length(.)-1)='.h']/..")
 373
 374     for f in h_files:
 375         if not (f.find("compoundname").text.endswith("private.h")):
 376             for n0 in f.xpath(".//*/memberdef[@kind='typedef' and @prot='public']"):
 377                 if not (DoxygenProcess.stringify_children(n0.find("type")).startswith("enum")):
 378                     e = DoxyTypedef.from_memberdef(n0)
 379                     typedefs.append(e)
 380
 381         for n0 in f.xpath(".//*/memberdef[@kind='enum' and @prot='public']"):
 382             e = DoxyEnum.from_memberdef(n0)
 383             enums.append(e)
 384
 385     for n0 in root.xpath(".//compounddef[@kind='struct' and @prot='public']"):
 386         e = DoxyStruct.from_compounddef(n0)
 387         other.append(e)
 388
 389     for f in c_files:
 390         for n0 in f.xpath(".//*/memberdef[@kind='function' and @prot='public']"):
 391             e = DoxyFunction.from_memberdef(n0)
 392             other.append(e)
 393
 394     if (opts.outfile):
 395         try:
 396             outfile = open(opts.outfile, "w+")
 397         except OSError as err:
 398             sys.stderr.write("failed to open \"%s\" for writing (%s)\n" % (opts.outfile, err.strerror))
 399             return 1
 400     else:
 401         outfile = sys.stdout
 402
 403     if (opts.scioutfile):
 404         try:
 405             scioutfile = open(opts.scioutfile, "w+")
 406         except OSError as err:
 407             sys.stderr.write("failed to open \"%s\" for writing (%s)\n" % (opts.scioutfile, err.strerror))
 408             return 1
 409     else:
 410         scioutfile = outfile
 411
 412     try:
 413         outfile.write("/*\n * Automatically generated file - do not edit\n */\n\n")
 414         outfile.write("#include \"gtkcompat.h\"\n")
 415         outfile.write("#include \"Scintilla.h\"\n")
 416         outfile.write("#include \"ScintillaWidget.h\"\n")
 417         if (scioutfile != outfile):
 418             scioutfile.write("/*\n * Automatically generated file - do not edit\n */\n\n")
 419             scioutfile.write("#include \"gtkcompat.h\"\n")
 420             scioutfile.write("#include \"Scintilla.h\"\n")
 421             scioutfile.write("#include \"ScintillaWidget.h\"\n")
 422
 423         # write enums first, so typedefs to them are valid (as forward enum declaration
 424         # is invalid).  It's fine as an enum can't contain reference to other types.
 425         for e in filter(lambda x: x.is_documented(), enums):
 426             outfile.write("\n\n")
 427             outfile.write(e.to_gtkdoc())
 428             outfile.write(e.definition)
 429             outfile.write("\n\n")
 430
 431         # write typedefs second, they are possibly undocumented but still required (even
 432         # if they are documented, they must be written out without gtkdoc)
 433         for e in typedefs:
 434             outfile.write(e.definition)
 435             outfile.write("\n\n")
 436
 437         # write the rest (structures, functions, ...)
 438         for e in filter(lambda x: x.is_documented(), other):
 439             outfile.write("\n\n")
 440             outfile.write(e.to_gtkdoc())
 441             outfile.write(e.definition)
 442             outfile.write("\n\n")
 443             if (e.name.startswith("sci_")):
 444                 if (scioutfile != outfile):
 445                     scioutfile.write("\n\n")
 446                     scioutfile.write(e.to_gtkdoc())
 447                     scioutfile.write(e.definition)
 448                     scioutfile.write("\n\n")
 449
 450     except BrokenPipeError:
 451         # probably piped to head or tail
 452         return 0
 453
 454     return 0
 455
 456 if __name__ == "__main__":
 457     sys.exit(main(sys.argv))