scripts/gen-api-gtkdoc.py

   1 #!/usr/bin/env python3
   2 #
   3 #  Copyright 2015 The Geany contributors
   4 #
   5 #  This program is free software; you can redistribute it and/or modify
   6 #  it under the terms of the GNU General Public License as published by
   7 #  the Free Software Foundation; either version 2 of the License, or
   8 #  (at your option) any later version.
   9 #
  10 #  This program is distributed in the hope that it will be useful,
  11 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #  GNU General Public License for more details.
  14 #
  15 #  You should have received a copy of the GNU General Public License
  16 #  along with this program; if not, write to the Free Software
  17 #  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
  18 #  MA 02110-1301, USA.
  19
  20 import os
  21 import sys
  22 import re
  23 from lxml import etree
  24 from optparse import OptionParser
  25
  26
  27 def normalize_text(s):
  28     r"""
  29     Normalizes whitespace in text.
  30
  31     >>> normalize_text("asd xxx")
  32     'asd xxx'
  33     >>> normalize_text(" asd\nxxx  ")
  34     'asd xxx'
  35     """
  36     return s.replace("\n", " ").strip()
  37
  38
  39 CXX_NAMESPACE_RE = re.compile(r'[_a-zA-Z][_0-9a-zA-Z]*::')
  40 def fix_definition(s):
  41     """
  42     Removes C++ name qualifications from some definitions.
  43
  44     For example:
  45
  46     >>> fix_definition("bool flag")
  47     'bool flag'
  48     >>> fix_definition("bool FooBar::flag")
  49     'bool flag'
  50     >>> fix_definition("void(* _GeanyObjectClass::project_open) (GKeyFile *keyfile)")
  51     'void(* project_open) (GKeyFile *keyfile)'
  52
  53     """
  54     return CXX_NAMESPACE_RE.sub(r"", s)
  55
  56
  57 class AtDoc(object):
  58     def __init__(self):
  59         self.retval = None
  60         self.since = ""
  61         self.annot = []
  62
  63     def cb(self, type, str):
  64         if (type == "param"):
  65             words = str.split(" ", 2)
  66             self.annot = []
  67         elif (type == "return"):
  68             self.annot = []
  69         elif (type == "since"):
  70             self.since = str.rstrip()
  71         elif type in ("geany:nullable",
  72                       "geany:optional",
  73                       "geany:out",
  74                       "geany:skip",
  75                       "geany:closure",
  76                       "geany:destroy"):
  77             self.annot.append(type.split(":")[1])
  78         elif type in ("geany:array",
  79                       "geany:transfer",
  80                       "geany:element-type",
  81                       "geany:scope"):
  82             type = type.split(":")[1]
  83             if len(str):
  84                 str = " " + str
  85             self.annot.append("%s%s" % (type, str))
  86         elif (type == "see"):
  87             return "See " + str
  88         elif type in ("a", "c") and str in ("NULL", "TRUE", "FALSE"):
  89             # FIXME: some of Geany does @a NULL instead of @c NULL
  90             return "%" + str
  91         elif (type == "a"):
  92             return "@" + str
  93         else:
  94             return str
  95
  96         return ""
  97
  98
  99 class DoxygenProcess(object):
 100     def __init__(self):
 101         self.at = None
 102
 103     # http://stackoverflow.com/questions/4624062/get-all-text-inside-a-tag-in-lxml
 104     @staticmethod
 105     def stringify_children(node):
 106         from lxml.etree import tostring
 107         from itertools import chain
 108         parts = ([node.text] +
 109                  list(chain(*([c.text, tostring(c, encoding='unicode'), c.tail] for c in node.getchildren()))) +
 110                  [node.tail])
 111         # filter removes possible Nones in texts and tails
 112         return "".join(filter(None, parts))
 113
 114     def get_program_listing(self, xml):
 115         from lxml.etree import tostring
 116         arr = ["", "|[<!-- language=\"C\" -->"]
 117         for l in xml.getchildren():
 118             if (l.tag == "codeline"):
 119                 # a codeline is of the form
 120                 # <highlight class="normal">GeanyDocument<sp/>*doc<sp/>=<sp/>...;</highlight>
 121                 # <sp/> tags must be replaced with spaces, then just use the text
 122                 h = l.find("highlight")
 123                 if h is not None:
 124                     html = tostring(h, encoding='unicode')
 125                     html = html.replace("<sp/>", " ")
 126                     arr.append("  " + tostring(etree.HTML(html), method="text", encoding='unicode'))
 127         arr.append("]|")
 128         return "\n".join(arr)
 129
 130     def join_annot(self):
 131         s = " ".join(map(lambda x: "(%s)" % x, self.at.annot))
 132         return s + ": " if s else ""
 133
 134     def process_element(self, xml):
 135         self.at = AtDoc()
 136         s = self.__process_element(xml)
 137         return s
 138
 139     def get_extra(self):
 140         return self.join_annot()
 141
 142     def get_return(self):
 143         return self.at.retval
 144
 145     def get_since(self):
 146         return self.at.since
 147
 148     def __process_element(self, xml):
 149         s = ""
 150
 151         if xml.text:
 152             s += xml.text
 153         for n in xml.getchildren():
 154             if n.tag == "emphasis":
 155                 s += self.at.cb("a", self.__process_element(n))
 156             if n.tag == "computeroutput":
 157                 s += self.at.cb("c", self.__process_element(n))
 158             if n.tag == "itemizedlist":
 159                 s += "\n" + self.__process_element(n)
 160             if n.tag == "listitem":
 161                 s += " - " + self.__process_element(n)
 162             if n.tag == "para":
 163                 s += self.__process_element(n) + "\n"
 164             if n.tag == "ref":
 165                 s += n.text if n.text else ""
 166             if n.tag == "simplesect":
 167                 ss = self.at.cb(n.get("kind"), self.__process_element(n))
 168                 s += ss + "\n" if ss else ""
 169             if n.tag == "programlisting":
 170                 s += self.get_program_listing(n)
 171             if n.tag == "xrefsect":
 172                 s += self.__process_element(n)
 173             if n.tag == "xreftitle":
 174                 s += self.__process_element(n) + ": "
 175             if n.tag == "xrefdescription":
 176                 s += self.__process_element(n)
 177             if n.tag == "ulink":
 178                 s += self.__process_element(n)
 179             if n.tag == "linebreak":
 180                 s += "\n"
 181             if n.tag == "ndash":
 182                 s += "--"
 183                 # workaround for doxygen bug #646002
 184             if n.tag == "htmlonly":
 185                 s += ""
 186             if n.tail:
 187                 s += n.tail
 188             if n.tag.startswith("param"):
 189                 pass  # parameters are handled separately in DoxyFunction::from_memberdef()
 190         return s
 191
 192
 193 class DoxyMember(object):
 194     def __init__(self, name, brief, extra=""):
 195         self.name       = name
 196         self.brief      = brief
 197         self.extra      = extra
 198
 199
 200 class DoxyElement(object):
 201
 202     def __init__(self, name, definition, **kwargs):
 203         self.name       = name
 204         self.definition = definition
 205         self.brief      = kwargs.get('brief', "")
 206         self.detail     = kwargs.get('detail', "")
 207         self.members    = kwargs.get('members', [])
 208         self.since      = kwargs.get('since', "")
 209         self.extra      = kwargs.get('extra', "")
 210         self.retval     = kwargs.get('retval', None)
 211
 212     def is_documented(self):
 213         if (normalize_text(self.brief)) != "":
 214             return True
 215         return False
 216
 217     def add_brief(self, xml):
 218         proc = DoxygenProcess()
 219         self.brief = proc.process_element(xml)
 220         self.extra += proc.get_extra()
 221
 222     def add_detail(self, xml):
 223         proc = DoxygenProcess()
 224         self.detail = proc.process_element(xml)
 225         self.extra += proc.get_extra()
 226         self.since = proc.get_since()
 227
 228     def add_member(self, xml):
 229         name = xml.find("name").text
 230         proc = DoxygenProcess()
 231         brief = proc.process_element(xml.find("briefdescription"))
 232         # optional doxygen command output appears within <detaileddescription />
 233         proc.process_element(xml.find("detaileddescription"))
 234         self.members.append(DoxyMember(name, normalize_text(brief), proc.get_extra()))
 235
 236     def add_param(self, xml):
 237         name = xml.find("parameternamelist").find("parametername").text
 238         proc = DoxygenProcess()
 239         brief = proc.process_element(xml.find("parameterdescription"))
 240         self.members.append(DoxyMember(name, normalize_text(brief), proc.get_extra()))
 241
 242     def add_return(self, xml):
 243         proc = DoxygenProcess()
 244         brief = proc.process_element(xml)
 245         self.retval = DoxyMember("ret", normalize_text(brief), proc.get_extra())
 246
 247     def to_gtkdoc(self):
 248         s = []
 249         s.append("/**")
 250         s.append(" * %s: %s" % (self.name, self.extra))
 251         for p in self.members:
 252             s.append(" * @%s: %s %s" % (p.name, p.extra, p.brief))
 253         s.append(" *")
 254         s.append(" * %s" % self.brief.replace("\n", "\n * "))
 255         s.append(" *")
 256         s.append(" * %s" % self.detail.replace("\n", "\n * "))
 257         s.append(" *")
 258         if self.retval:
 259             s.append(" * Returns: %s %s" % (self.retval.extra, self.retval.brief))
 260         if self.since:
 261             s.append(" *")
 262             s.append(" * Since: %s" % self.since)
 263         s.append(" */")
 264         s.append("")
 265         return "\n".join(s)
 266
 267
 268 class DoxyTypedef(DoxyElement):
 269     @staticmethod
 270     def from_memberdef(xml):
 271         name = xml.find("name").text
 272         d = normalize_text(xml.find("definition").text)
 273         d += ";"
 274         return DoxyTypedef(name, d)
 275
 276
 277 class DoxyEnum(DoxyElement):
 278     @staticmethod
 279     def from_memberdef(xml):
 280         name = xml.find("name").text
 281         d = "typedef enum {\n"
 282         for member in xml.findall("enumvalue"):
 283             v = member.find("initializer")
 284             d += "\t%s%s,\n" % (member.find("name").text, " "+v.text if v is not None else "")
 285         d += "} %s;\n" % name
 286
 287         e = DoxyEnum(name, d)
 288         e.add_brief(xml.find("briefdescription"))
 289         for p in xml.findall("enumvalue"):
 290             e.add_member(p)
 291         return e
 292
 293
 294 class DoxyStruct(DoxyElement):
 295     @staticmethod
 296     def from_compounddef(xml, typedefs=[]):
 297         name = xml.find("compoundname").text
 298         d = "struct %s {\n" % name
 299         memberdefs = xml.xpath(".//sectiondef[@kind='public-attrib']/memberdef")
 300         for p in memberdefs:
 301             # workaround for struct members. g-ir-scanner can't properly map struct members
 302             # (beginning with struct GeanyFoo) to the typedef and assigns a generic type for them
 303             # thus we fix that up here and enforce usage of the typedef. These are written
 304             # out first, before any struct definition, for this reason
 305             # Exception: there are no typedefs for GeanyFooPrivate so skip those. Their exact
 306             # type isn't needed anyway
 307             s = fix_definition(p.find("definition").text).lstrip()
 308             proc = DoxygenProcess()
 309             brief = proc.process_element(p.find("briefdescription"))
 310             private = (normalize_text(brief) == "")
 311             words = s.split()
 312             if (words[0] == "struct"):
 313                 if not (words[1].endswith("Private") or words[1].endswith("Private*")):
 314                     s = " ".join(words[1:])
 315             d += "\t/*< %s >*/\n\t%s;\n" % ("private" if private else "public", s)
 316
 317         d += "};\n"
 318         e = DoxyStruct(name, d)
 319         e.add_brief(xml.find("briefdescription"))
 320         for p in memberdefs:
 321             e.add_member(p)
 322         return e
 323
 324
 325 class DoxyFunction(DoxyElement):
 326     @staticmethod
 327     def from_memberdef(xml):
 328         name = xml.find("name").text
 329         d = normalize_text(xml.find("definition").text)
 330         d += " " + xml.find("argsstring").text + ";"
 331         d = normalize_text(d)
 332
 333         e = DoxyFunction(name, d)
 334         e.add_brief(xml.find("briefdescription"))
 335         e.add_detail(xml.find("detaileddescription"))
 336         for p in xml.xpath(".//detaileddescription/*/parameterlist[@kind='param']/parameteritem"):
 337             e.add_param(p)
 338         x = xml.xpath(".//detaileddescription/*/simplesect[@kind='return']")
 339         if (len(x) > 0):
 340             e.add_return(x[0])
 341         return e
 342
 343
 344 def main(args):
 345     xml_dir = None
 346     outfile = None
 347     scioutfile = None
 348
 349     parser = OptionParser(usage="usage: %prog [options] XML_DIR")
 350     parser.add_option("--xmldir", metavar="DIRECTORY", help="Path to Doxygen-generated XML files",
 351                       action="store", dest="xml_dir")
 352     parser.add_option("-d", "--outdir", metavar="DIRECTORY", help="Path to Doxygen-generated XML files",
 353                       action="store", dest="outdir", default=".")
 354     parser.add_option("-o", "--output", metavar="FILE", help="Write output to FILE",
 355                       action="store", dest="outfile")
 356     parser.add_option("--sci-output", metavar="FILE", help="Write output to FILE (only sciwrappers)",
 357                       action="store", dest="scioutfile")
 358     opts, args = parser.parse_args(args[1:])
 359
 360     xml_dir = args[0]
 361
 362     if not (os.path.exists(xml_dir)):
 363         sys.stderr.write("invalid xml directory\n")
 364         return 1
 365
 366     transform = etree.XSLT(etree.parse(os.path.join(xml_dir, "combine.xslt")))
 367     doc = etree.parse(os.path.join(xml_dir, "index.xml"))
 368     root = transform(doc)
 369
 370     other = []
 371     enums = []
 372     typedefs = []
 373
 374     c_files = root.xpath(".//compounddef[@kind='file']/compoundname[substring(.,string-length(.)-1)='.c']/..")
 375     h_files = root.xpath(".//compounddef[@kind='file']/compoundname[substring(.,string-length(.)-1)='.h']/..")
 376
 377     for f in h_files:
 378         if not (f.find("compoundname").text.endswith("private.h")):
 379             for n0 in f.xpath(".//*/memberdef[@kind='typedef' and @prot='public']"):
 380                 if not (DoxygenProcess.stringify_children(n0.find("type")).startswith("enum")):
 381                     e = DoxyTypedef.from_memberdef(n0)
 382                     typedefs.append(e)
 383
 384         for n0 in f.xpath(".//*/memberdef[@kind='enum' and @prot='public']"):
 385             e = DoxyEnum.from_memberdef(n0)
 386             enums.append(e)
 387
 388     for n0 in root.xpath(".//compounddef[@kind='struct' and @prot='public']"):
 389         e = DoxyStruct.from_compounddef(n0)
 390         other.append(e)
 391
 392     for f in c_files:
 393         for n0 in f.xpath(".//*/memberdef[@kind='function' and @prot='public']"):
 394             e = DoxyFunction.from_memberdef(n0)
 395             other.append(e)
 396
 397     if (opts.outfile):
 398         try:
 399             outfile = open(opts.outfile, "w+")
 400         except OSError as err:
 401             sys.stderr.write("failed to open \"%s\" for writing (%s)\n" % (opts.outfile, err.strerror))
 402             return 1
 403     else:
 404         outfile = sys.stdout
 405
 406     if (opts.scioutfile):
 407         try:
 408             scioutfile = open(opts.scioutfile, "w+")
 409         except OSError as err:
 410             sys.stderr.write("failed to open \"%s\" for writing (%s)\n" % (opts.scioutfile, err.strerror))
 411             return 1
 412     else:
 413         scioutfile = outfile
 414
 415     try:
 416         outfile.write("/*\n * Automatically generated file - do not edit\n */\n\n")
 417         outfile.write("#include \"gtkcompat.h\"\n")
 418         outfile.write("#include \"Scintilla.h\"\n")
 419         outfile.write("#include \"ScintillaWidget.h\"\n")
 420         if (scioutfile != outfile):
 421             scioutfile.write("/*\n * Automatically generated file - do not edit\n */\n\n")
 422             scioutfile.write("#include \"gtkcompat.h\"\n")
 423             scioutfile.write("#include \"Scintilla.h\"\n")
 424             scioutfile.write("#include \"ScintillaWidget.h\"\n")
 425
 426         # write enums first, so typedefs to them are valid (as forward enum declaration
 427         # is invalid).  It's fine as an enum can't contain reference to other types.
 428         for e in filter(lambda x: x.is_documented(), enums):
 429             outfile.write("\n\n")
 430             outfile.write(e.to_gtkdoc())
 431             outfile.write(e.definition)
 432             outfile.write("\n\n")
 433
 434         # write typedefs second, they are possibly undocumented but still required (even
 435         # if they are documented, they must be written out without gtkdoc)
 436         for e in typedefs:
 437             outfile.write(e.definition)
 438             outfile.write("\n\n")
 439
 440         # write the rest (structures, functions, ...)
 441         for e in filter(lambda x: x.is_documented(), other):
 442             outfile.write("\n\n")
 443             outfile.write(e.to_gtkdoc())
 444             outfile.write(e.definition)
 445             outfile.write("\n\n")
 446             if (e.name.startswith("sci_")):
 447                 if (scioutfile != outfile):
 448                     scioutfile.write("\n\n")
 449                     scioutfile.write(e.to_gtkdoc())
 450                     scioutfile.write(e.definition)
 451                     scioutfile.write("\n\n")
 452
 453     except BrokenPipeError:
 454         # probably piped to head or tail
 455         return 0
 456
 457     return 0
 458
 459 if __name__ == "__main__":
 460     sys.exit(main(sys.argv))