scripts/gen-api-gtkdoc.py

   1 #!/usr/bin/env python
   2
   3 import os
   4 import sys
   5 import re
   6 from lxml import etree
   7 from optparse import OptionParser
   8
   9 def normalize_text(s):
  10     r"""
  11     Normalizes whitespace in text.
  12
  13     >>> normalize_text("asd xxx")
  14     'asd xxx'
  15     >>> normalize_text(" asd\nxxx  ")
  16     'asd xxx'
  17     """
  18     return s.replace("\n", " ").strip()
  19
  20 CXX_NAMESPACE_RE = re.compile(r'[_a-zA-Z][_0-9a-zA-Z]*::')
  21 def fix_definition(s):
  22     """
  23     Removes C++ name qualifications from some definitions.
  24
  25     For example:
  26
  27     >>> fix_definition("bool flag")
  28     'bool flag'
  29     >>> fix_definition("bool FooBar::flag")
  30     'bool flag'
  31     >>> fix_definition("void(* _GeanyObjectClass::project_open) (GKeyFile *keyfile)")
  32     'void(* project_open) (GKeyFile *keyfile)'
  33
  34     """
  35     return CXX_NAMESPACE_RE.sub(r"", s);
  36
  37 class AtAt(object):
  38
  39     def __init__(self):
  40         self.retval = None
  41         self.since = ""
  42         self.annot = []
  43
  44     def cb(type, str):
  45         return "@%s %s" % (type, str)
  46
  47 class AtDoc(object):
  48     def __init__(self):
  49         self.retval = None
  50         self.since = ""
  51         self.annot = []
  52
  53     def cb(self, type, str):
  54         if (type == "param"):
  55             words = str.split(" ", 2);
  56             #~ self.params.append(GtkDocParam.new(words[0], words[1].rstrip, self.annot))
  57             self.annot = []
  58         elif (type == "return"):
  59             #~ self.retval = GtkDocReturn.new(str.rstrip, self.annot)
  60             self.annot = []
  61         elif (type == "since"):
  62             self.since = str.rstrip()
  63         elif (type == "geany:skip"):
  64             self.annot.append("skip")
  65         elif (type == "geany:nullable") or (type == "geany:skip"):
  66             self.annot.append(type.split(":")[1])
  67         elif (type == "geany:cb"):
  68             self.annot.append("scope notified")
  69         elif (type == "geany:cbdata"):
  70             self.annot.append("closure")
  71         elif (type == "geany:cbfree"):
  72             self.annot.append("destroy")
  73         elif (type == "geany:transfer") or (type == "geany:element-type") or (type == "geany:scope"):
  74             type = type.split(":")[1]
  75             self.annot.append("%s %s" % (type, str))
  76         elif (type ==  "see"):
  77             return "See " + str
  78         elif (type ==  "a"):
  79             if (str != "NULL"): # FIXME: some of geany does @a NULL
  80                 return "@" + str
  81             else:
  82                 return str
  83         else:
  84             return str
  85
  86         return ""
  87
  88 class At(object):
  89     def __init__(self, cb):
  90         self.cb = cb
  91
  92 class DoxygenProcess(object):
  93     def __init__(self):
  94         self.at = None
  95
  96     # http://stackoverflow.com/questions/4624062/get-all-text-inside-a-tag-in-lxml
  97     @staticmethod
  98     def stringify_children(node):
  99         from lxml.etree import tostring
 100         from itertools import chain
 101         parts = ([node.text] +
 102                 list(chain(*([c.text, tostring(c).decode("utf-8"), c.tail] for c in node.getchildren()))) +
 103                 [node.tail])
 104         # filter removes possible Nones in texts and tails
 105         return "".join(filter(None, parts))
 106
 107     def get_program_listing(self, xml):
 108         #~ return "--- CODE ---"
 109         from lxml.etree import tostring
 110         arr = ["", "|[<!-- language=\"C\" -->"]
 111         for l in xml.getchildren():
 112             if (l.tag == "codeline"):
 113                 # a codeline is of the form
 114                 # <highlight class="normal">GeanyDocument<sp/>*doc<sp/>=<sp/>...;</highlight>
 115                 # <sp/> tags must be replaced with spaces, then just use the text
 116                 #~ html = self.stringify_children(l)
 117                 #~ print(etree.HTML(html))
 118                 h = l.find("highlight")
 119                 if h is not None:
 120                     html = tostring(h).decode("utf-8")
 121                     html = html.replace("<sp/>", " ")
 122                     arr.append("  " + tostring(etree.HTML(html), method="text").decode("utf-8"))
 123         arr.append("]|")
 124         return "\n".join(arr)
 125
 126     def join_annot(self):
 127         s = " ".join(map(lambda x: "(%s)" % x, self.at.annot))
 128         return s + ": " if s else ""
 129
 130     def process_element(self, xml):
 131         self.at = AtDoc()
 132         s = self.__process_element(xml)
 133         return s
 134
 135     def get_extra(self):
 136         return self.join_annot()
 137
 138     def get_return(self):
 139         return self.at.retval
 140
 141     def get_since(self):
 142         return self.at.since
 143
 144     def __process_element(self, xml):
 145         s = ""
 146
 147         if xml.text:
 148             s += xml.text
 149         for n in xml.getchildren():
 150             if n.tag == "emphasis":
 151                 s += self.at.cb("a", self.__process_element(n))
 152             if n.tag == "computeroutput":
 153                 s += self.at.cb("c", self.__process_element(n))
 154             if n.tag == "itemizedlist":
 155                 s += "\n" + self.__process_element(n)
 156             if n.tag == "listitem":
 157                 s += " - " + self.__process_element(n)
 158             if n.tag == "para":
 159                 s += self.__process_element(n) + "\n"
 160             if n.tag == "ref":
 161                 s += n.text if n.text else ""
 162             if n.tag == "simplesect":
 163                 ss = self.at.cb(n.get("kind"), self.__process_element(n))
 164                 s += ss if ss + "\n" else ""
 165             if n.tag == "programlisting":
 166                 s += self.get_program_listing(n)
 167             if n.tag == "xrefsect":
 168                 s += self.__process_element(n)
 169             if n.tag == "xreftitle":
 170                 s += self.__process_element(n) + ": "
 171             if n.tag == "xrefdescription":
 172                 s += self.__process_element(n)
 173             if n.tag == "ulink":
 174                 s += self.__process_element(n)
 175             if n.tag == "linebreak":
 176                 s += "\n"
 177             if n.tag == "ndash":
 178                 s += "--"
 179                 # workaround for doxygen bug #646002
 180             if n.tag == "htmlonly":
 181                 s += ""
 182             if n.tail:
 183                 s += n.tail
 184             if n.tag.startswith("param"):
 185                 pass # parameters are handled separately in DoxyFunction::from_memberdef()
 186         return s
 187
 188 class DoxyMember(object):
 189     def __init__(self, name, brief, extra = ""):
 190         self.name       = name
 191         self.brief      = brief
 192         self.extra      = extra
 193
 194
 195 class DoxyElement(object):
 196
 197     def __init__(self, name, definition, **kwargs):
 198         self.name       = name
 199         self.definition = definition
 200         self.brief      = kwargs.get('brief', "")
 201         self.detail     = kwargs.get('detail', "")
 202         self.members    = kwargs.get('members', [])
 203         self.since      = kwargs.get('since', "")
 204         self.extra      = kwargs.get('extra', "")
 205         self.retval     = kwargs.get('retval', None)
 206
 207     def is_documented(self):
 208         if (normalize_text(self.brief)) != "":
 209             return True
 210         return False
 211
 212     def add_brief(self, xml):
 213         proc = DoxygenProcess()
 214         self.brief = proc.process_element(xml)
 215         self.extra += proc.get_extra()
 216
 217     def add_detail(self, xml):
 218         proc = DoxygenProcess()
 219         self.detail = proc.process_element(xml)
 220         self.extra += proc.get_extra()
 221         self.since = proc.get_since()
 222
 223     def add_member(self, xml):
 224         name = xml.find("name").text
 225         proc = DoxygenProcess()
 226         brief = proc.process_element(xml.find("briefdescription"))
 227         # optional doxygen command output appears within <detaileddescription />
 228         proc.process_element(xml.find("detaileddescription"))
 229         self.members.append(DoxyMember(name, normalize_text(brief), proc.get_extra()))
 230
 231     def add_param(self, xml):
 232         name = xml.find("parameternamelist").find("parametername").text
 233         proc = DoxygenProcess()
 234         brief = proc.process_element(xml.find("parameterdescription"))
 235         self.members.append(DoxyMember(name, normalize_text(brief), proc.get_extra()))
 236
 237     def add_return(self, xml):
 238         proc = DoxygenProcess()
 239         brief = proc.process_element(xml)
 240         self.retval = DoxyMember("ret", normalize_text(brief), proc.get_extra())
 241
 242     def to_gtkdoc(self):
 243         s  = []
 244         s.append("/**")
 245         s.append(" * %s: %s" % (self.name, self.extra))
 246         for p in self.members:
 247             s.append(" * @%s: %s %s" % (p.name, p.extra, p.brief))
 248         s.append(" *")
 249         s.append(" * %s" % self.brief.replace("\n", "\n * "))
 250         s.append(" *")
 251         s.append(" * %s" % self.detail.replace("\n", "\n * "))
 252         s.append(" *")
 253         if self.retval:
 254             s.append(" * Returns: %s %s" % (self.retval.extra, self.retval.brief))
 255         if self.since:
 256             s.append(" *")
 257             s.append(" * Since: %s" % self.since)
 258         s.append(" */")
 259         s.append("")
 260         return "\n".join(s)
 261
 262 class DoxyTypedef(DoxyElement):
 263
 264     @staticmethod
 265     def from_memberdef(xml):
 266         name = xml.find("name").text
 267         d = normalize_text(xml.find("definition").text).replace("G_BEGIN_DECLS", "")
 268         d += ";"
 269         return DoxyTypedef(name, d)
 270
 271 class DoxyEnum(DoxyElement):
 272
 273     @staticmethod
 274     def from_memberdef(xml):
 275         name = xml.find("name").text
 276         d = "typedef enum {\n"
 277         for member in xml.findall("enumvalue"):
 278             v = member.find("initializer")
 279             d += "\t%s%s,\n" % ( member.find("name").text, " "+v.text if v is not None else "")
 280         d += "} %s;\n" % name
 281
 282         e = DoxyEnum(name, d)
 283         e.add_brief(xml.find("briefdescription"))
 284         for p in xml.findall("enumvalue"):
 285             e.add_member(p)
 286         return e
 287
 288 class DoxyStruct(DoxyElement):
 289
 290     @staticmethod
 291     def from_compounddef(xml, typedefs = []):
 292         name = xml.find("compoundname").text
 293         section = xml.find("sectiondef")
 294         d = "struct %s {\n" % name;
 295         for p in section.findall("memberdef"):
 296             # workaround for struct members. g-ir-scanner can't properly map struct members
 297             # (beginning with struct GeanyFoo) to the typedef and assigns a generic type for them
 298             # thus we fix that up here and enforce usage of the typedef. These are written
 299             # out first, before any struct definition, for this reason
 300             # Exception: there are no typedefs for GeanyFooPrivate so skip those. Their exact
 301             # type isn't needed anyway
 302             s = fix_definition(p.find("definition").text).lstrip()
 303             words = s.split()
 304             if (words[0] == "struct"):
 305                 if not (words[1].endswith("Private") or words[1].endswith("Private*")):
 306                     s = " ".join(words[1:])
 307             d += "\t%s;\n"  % s
 308
 309         d += "};\n"
 310         e = DoxyStruct(name, d)
 311         e.add_brief(xml.find("briefdescription"))
 312         for p in section.findall("memberdef"):
 313             e.add_member(p)
 314         return e
 315
 316 class DoxyFunction(DoxyElement):
 317
 318     @staticmethod
 319     def from_memberdef(xml):
 320         name = xml.find("name").text
 321         d = normalize_text(xml.find("definition").text.replace("G_BEGIN_DECLS", ""))
 322         d += " " + xml.find("argsstring").text + ";"
 323         d = normalize_text(d.replace("GEANY_API_SYMBOL", ""))
 324
 325         e = DoxyFunction(name, d)
 326         e.add_brief(xml.find("briefdescription"))
 327         e.add_detail(xml.find("detaileddescription"))
 328         for p in xml.xpath(".//detaileddescription/*/parameterlist[@kind='param']/parameteritem"):
 329             e.add_param(p)
 330         x = xml.xpath(".//detaileddescription/*/simplesect[@kind='return']")
 331         if (len(x) > 0):
 332             e.add_return(x[0])
 333         return e
 334
 335 def main(args):
 336
 337     xml_dir = None
 338     outfile = None
 339     scioutfile = None
 340
 341     parser = OptionParser(usage="usage: %prog [options] XML_DIR")
 342     parser.add_option("--xmldir", metavar="DIRECTORY", help="Path to Doxygen-generated XML files",
 343         action="store", dest="xml_dir")
 344     parser.add_option("-d", "--outdir", metavar="DIRECTORY", help="Path to Doxygen-generated XML files",
 345         action="store", dest="outdir", default=".")
 346     parser.add_option("-o", "--output", metavar="FILE", help="Write output to FILE",
 347         action="store", dest="outfile")
 348     parser.add_option("--sci-output", metavar="FILE", help="Write scintilla_object_* output to FILE",
 349         action="store", dest="scioutfile")
 350     opts, args = parser.parse_args(args[1:])
 351
 352     xml_dir = args[0]
 353     if (opts.outfile):
 354         outfile = open(opts.outfile, "w+")
 355     else:
 356         outfile=sys.stdout
 357
 358     if (opts.scioutfile):
 359         scioutfile = open(opts.scioutfile, "w+")
 360     else:
 361         scioutfile = outfile
 362
 363     if (outfile is None):
 364         sys.stderr.write("no output file\n")
 365         return 1
 366
 367     if not (os.path.exists(xml_dir)):
 368         sys.stderr.write("invalid xml directory\n")
 369         return 1
 370
 371     transform = etree.XSLT(etree.parse(os.path.join(xml_dir, "combine.xslt")))
 372     doc = etree.parse(os.path.join(xml_dir, "index.xml"))
 373     root = transform(doc)
 374
 375     other = []
 376     typedefs = []
 377
 378     c_files = root.xpath(".//compounddef[@kind='file']/compoundname[substring(.,string-length(.)-1)='.c']/..")
 379     h_files = root.xpath(".//compounddef[@kind='file']/compoundname[substring(.,string-length(.)-1)='.h']/..")
 380
 381     for f in h_files:
 382         if not (f.find("compoundname").text.endswith("private.h")):
 383             for n0 in f.xpath(".//*/memberdef[@kind='typedef' and @prot='public']"):
 384                 if not (n0.find("type").text.replace("G_BEGIN_DECLS", "").lstrip().startswith("enum")):
 385                     e = DoxyTypedef.from_memberdef(n0)
 386                     typedefs.append(e)
 387
 388         for n0 in f.xpath(".//*/memberdef[@kind='enum' and @prot='public']"):
 389             e = DoxyEnum.from_memberdef(n0)
 390             other.append(e)
 391
 392     for n0 in root.xpath(".//compounddef[@kind='struct' and @prot='public']"):
 393         e = DoxyStruct.from_compounddef(n0)
 394         other.append(e)
 395
 396     for f in c_files:
 397         for n0 in f.xpath(".//*/memberdef[@kind='function' and @prot='public']"):
 398             e = DoxyFunction.from_memberdef(n0)
 399             other.append(e)
 400
 401     outfile.write("#include <glib.h>\n")
 402     outfile.write("#include <gtk/gtk.h>\n")
 403     outfile.write("typedef struct _ScintillaObject ScintillaObject;\n")
 404     outfile.write("typedef struct TMSourceFile TMSourceFile;\n")
 405     outfile.write("typedef struct TMWorkspace TMWorkspace;\n")
 406
 407     # write typedefs first, they are possibly undocumented but still required (even
 408     # if they are documented, they must be written out without gtkdoc)
 409     for e in typedefs:
 410         outfile.write(e.definition)
 411         outfile.write("\n\n")
 412
 413     for e in filter(lambda x: x.is_documented(), other):
 414         outfile.write("\n\n")
 415         outfile.write(e.to_gtkdoc())
 416         outfile.write(e.definition)
 417         outfile.write("\n\n")
 418         if (e.name.startswith("sci_")):
 419             scioutfile.write(e.to_gtkdoc().replace("sci_", "scintilla_object_"))
 420             scioutfile.write(e.definition.replace("sci_", "scintilla_object_"))
 421             scioutfile.write("\n\n")
 422
 423     return 0
 424
 425 if __name__ == "__main__":
 426     sys.exit(main(sys.argv))