scripts/gen-api-gtkdoc.py

   1 #!/usr/bin/env python
   2
   3 import os
   4 import sys
   5 import re
   6 from lxml import etree
   7 from optparse import OptionParser
   8
   9 # " asd\nxxx  " => "asd xxx"
  10 def normalize_text(s):
  11         return s.replace("\n", " ").lstrip().rstrip()
  12
  13 assert(normalize_text("asd xxx") == "asd xxx")
  14 assert(normalize_text(" asd\nxxx  ") == "asd xxx")
  15
  16 # doxygen records some definitions in C++ style, fix those
  17 # "bool FooBar::flag" => "bool flag"
  18 # void(* _GeanyObjectClass::project_open) (GKeyFile *keyfile) => void(* project_open) (GKeyFile *keyfile)
  19 prog = re.compile(r'[_a-zA-Z][_0-9a-zA-Z]*::')
  20 def fix_definition(s):
  21         return prog.sub(r"", s);
  22
  23 assert(fix_definition("bool flag") == "bool flag")
  24 assert(fix_definition("bool FooBar::flag") == "bool flag")
  25 assert(fix_definition("void(* _GeanyObjectClass::project_open) (GKeyFile *keyfile)") == "void(* project_open) (GKeyFile *keyfile)")
  26
  27 class AtAt(object):
  28
  29         def __init__(self):
  30                 self.retval = None
  31                 self.since = ""
  32                 self.annot = []
  33
  34         def cb(type, str):
  35                 return "@%s %s" % (type, str)
  36
  37 class AtDoc(object):
  38         def __init__(self):
  39                 self.retval = None
  40                 self.since = ""
  41                 self.annot = []
  42
  43         def cb(self, type, str):
  44                 if (type == "param"):
  45                         words = str.split(" ", 2);
  46                         #~ self.params.append(GtkDocParam.new(words[0], words[1].rstrip, self.annot))
  47                         self.annot = []
  48                 elif (type == "return"):
  49                         #~ self.retval = GtkDocReturn.new(str.rstrip, self.annot)
  50                         self.annot = []
  51                 elif (type == "since"):
  52                         self.since = str.rstrip()
  53                 elif (type == "geany:skip"):
  54                         self.annot.append("skip")
  55                 elif (type == "geany:nullable") or (type == "geany:skip"):
  56                         self.annot.append(type.split(":")[1])
  57                 elif (type == "geany:cb"):
  58                         self.annot.append("scope notified")
  59                 elif (type == "geany:cbdata"):
  60                         self.annot.append("closure")
  61                 elif (type == "geany:cbfree"):
  62                         self.annot.append("destroy")
  63                 elif (type == "geany:transfer") or (type == "geany:element-type") or (type == "geany:scope"):
  64                         type = type.split(":")[1]
  65                         self.annot.append("%s %s" % (type, str))
  66                 elif (type ==  "see"):
  67                         return "See " + str
  68                 elif (type ==  "a"):
  69                         if (str != "NULL"): # FIXME: some of geany does @a NULL
  70                                 return "@" + str
  71                         else:
  72                                 return str
  73                 else:
  74                         return str
  75
  76                 return ""
  77
  78 class At(object):
  79         def __init__(self, cb):
  80                 self.cb = cb
  81
  82 class DoxygenProcess(object):
  83         def __init__(self):
  84                 self.at = None
  85
  86         # http://stackoverflow.com/questions/4624062/get-all-text-inside-a-tag-in-lxml
  87         @staticmethod
  88         def stringify_children(node):
  89                 from lxml.etree import tostring
  90                 from itertools import chain
  91                 parts = ([node.text] +
  92                                 list(chain(*([c.text, tostring(c).decode("utf-8"), c.tail] for c in node.getchildren()))) +
  93                                 [node.tail])
  94                 # filter removes possible Nones in texts and tails
  95                 return "".join(filter(None, parts))
  96
  97         def get_program_listing(self, xml):
  98                 #~ return "--- CODE ---"
  99                 from lxml.etree import tostring
 100                 arr = ["", "|[<!-- language=\"C\" -->"]
 101                 for l in xml.getchildren():
 102                         if (l.tag == "codeline"):
 103                                 # a codeline is of the form
 104                                 # <highlight class="normal">GeanyDocument<sp/>*doc<sp/>=<sp/>...;</highlight>
 105                                 # <sp/> tags must be replaced with spaces, then just use the text
 106                                 #~ html = self.stringify_children(l)
 107                                 #~ print(etree.HTML(html))
 108                                 h = l.find("highlight")
 109                                 if h is not None:
 110                                         html = tostring(h).decode("utf-8")
 111                                         html = html.replace("<sp/>", " ")
 112                                         arr.append("  " + tostring(etree.HTML(html), method="text").decode("utf-8"))
 113                 arr.append("]|")
 114                 return "\n".join(arr)
 115
 116         def join_annot(self):
 117                 s = " ".join(map(lambda x: "(%s)" % x, self.at.annot))
 118                 return s + ": " if s else ""
 119
 120         def process_element(self, xml):
 121                 self.at = AtDoc()
 122                 s = self.__process_element(xml)
 123                 return s
 124
 125         def get_extra(self):
 126                 return self.join_annot()
 127
 128         def get_return(self):
 129                 return self.at.retval
 130
 131         def get_since(self):
 132                 return self.at.since
 133
 134         def __process_element(self, xml):
 135                 s = ""
 136
 137                 if xml.text:
 138                         s += xml.text
 139                 for n in xml.getchildren():
 140                         if n.tag == "emphasis":
 141                                 s += self.at.cb("a", self.__process_element(n))
 142                         if n.tag == "computeroutput":
 143                                 s += self.at.cb("c", self.__process_element(n))
 144                         if n.tag == "itemizedlist":
 145                                 s += "\n" + self.__process_element(n)
 146                         if n.tag == "listitem":
 147                                 s += " - " + self.__process_element(n)
 148                         if n.tag == "para":
 149                                 s += self.__process_element(n) + "\n"
 150                         if n.tag == "ref":
 151                                 s += n.text if n.text else ""
 152                         if n.tag == "simplesect":
 153                                 ss = self.at.cb(n.get("kind"), self.__process_element(n))
 154                                 s += ss if ss + "\n" else ""
 155                         if n.tag == "programlisting":
 156                                 s += self.get_program_listing(n)
 157                         if n.tag == "xrefsect":
 158                                 s += self.__process_element(n)
 159                         if n.tag == "xreftitle":
 160                                 s += self.__process_element(n) + ": "
 161                         if n.tag == "xrefdescription":
 162                                 s += self.__process_element(n)
 163                         if n.tag == "ulink":
 164                                 s += self.__process_element(n)
 165                         if n.tag == "linebreak":
 166                                 s += "\n"
 167                         if n.tag == "ndash":
 168                                 s += "--"
 169                                 # workaround for doxygen bug #646002
 170                         if n.tag == "htmlonly":
 171                                 s += ""
 172                         if n.tail:
 173                                 s += n.tail
 174                         if n.tag.startswith("param"):
 175                                 pass # parameters are handled separately in DoxyFunction::from_memberdef()
 176                 return s
 177
 178 class DoxyMember(object):
 179         def __init__(self, name, brief, extra = ""):
 180                 self.name       = name
 181                 self.brief      = brief
 182                 self.extra      = extra
 183
 184
 185 class DoxyElement(object):
 186
 187         def __init__(self, name, definition, **kwargs):
 188                 self.name       = name
 189                 self.definition = definition
 190                 self.brief      = kwargs.get('brief', "")
 191                 self.detail     = kwargs.get('detail', "")
 192                 self.members    = kwargs.get('members', [])
 193                 self.since      = kwargs.get('since', "")
 194                 self.extra      = kwargs.get('extra', "")
 195                 self.retval     = kwargs.get('retval', None)
 196
 197         def is_documented(self):
 198                 if (normalize_text(self.brief)) != "":
 199                         return True
 200                 return False
 201
 202         def add_brief(self, xml):
 203                 proc = DoxygenProcess()
 204                 self.brief = proc.process_element(xml)
 205                 self.extra += proc.get_extra()
 206
 207         def add_detail(self, xml):
 208                 proc = DoxygenProcess()
 209                 self.detail = proc.process_element(xml)
 210                 self.extra += proc.get_extra()
 211                 self.since = proc.get_since()
 212
 213         def add_member(self, xml):
 214                 name = xml.find("name").text
 215                 proc = DoxygenProcess()
 216                 brief = proc.process_element(xml.find("briefdescription"))
 217                 # optional doxygen command output appears within <detaileddescription />
 218                 proc.process_element(xml.find("detaileddescription"))
 219                 self.members.append(DoxyMember(name, normalize_text(brief), proc.get_extra()))
 220
 221         def add_param(self, xml):
 222                 name = xml.find("parameternamelist").find("parametername").text
 223                 proc = DoxygenProcess()
 224                 brief = proc.process_element(xml.find("parameterdescription"))
 225                 self.members.append(DoxyMember(name, normalize_text(brief), proc.get_extra()))
 226
 227         def add_return(self, xml):
 228                 proc = DoxygenProcess()
 229                 brief = proc.process_element(xml)
 230                 self.retval = DoxyMember("ret", normalize_text(brief), proc.get_extra())
 231
 232         def to_gtkdoc(self):
 233                 s  = []
 234                 s.append("/**")
 235                 s.append(" * %s: %s" % (self.name, self.extra))
 236                 for p in self.members:
 237                         s.append(" * @%s: %s %s" % (p.name, p.extra, p.brief))
 238                 s.append(" *")
 239                 s.append(" * %s" % self.brief.replace("\n", "\n * "))
 240                 s.append(" *")
 241                 s.append(" * %s" % self.detail.replace("\n", "\n * "))
 242                 s.append(" *")
 243                 if self.retval:
 244                         s.append(" * Returns: %s %s" % (self.retval.extra, self.retval.brief))
 245                 if self.since:
 246                         s.append(" *")
 247                         s.append(" * Since: %s" % self.since)
 248                 s.append(" */")
 249                 s.append("")
 250                 return "\n".join(s)
 251
 252 class DoxyTypedef(DoxyElement):
 253
 254         @staticmethod
 255         def from_memberdef(xml):
 256                 name = xml.find("name").text
 257                 d = normalize_text(xml.find("definition").text).replace("G_BEGIN_DECLS", "")
 258                 d += ";"
 259                 return DoxyTypedef(name, d)
 260
 261 class DoxyEnum(DoxyElement):
 262
 263         @staticmethod
 264         def from_memberdef(xml):
 265                 name = xml.find("name").text
 266                 d = "typedef enum {\n"
 267                 for member in xml.findall("enumvalue"):
 268                         v = member.find("initializer")
 269                         d += "\t%s%s,\n" % ( member.find("name").text, " "+v.text if v is not None else "")
 270                 d += "} %s;\n" % name
 271
 272                 e = DoxyEnum(name, d)
 273                 e.add_brief(xml.find("briefdescription"))
 274                 for p in xml.findall("enumvalue"):
 275                         e.add_member(p)
 276                 return e
 277
 278 class DoxyStruct(DoxyElement):
 279
 280         @staticmethod
 281         def from_compounddef(xml, typedefs = []):
 282                 name = xml.find("compoundname").text
 283                 section = xml.find("sectiondef")
 284                 d = "struct %s {\n" % name;
 285                 for p in section.findall("memberdef"):
 286                         # workaround for struct members. g-ir-scanner can't properly map struct members
 287                         # (beginning with struct GeanyFoo) to the typedef and assigns a generic type for them
 288                         # thus we fix that up here and enforce usage of the typedef. These are written
 289                         # out first, before any struct definition, for this reason
 290                         # Exception: there are no typedefs for GeanyFooPrivate so skip those. Their exact
 291                         # type isn't needed anyway
 292                         s = fix_definition(p.find("definition").text).lstrip()
 293                         words = s.split()
 294                         if (words[0] == "struct"):
 295                                 if not (words[1].endswith("Private") or words[1].endswith("Private*")):
 296                                         s = " ".join(words[1:])
 297                         d += "\t%s;\n"  % s
 298
 299                 d += "};\n"
 300                 e = DoxyStruct(name, d)
 301                 e.add_brief(xml.find("briefdescription"))
 302                 for p in section.findall("memberdef"):
 303                         e.add_member(p)
 304                 return e
 305
 306 class DoxyFunction(DoxyElement):
 307
 308         @staticmethod
 309         def from_memberdef(xml):
 310                 name = xml.find("name").text
 311                 d = normalize_text(xml.find("definition").text.replace("G_BEGIN_DECLS", ""))
 312                 d += " " + xml.find("argsstring").text + ";"
 313                 d = normalize_text(d.replace("GEANY_API_SYMBOL", ""))
 314
 315                 e = DoxyFunction(name, d)
 316                 e.add_brief(xml.find("briefdescription"))
 317                 e.add_detail(xml.find("detaileddescription"))
 318                 for p in xml.xpath(".//detaileddescription/*/parameterlist[@kind='param']/parameteritem"):
 319                         e.add_param(p)
 320                 x = xml.xpath(".//detaileddescription/*/simplesect[@kind='return']")
 321                 if (len(x) > 0):
 322                         e.add_return(x[0])
 323                 return e
 324
 325 def main(args):
 326
 327         xml_dir = None
 328         outfile = None
 329         scioutfile = None
 330
 331         parser = OptionParser(usage="usage: %prog [options] XML_DIR")
 332         parser.add_option("--xmldir", metavar="DIRECTORY", help="Path to Doxygen-generated XML files",
 333                 action="store", dest="xml_dir")
 334         parser.add_option("-d", "--outdir", metavar="DIRECTORY", help="Path to Doxygen-generated XML files",
 335                 action="store", dest="outdir", default=".")
 336         parser.add_option("-o", "--output", metavar="FILE", help="Write output to FILE",
 337                 action="store", dest="outfile")
 338         parser.add_option("--sci-output", metavar="FILE", help="Write scintilla_object_* output to FILE",
 339                 action="store", dest="scioutfile")
 340         opts, args = parser.parse_args(args[1:])
 341
 342         xml_dir = args[0]
 343         if (opts.outfile):
 344                 outfile = open(opts.outfile, "w+")
 345         else:
 346                 outfile=sys.stdout
 347
 348         if (opts.scioutfile):
 349                 scioutfile = open(opts.scioutfile, "w+")
 350         else:
 351                 scioutfile = outfile
 352
 353         if (outfile is None):
 354                 sys.stderr.write("no output file\n")
 355                 return 1
 356
 357         if not (os.path.exists(xml_dir)):
 358                 sys.stderr.write("invalid xml directory\n")
 359                 return 1
 360
 361         transform = etree.XSLT(etree.parse(os.path.join(xml_dir, "combine.xslt")))
 362         doc = etree.parse(os.path.join(xml_dir, "index.xml"))
 363         root = transform(doc)
 364
 365         other = []
 366         typedefs = []
 367
 368         c_files = root.xpath(".//compounddef[@kind='file']/compoundname[substring(.,string-length(.)-1)='.c']/..")
 369         h_files = root.xpath(".//compounddef[@kind='file']/compoundname[substring(.,string-length(.)-1)='.h']/..")
 370
 371         for f in h_files:
 372                 if not (f.find("compoundname").text.endswith("private.h")):
 373                         for n0 in f.xpath(".//*/memberdef[@kind='typedef' and @prot='public']"):
 374                                 if not (n0.find("type").text.replace("G_BEGIN_DECLS", "").lstrip().startswith("enum")):
 375                                         e = DoxyTypedef.from_memberdef(n0)
 376                                         typedefs.append(e)
 377
 378                 for n0 in f.xpath(".//*/memberdef[@kind='enum' and @prot='public']"):
 379                         e = DoxyEnum.from_memberdef(n0)
 380                         other.append(e)
 381
 382         for n0 in root.xpath(".//compounddef[@kind='struct' and @prot='public']"):
 383                 e = DoxyStruct.from_compounddef(n0)
 384                 other.append(e)
 385
 386         for f in c_files:
 387                 for n0 in f.xpath(".//*/memberdef[@kind='function' and @prot='public']"):
 388                         e = DoxyFunction.from_memberdef(n0)
 389                         other.append(e)
 390
 391         outfile.write("#include <glib.h>\n")
 392         outfile.write("#include <gtk/gtk.h>\n")
 393         outfile.write("typedef struct _ScintillaObject ScintillaObject;\n")
 394         outfile.write("typedef struct TMSourceFile TMSourceFile;\n")
 395         outfile.write("typedef struct TMWorkspace TMWorkspace;\n")
 396
 397         # write typedefs first, they are possibly undocumented but still required (even
 398         # if they are documented, they must be written out without gtkdoc)
 399         for e in typedefs:
 400                 outfile.write(e.definition)
 401                 outfile.write("\n\n")
 402
 403         for e in filter(lambda x: x.is_documented(), other):
 404                 outfile.write("\n\n")
 405                 outfile.write(e.to_gtkdoc())
 406                 outfile.write(e.definition)
 407                 outfile.write("\n\n")
 408                 if (e.name.startswith("sci_")):
 409                         scioutfile.write(e.to_gtkdoc().replace("sci_", "scintilla_object_"))
 410                         scioutfile.write(e.definition.replace("sci_", "scintilla_object_"))
 411                         scioutfile.write("\n\n")
 412
 413         return 0
 414
 415 if __name__ == "__main__":
 416         sys.exit(main(sys.argv))