gtkdoc-mkdb.in

   1 #!@PYTHON@
   2 # -*- python; coding: utf-8 -*-
   3 #
   4 # gtk-doc - GTK DocBook documentation generator.
   5 # Copyright (C) 1998  Damon Chaplin
   6 #               2007-2016  Stefan Sauer
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  21 #
  22
  23 #############################################################################
  24 # Script      : gtkdoc-mkdb
  25 # Description : This creates the DocBook files from the edited templates.
  26 #############################################################################
  27
  28 from __future__ import print_function
  29
  30 import argparse
  31 from collections import OrderedDict
  32 import logging
  33 import os
  34 import re
  35 import string
  36 import sys
  37
  38 sys.path.append('@PYTHON_PACKAGE_DIR@')
  39 from gtkdoc import common
  40
  41 # Options
  42
  43 # name of documentation module
  44 MODULE = None
  45 TMPL_DIR = None
  46 DB_OUTPUT_DIR = None
  47 SOURCE_DIRS = None
  48 SOURCE_SUFFIXES = ''
  49 IGNORE_FILES = ''
  50 MAIN_SGML_FILE = None
  51 EXPAND_CONTENT_FILES = ''
  52 INLINE_MARKUP_MODE = None
  53 DEFAULT_STABILITY = None
  54 DEFAULT_INCLUDES = None
  55 OUTPUT_FORMAT = None
  56 NAME_SPACE = ''
  57 OUTPUT_ALL_SYMBOLS = None
  58 OUTPUT_SYMBOLS_WITHOUT_SINCE = None
  59 ROOT_DIR = "."
  60 OBJECT_TREE_FILE = None
  61 INTERFACES_FILE = None
  62 PREREQUISITES_FILE = None
  63 SIGNALS_FILE = None
  64 ARGS_FILE = None
  65
  66 # These global arrays store information on signals. Each signal has an entry
  67 # in each of these arrays at the same index, like a multi-dimensional array.
  68 SignalObjects = []        # The GtkObject which emits the signal.
  69 SignalNames = []        # The signal name.
  70 SignalReturns = []        # The return type.
  71 SignalFlags = []        # Flags for the signal
  72 SignalPrototypes = []        # The rest of the prototype of the signal handler.
  73
  74 # These global arrays store information on Args. Each Arg has an entry
  75 # in each of these arrays at the same index, like a multi-dimensional array.
  76 ArgObjects = []                # The GtkObject which has the Arg.
  77 ArgNames = []                # The Arg name.
  78 ArgTypes = []                # The Arg type - gint, GtkArrowType etc.
  79 ArgFlags = []                # How the Arg can be used - readable/writable etc.
  80 ArgNicks = []                # The nickname of the Arg.
  81 ArgBlurbs = []          # Docstring of the Arg.
  82 ArgDefaults = []        # Default value of the Arg.
  83 ArgRanges = []                # The range of the Arg type
  84
  85 # These global hashes store declaration info keyed on a symbol name.
  86 Declarations = {}
  87 DeclarationTypes = {}
  88 DeclarationConditional = {}
  89 DeclarationOutput = {}
  90 Deprecated = {}
  91 Since = {}
  92 StabilityLevel = {}
  93 StructHasTypedef = {}
  94
  95 # These global hashes store the existing documentation.
  96 SymbolDocs = {}
  97 SymbolTypes = {}
  98 SymbolParams = {}
  99 SymbolSourceFile = {}
 100 SymbolSourceLine = {}
 101 SymbolAnnotations = {}
 102
 103 # These global hashes store documentation scanned from the source files.
 104 SourceSymbolDocs = {}
 105 SourceSymbolParams = {}
 106 SourceSymbolSourceFile = {}
 107 SourceSymbolSourceLine = {}
 108
 109 # all documentation goes in here, so we can do coverage analysis
 110 AllSymbols = {}
 111 AllIncompleteSymbols = {}
 112 AllUnusedSymbols = {}
 113 AllDocumentedSymbols = {}
 114
 115 # Undeclared yet documented symbols
 116 UndeclaredSymbols = {}
 117
 118 # These global arrays store GObject, subclasses and the hierarchy (also of
 119 # non-object derived types).
 120 Objects = []
 121 ObjectLevels = []
 122 ObjectRoots = {}
 123
 124 Interfaces = {}
 125 Prerequisites = {}
 126
 127 # holds the symbols which are mentioned in <MODULE>-sections.txt and in which
 128 # section they are defined
 129 KnownSymbols = {}
 130 SymbolSection = {}
 131 SymbolSectionId = {}
 132
 133 # collects index entries
 134 IndexEntriesFull = {}
 135 IndexEntriesSince = {}
 136 IndexEntriesDeprecated = {}
 137
 138 # Standard C preprocessor directives, which we ignore for '#' abbreviations.
 139 PreProcessorDirectives = {
 140     'assert': 1,
 141     'define': 1,
 142     'elif': 1,
 143     'else': 1,
 144     'endif': 1,
 145     'error': 1,
 146     'if': 1,
 147     'ifdef': 1,
 148     'ifndef': 1,
 149     'include': 1,
 150     'line': 1,
 151     'pragma': 1,
 152     'unassert': 1,
 153     'undef': 1,
 154     'warning': 1
 155 }
 156
 157 # remember used annotation (to write minimal glossary)
 158 AnnotationsUsed = {}
 159
 160 # the regexp that parses the annotation is in ScanSourceFile()
 161 AnnotationDefinition = {
 162     # the GObjectIntrospection annotations are defined at:
 163     # https://live.gnome.org/GObjectIntrospection/Annotations
 164     'allow-none': "NULL is OK, both for passing and for returning.",
 165     'nullable': "NULL may be passed as the value in, out, in-out; or as a return value.",
 166     'not nullable': "NULL must not be passed as the value in, out, in-out; or as a return value.",
 167     'optional': "NULL may be passed instead of a pointer to a location.",
 168     'array': "Parameter points to an array of items.",
 169     'attribute': "Deprecated free-form custom annotation, replaced by (attributes) annotation.",
 170     'attributes': "Free-form key-value pairs.",
 171     'closure': "This parameter is a 'user_data', for callbacks; many bindings can pass NULL here.",
 172     'constructor': "This symbol is a constructor, not a static method.",
 173     'destroy': "This parameter is a 'destroy_data', for callbacks.",
 174     'default': "Default parameter value (for in case the <acronym>shadows</acronym>-to function has less parameters).",
 175     'element-type': "Generics and defining elements of containers and arrays.",
 176     'error-domains': "Typed errors. Similar to throws in Java.",
 177     'foreign': "This is a foreign struct.",
 178     'get-value-func': "The specified function is used to convert a struct from a GValue, must be a GTypeInstance.",
 179     'in': "Parameter for input. Default is <acronym>transfer none</acronym>.",
 180     'inout': "Parameter for input and for returning results. Default is <acronym>transfer full</acronym>.",
 181     'in-out': "Parameter for input and for returning results. Default is <acronym>transfer full</acronym>.",
 182     'method': "This is a method",
 183     'not-error': "A GError parameter is not to be handled like a normal GError.",
 184     'out': "Parameter for returning results. Default is <acronym>transfer full</acronym>.",
 185     'out caller-allocates': "Out parameter, where caller must allocate storage.",
 186     'out callee-allocates': "Out parameter, where caller must allocate storage.",
 187     'ref-func': "The specified function is used to ref a struct, must be a GTypeInstance.",
 188     'rename-to': "Rename the original symbol's name to SYMBOL.",
 189     'scope call': "The callback is valid only during the call to the method.",
 190     'scope async': "The callback is valid until first called.",
 191     'scope notified': "The callback is valid until the GDestroyNotify argument is called.",
 192     'set-value-func': "The specified function is used to convert from a struct to a GValue, must be a GTypeInstance.",
 193     'skip': "Exposed in C code, not necessarily available in other languages.",
 194     'transfer container': "Free data container after the code is done.",
 195     'transfer floating': "Alias for <acronym>transfer none</acronym>, used for objects with floating refs.",
 196     'transfer full': "Free data after the code is done.",
 197     'transfer none': "Don't free data after the code is done.",
 198     'type': "Override the parsed C type with given type.",
 199     'unref-func': "The specified function is used to unref a struct, must be a GTypeInstance.",
 200     'virtual': "This is the invoker for a virtual method.",
 201     'value': "The specified value overrides the evaluated value of the constant.",
 202     # Stability Level definition
 203     # https://bugzilla.gnome.org/show_bug.cgi?id=170860
 204     'Stable': '''The intention of a Stable interface is to enable arbitrary third parties to
 205 develop applications to these interfaces, release them, and have confidence that
 206 they will run on all minor releases of the product (after the one in which the
 207 interface was introduced, and within the same major release). Even at a major
 208 release, incompatible changes are expected to be rare, and to have strong
 209 justifications.
 210 ''',
 211     'Unstable': '''Unstable interfaces are experimental or transitional. They are typically used to
 212 give outside developers early access to new or rapidly changing technology, or
 213 to provide an interim solution to a problem where a more general solution is
 214 anticipated. No claims are made about either source or binary compatibility from
 215 one minor release to the next.
 216
 217 The Unstable interface level is a warning that these interfaces are  subject to
 218 change without warning and should not be used in unbundled products.
 219
 220 Given such caveats, customer impact need not be a factor when considering
 221 incompatible changes to an Unstable interface in a major or minor release.
 222 Nonetheless, when such changes are introduced, the changes should still be
 223 mentioned in the release notes for the affected release.
 224 ''',
 225     'Private': '''An interface that can be used within the GNOME stack itself, but that is not
 226 documented for end-users.  Such functions should only be used in specified and
 227 documented ways.
 228 ''',
 229 }
 230
 231 # Elements to consider non-block items in MarkDown parsing
 232 MD_TEXT_LEVEL_ELEMENTS = {"literal": 1,
 233                           "emphasis": 1,
 234                           "envar": 1,
 235                           "filename": 1,
 236                           "firstterm": 1,
 237                           "footnote": 1,
 238                           "function": 1,
 239                           "manvolnum": 1,
 240                           "option": 1,
 241                           "replaceable": 1,
 242                           "structfield": 1,
 243                           "structname": 1,
 244                           "title": 1,
 245                           "varname": 1,
 246                          }
 247 MD_ESCAPABLE_CHARS = {"\\": 1,
 248                       "`": 1,
 249                       "*": 1,
 250                       "_": 1,
 251                       "{": 1,
 252                       "}": 1,
 253                       "[": 1,
 254                       "]": 1,
 255                       "(": 1,
 256                       ")": 1,
 257                       '>': 1,
 258                       '#': 1,
 259                       "+": 1,
 260                       "-": 1,
 261                       ".": 1,
 262                       '!': 1,
 263                      }
 264 MD_GTK_ESCAPABLE_CHARS = {"@": 1,
 265                           "%": 1,
 266                          }
 267
 268 # Function and other declaration output settings.
 269 RETURN_TYPE_FIELD_WIDTH = 20
 270 SYMBOL_FIELD_WIDTH = 36
 271 MAX_SYMBOL_FIELD_WIDTH = 40
 272 SIGNAL_FIELD_WIDTH = 16
 273 PARAM_FIELD_COUNT = 2
 274
 275 # XML, SGML formatting helper
 276 doctype_header = None
 277
 278 # refentry template
 279 REFENTRY = string.Template(
 280 '''${header}
 281 <refentry id="${section_id}">
 282 <refmeta>
 283 <refentrytitle role="top_of_page" id="${section_id}.top_of_page">${title}</refentrytitle>
 284 <manvolnum>3</manvolnum>
 285 <refmiscinfo>${MODULE} Library${image}</refmiscinfo>
 286 </refmeta>
 287 <refnamediv>
 288 <refname>${title}</refname>
 289 <refpurpose>${short_desc}</refpurpose>
 290 </refnamediv>
 291 ${stability}
 292 ${functions_synop}${args_synop}${signals_synop}${object_anchors}${other_synop}${hierarchy}${prerequisites}${derived}${interfaces}${implementations}
 293 ${include_output}
 294 <refsect1 id="${section_id}.description" role="desc">
 295 <title role="desc.title">Description</title>
 296 ${extralinks}${long_desc}
 297 </refsect1>
 298 <refsect1 id="${section_id}.functions_details" role="details">
 299 <title role="details.title">Functions</title>
 300 ${functions_details}
 301 </refsect1>
 302 <refsect1 id="${section_id}.other_details" role="details">
 303 <title role="details.title">Types and Values</title>
 304 ${other_details}
 305 </refsect1>
 306 ${args_desc}${signals_desc}${see_also}
 307 </refentry>
 308 ''')
 309
 310
 311 parser = argparse.ArgumentParser()
 312 parser.add_argument('--module', default='')
 313 parser.add_argument('--source-dir', action='append', dest='source_dir', default=[])
 314 parser.add_argument('--source-suffixes', dest='source_suffixes', default='')
 315 parser.add_argument('--ignore-files', dest='ignore_files', default='')
 316 parser.add_argument('--output-dir', dest='output_dir', default='')
 317 parser.add_argument('--tmpl-dir', dest='tmpl_dir', default='')
 318 parser.add_argument('--main-sgml-file', dest='main_sgml_file', default='')
 319 parser.add_argument('--expand-content-files', dest='expand_content_files', default='')
 320 group = parser.add_mutually_exclusive_group()
 321 group.add_argument('--sgml-mode', action='store_true', default=False, dest='sgml_mode')
 322 group.add_argument('--xml-mode', action='store_true', default=False, dest='xml_mode')
 323 parser.add_argument('--default-stability', dest='default_stability', choices=['', 'Stable', 'Private', 'Unstable'], default='')
 324 parser.add_argument('--default-includes', dest='default_includes', default='')
 325 parser.add_argument('--output-format', dest='default_format', default='')
 326 parser.add_argument('--name-space', dest='name_space', default='')
 327 parser.add_argument('--outputallsymbols', default=False, action='store_true')
 328 parser.add_argument('--outputsymbolswithoutsince', default=False, action='store_true')
 329
 330
 331 def Run():
 332     global MODULE, TMPL_DIR, SOURCE_DIRS, SOURCE_SUFFIXES, IGNORE_FILES, MAIN_SGML_FILE, EXPAND_CONTENT_FILES, \
 333         INLINE_MARKUP_MODE, DEFAULT_STABILITY, DEFAULT_INCLUDES, OUTPUT_FORMAT, NAME_SPACE, OUTPUT_ALL_SYMBOLS, \
 334         OUTPUT_SYMBOLS_WITHOUT_SINCE, DB_OUTPUT_DIR, OBJECT_TREE_FILE, INTERFACES_FILE, PREREQUISITES_FILE, \
 335         SIGNALS_FILE, ARGS_FILE, doctype_header
 336
 337     options = parser.parse_args()
 338
 339     common.setup_logging()
 340
 341     # We should pass the options variable around instead of this global variable horror
 342     # but too much of the code expects these to be around. Fix this once the transition is done.
 343     MODULE = options.module
 344     TMPL_DIR = options.tmpl_dir
 345     SOURCE_DIRS = options.source_dir
 346     SOURCE_SUFFIXES = options.source_suffixes
 347     IGNORE_FILES = options.ignore_files
 348     MAIN_SGML_FILE = options.main_sgml_file
 349     EXPAND_CONTENT_FILES = options.expand_content_files
 350     INLINE_MARKUP_MODE = options.xml_mode or options.sgml_mode
 351     DEFAULT_STABILITY = options.default_stability
 352     DEFAULT_INCLUDES = options.default_includes
 353     OUTPUT_FORMAT = options.default_format
 354     NAME_SPACE = options.name_space
 355     OUTPUT_ALL_SYMBOLS = options.outputallsymbols
 356     OUTPUT_SYMBOLS_WITHOUT_SINCE = options.outputsymbolswithoutsince
 357
 358     logging.info(" ignore files: " + IGNORE_FILES)
 359
 360     # check output format
 361     if OUTPUT_FORMAT is None or OUTPUT_FORMAT == '':
 362         OUTPUT_FORMAT = "xml"
 363     else:
 364         OUTPUT_FORMAT = OUTPUT_FORMAT.lower()
 365     if OUTPUT_FORMAT != "xml":
 366         sys.exit("Invalid format '%s' passed to --output.format" % OUTPUT_FORMAT)
 367
 368     if not MAIN_SGML_FILE:
 369         # backwards compatibility
 370         if os.path.exists(MODULE + "-docs.sgml"):
 371             MAIN_SGML_FILE = MODULE + "-docs.sgml"
 372         else:
 373             MAIN_SGML_FILE = MODULE + "-docs.xml"
 374
 375     # extract docbook header or define default
 376     if os.path.exists(MAIN_SGML_FILE):
 377         INPUT = open(MAIN_SGML_FILE)
 378         doctype_header = ''
 379         for line in INPUT:
 380             if re.search(r'^\s*<(book|chapter|article)', line):
 381                 # check that the top-level tagSYSTEM or the doctype decl contain the xinclude namespace decl
 382                 if not re.search(r'http:\/\/www.w3.org\/200[13]\/XInclude', line) and not re.search(r'http:\/\/www.w3.org\/200[13]\/XInclude', doctype_header, flags=re.MULTILINE):
 383                     doctype_header = ''
 384                 break
 385
 386             # if there are SYSTEM ENTITIES here, we should prepend "../" to the path
 387             # FIXME: not sure if we can do this now, as people already work-around the problem
 388             # r'#<!ENTITY % ([a-zA-Z-]+) SYSTEM \"([^/][a-zA-Z./]+)\">', r'<!ENTITY % \1 SYSTEM \"../\2\">';
 389             line = re.sub(r'<!ENTITY % gtkdocentities SYSTEM "([^"]*)">', r'<!ENTITY % gtkdocentities SYSTEM "../\1">', line)
 390             doctype_header += line
 391         INPUT.close()
 392         doctype_header = doctype_header.strip()
 393     else:
 394         doctype_header = '''<?xml version="1.0"?>
 395 <!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN"
 396                "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd"
 397 [
 398   <!ENTITY % local.common.attrib "xmlns:xi  CDATA  #FIXED 'http://www.w3.org/2003/XInclude'">
 399   <!ENTITY % gtkdocentities SYSTEM "../xml/gtkdocentities.ent">
 400   %gtkdocentities;
 401 ]>'''
 402
 403     # All the files are written in subdirectories beneath here.
 404     TMPL_DIR = TMPL_DIR if TMPL_DIR else os.path.join(ROOT_DIR, "tmpl")
 405
 406     # This is where we put all the DocBook output.
 407     DB_OUTPUT_DIR = DB_OUTPUT_DIR if DB_OUTPUT_DIR else os.path.join(ROOT_DIR, "xml")
 408
 409     # This file contains the object hierarchy.
 410     OBJECT_TREE_FILE = os.path.join(ROOT_DIR, MODULE + ".hierarchy")
 411
 412     # This file contains the interfaces.
 413     INTERFACES_FILE = os.path.join(ROOT_DIR, MODULE + ".interfaces")
 414
 415     # This file contains the prerequisites.
 416     PREREQUISITES_FILE = os.path.join(ROOT_DIR, MODULE + ".prerequisites")
 417
 418     # This file contains signal arguments and names.
 419     SIGNALS_FILE = os.path.join(ROOT_DIR, MODULE + ".signals")
 420
 421     # The file containing Arg information.
 422     ARGS_FILE = os.path.join(ROOT_DIR, MODULE + ".args")
 423
 424     # Create the root DocBook output directory if it doens't exist.
 425     if not os.path.isdir(DB_OUTPUT_DIR):
 426         os.mkdir(DB_OUTPUT_DIR)
 427
 428     ReadKnownSymbols(os.path.join(ROOT_DIR, MODULE + "-sections.txt"))
 429     ReadSignalsFile(SIGNALS_FILE)
 430     ReadArgsFile(ARGS_FILE)
 431     ReadObjectHierarchy()
 432     ReadInterfaces()
 433     ReadPrerequisites()
 434
 435     ReadDeclarationsFile(os.path.join(ROOT_DIR, MODULE + "-decl.txt"), 0)
 436     if os.path.isfile(os.path.join(ROOT_DIR, MODULE + "-overrides.txt")):
 437         ReadDeclarationsFile(os.path.join(ROOT_DIR, MODULE + "-overrides.txt"), 1)
 438
 439     for sdir in SOURCE_DIRS:
 440         ReadSourceDocumentation(sdir)
 441
 442     changed = OutputDB(os.path.join(ROOT_DIR, MODULE + "-sections.txt"))
 443
 444     # If any of the DocBook files have changed, update the timestamp file (so
 445     # it can be used for Makefile dependencies).
 446     if changed or not os.path.exists(os.path.join(ROOT_DIR, "sgml.stamp")):
 447
 448         # try to detect the common prefix
 449         # GtkWidget, GTK_WIDGET, gtk_widget -> gtk
 450         if NAME_SPACE == '':
 451             pos = 0
 452             ratio = 0.0
 453             while True:
 454                 prefix = {}
 455                 letter = ''
 456                 for symbol in IndexEntriesFull.keys():
 457                     if NAME_SPACE == '' or NAME_SPACE.lower() in symbol.lower():
 458                         if len(symbol) > pos:
 459                             letter = symbol[pos:pos+1]
 460                             # stop prefix scanning
 461                             if letter == "_":
 462                                 # stop on "_"
 463                                 break
 464                             # Should we also stop on a uppercase char, if last was lowercase
 465                             #   GtkWidget, if we have the 'W' and had the 't' before
 466                             # or should we count upper and lowercase, and stop one 2nd uppercase, if we already had a lowercase
 467                             #   GtkWidget, the 'W' would be the 2nd uppercase and with 't','k' we had lowercase chars before
 468                             # need to recound each time as this is per symbol
 469                             ul = letter.upper()
 470                             if ul in prefix:
 471                                 prefix[ul] += 1
 472                             else:
 473                                 prefix[ul] = 1
 474
 475                 if letter != '' and letter != "_":
 476                     maxletter = ''
 477                     maxsymbols = 0
 478                     for letter in prefix.keys():
 479                         logging.debug("ns prefix: %s: %s", letter, prefix[letter])
 480                         if prefix[letter] > maxsymbols:
 481                             maxletter = letter
 482                             maxsymbols = prefix[letter]
 483
 484                     ratio = float(len(IndexEntriesFull)) / prefix[maxletter]
 485                     logging.debug('most symbols start with %s, that is %f', maxletter, (100 * ratio))
 486                     if ratio > 0.9:
 487                         # do another round
 488                         NAME_SPACE += maxletter
 489
 490                     pos += 1
 491
 492                 else:
 493                     ratio = 0.0
 494
 495                 if ratio < 0.9:
 496                     break
 497
 498         logging.info('namespace prefix ="%s"', NAME_SPACE)
 499
 500         OutputIndexFull()
 501         OutputDeprecatedIndex()
 502         OutputSinceIndexes()
 503         OutputAnnotationGlossary()
 504
 505         open(os.path.join(ROOT_DIR, 'sgml.stamp'), 'w').write('timestamp')
 506
 507 #############################################################################
 508 # Function    : OutputObjectList
 509 # Description : This outputs the alphabetical list of objects, in a columned
 510 #                table.
 511 #               FIXME: Currently this also outputs ancestor objects
 512 #                which may not actually be in this module.
 513 # Arguments   : none
 514 #############################################################################
 515
 516 def OutputObjectList():
 517     cols = 3
 518
 519     # FIXME: use .xml
 520     old_object_index = os.path.join(DB_OUTPUT_DIR, "object_index.sgml")
 521     new_object_index = os.path.join(DB_OUTPUT_DIR, "object_index.new")
 522
 523     OUTPUT = open(new_object_index, 'w')
 524
 525     OUTPUT.write('''%s
 526 <informaltable pgwide="1" frame="none">
 527 <tgroup cols="%s">
 528 <colspec colwidth="1*"/>
 529 <colspec colwidth="1*"/>
 530 <colspec colwidth="1*"/>
 531 <tbody>
 532 ''' % (MakeDocHeader("informaltable"), cols))
 533
 534     count = 0
 535     object = None
 536     for object in sorted(Objects):
 537         xref = MakeXRef(object)
 538         if count % cols == 0:
 539             OUTPUT.write("<row>\n")
 540         OUTPUT.write("<entry>%s</entry>\n" % xref)
 541         if count % cols == cols - 1:
 542             OUTPUT.write("</row>\n")
 543         count += 1
 544
 545     if count == 0:
 546         # emit an empty row, since empty tables are invalid
 547         OUTPUT.write("<row><entry> </entry></row>\n")
 548
 549     else:
 550         if count % cols > 0:
 551             OUTPUT.write("</row>\n")
 552
 553     OUTPUT.write('''</tbody></tgroup></informaltable>\n''')
 554     OUTPUT.close()
 555
 556     common.UpdateFileIfChanged(old_object_index, new_object_index, 0)
 557
 558
 559 #############################################################################
 560 # Function    : TrimTextBlock
 561 # Description : Trims extra whitespace. Empty lines inside a block are
 562 #                preserved.
 563 # Arguments   : $desc - the text block to trim. May contain newlines.
 564 #############################################################################
 565
 566 def TrimTextBlock(desc):
 567     # strip leading spaces on the block
 568     desc = re.sub(r'^\s+', '', desc)
 569     # strip trailing spaces on every line
 570     desc = re.sub(r'\s+$', '\n', desc, flags=re.MULTILINE)
 571
 572     return desc
 573
 574
 575 #############################################################################
 576 # Function    : OutputDB
 577 # Description : This collects the output for each section of the docs, and
 578 #                outputs each file when the end of the section is found.
 579 # Arguments   : $file - the $MODULE-sections.txt file which contains all of
 580 #                the functions/macros/structs etc. being documented, organised
 581 #                into sections and subsections.
 582 #############################################################################
 583
 584 def OutputDB(file):
 585
 586     logging.info("Reading: %s", file)
 587     INPUT = open(file)
 588     filename = ''
 589     book_top = ''
 590     book_bottom = ''
 591     includes = DEFAULT_INCLUDES if DEFAULT_INCLUDES else ''
 592     section_includes = ''
 593     in_section = 0
 594     title = ''
 595     section_id = ''
 596     subsection = ''
 597     num_symbols = 0
 598     changed = 0
 599     functions_synop = ''
 600     other_synop = ''
 601     functions_details = ''
 602     other_details = ''
 603     signals_synop = ''
 604     signals_desc = ''
 605     args_synop = ''
 606     child_args_synop = ''
 607     style_args_synop = ''
 608     args_desc = ''
 609     child_args_desc = ''
 610     style_args_desc = ''
 611     hierarchy_str = ''
 612     hierarchy = []
 613     interfaces = ''
 614     implementations = ''
 615     prerequisites = ''
 616     derived = ''
 617     file_objects = []
 618     templates = {}
 619     symbol_def_line = {}
 620
 621     # merge the source docs, in case there are no templates
 622     MergeSourceDocumentation()
 623
 624     line_number = 0
 625     for line in INPUT:
 626         line_number += 1
 627
 628         if line.startswith('#'):
 629             continue
 630
 631         logging.info("section file data: %d: %s", line_number, line)
 632
 633         m1 = re.search(r'^<SUBSECTION\s*(.*)>', line, re.I)
 634         m2 = re.search(r'^<TITLE>(.*)<\/TITLE', line)
 635         m3 = re.search(r'^<FILE>(.*)<\/FILE>', line)
 636         m4 = re.search(r'^<INCLUDE>(.*)<\/INCLUDE>', line)
 637         m5 = re.search(r'^(\S+)', line)
 638
 639         if line.startswith('<SECTION>'):
 640             num_symbols = 0
 641             in_section = False
 642             file_objects = []
 643             symbol_def_line = {}
 644
 645         elif m1:
 646             other_synop += "\n"
 647             functions_synop += "\n"
 648             subsection = m1.group(1)
 649
 650         elif line.startswith('<SUBSECTION>'):
 651             continue
 652         elif m2:
 653             title = m2.group(1)
 654             logging.info("Section: %s", title)
 655
 656             # We don't want warnings if object & class structs aren't used.
 657             DeclarationOutput[title] = 1
 658             DeclarationOutput["%sClass" % title] = 1
 659             DeclarationOutput["%sIface" % title] = 1
 660             DeclarationOutput["%sInterface" % title] = 1
 661
 662         elif m3:
 663             filename = m3.group(1)
 664             if not filename in templates:
 665                 if ReadTemplateFile(os.path.join(TMPL_DIR, filename), 1):
 666                     MergeSourceDocumentation()
 667                     templates[filename] = line_number
 668             else:
 669                 common.LogWarning(file, line_number, "Double <FILE>%s</FILE> entry. Previous occurrence on line %s." % (filename, templates[filename]))
 670             if title == ''  and ("%s/%s:Title" % (TMPL_DIR, filename)) in SourceSymbolDocs:
 671                 title = SourceSymbolDocs["%s/%s:Title" % (TMPL_DIR, filename)]
 672                  # Remove trailing blanks
 673                 title = title.rstrip()
 674
 675         elif m4:
 676             if in_section:
 677                 section_includes = m4.group(1)
 678             else:
 679                 if DEFAULT_INCLUDES:
 680                     common.LogWarning(file, line_number, "Default <INCLUDE> being overridden by command line option.")
 681                 else:
 682                     includes = m4.group(1)
 683
 684         elif re.search(r'^<\/SECTION>', line):
 685             logging.info("End of section: %s", title)
 686             if num_symbols > 0:
 687                 # collect documents
 688                 book_bottom += "    <xi:include href=\"xml/%s.xml\"/>\n" % filename
 689
 690                 i = os.path.join(TMPL_DIR, filename + ":Include")
 691
 692                 if i in SourceSymbolDocs:
 693                     if section_includes:
 694                         common.LogWarning(file, line_number, "Section <INCLUDE> being overridden by inline comments.")
 695                     section_includes = SourceSymbolDocs[i]
 696
 697                 if section_includes == '':
 698                     section_includes = includes
 699
 700                 signals_synop = re.sub(r'^\n*', '', signals_synop)
 701                 signals_synop = re.sub(r'\n+$', '\n', signals_synop)
 702
 703                 if signals_synop != '':
 704                     signals_synop = '''<refsect1 id="%s.signals" role="signal_proto">
 705 <title role="signal_proto.title">Signals</title>
 706 <informaltable frame="none">
 707 <tgroup cols="3">
 708 <colspec colname="signals_return" colwidth="150px"/>
 709 <colspec colname="signals_name" colwidth="300px"/>
 710 <colspec colname="signals_flags" colwidth="200px"/>
 711 <tbody>
 712 %s
 713 </tbody>
 714 </tgroup>
 715 </informaltable>
 716 </refsect1>
 717 ''' % (section_id, signals_synop)
 718                     signals_desc = TrimTextBlock(signals_desc)
 719                     signals_desc = '''<refsect1 id="%s.signal-details" role="signals">
 720 <title role="signals.title">Signal Details</title>
 721 %s
 722 </refsect1>
 723 ''' % (section_id, signals_desc)
 724
 725                 args_synop = re.sub(r'^\n*', '', args_synop)
 726                 args_synop = re.sub(r'\n+$', '\n', args_synop)
 727                 if args_synop != '':
 728                     args_synop = '''<refsect1 id="%s.properties" role="properties">
 729 <title role="properties.title">Properties</title>
 730 <informaltable frame="none">
 731 <tgroup cols="3">
 732 <colspec colname="properties_type" colwidth="150px"/>
 733 <colspec colname="properties_name" colwidth="300px"/>
 734 <colspec colname="properties_flags" colwidth="200px"/>
 735 <tbody>
 736 %s
 737 </tbody>
 738 </tgroup>
 739 </informaltable>
 740 </refsect1>
 741 ''' %(section_id, args_synop)
 742                     args_desc = TrimTextBlock(args_desc)
 743                     args_desc = '''<refsect1 id="%s.property-details" role="property_details">
 744 <title role="property_details.title">Property Details</title>
 745 %s
 746 </refsect1>
 747 ''' % (section_id, args_desc)
 748
 749                 child_args_synop = re.sub(r'^\n*', '', child_args_synop)
 750                 child_args_synop = re.sub(r'\n+$', '\n', child_args_synop)
 751                 if child_args_synop != '':
 752                     args_synop += '''<refsect1 id="%s.child-properties" role="child_properties">
 753 <title role="child_properties.title">Child Properties</title>
 754 <informaltable frame="none">
 755 <tgroup cols="3">
 756 <colspec colname="child_properties_type" colwidth="150px"/>
 757 <colspec colname="child_properties_name" colwidth="300px"/>
 758 <colspec colname="child_properties_flags" colwidth="200px"/>
 759 <tbody>
 760 %s
 761 </tbody>
 762 </tgroup>
 763 </informaltable>
 764 </refsect1>
 765 ''' % (section_id, child_args_synop)
 766                     child_args_desc = TrimTextBlock(child_args_desc)
 767                     args_desc += '''<refsect1 id="%s.child-property-details" role="child_property_details">
 768 <title role="child_property_details.title">Child Property Details</title>
 769 %s
 770 </refsect1>
 771 ''' % (section_id, child_args_desc)
 772
 773                 style_args_synop = re.sub(r'^\n*', '', style_args_synop)
 774                 style_args_synop = re.sub(r'\n+$', '\n', style_args_synop)
 775                 if style_args_synop != '':
 776                     args_synop += '''<refsect1 id="%s.style-properties" role="style_properties">
 777 <title role="style_properties.title">Style Properties</title>
 778 <informaltable frame="none">
 779 <tgroup cols="3">
 780 <colspec colname="style_properties_type" colwidth="150px"/>
 781 <colspec colname="style_properties_name" colwidth="300px"/>
 782 <colspec colname="style_properties_flags" colwidth="200px"/>
 783 <tbody>
 784 %s
 785 </tbody>
 786 </tgroup>
 787 </informaltable>
 788 </refsect1>
 789 ''' % (section_id, style_args_synop)
 790                     style_args_desc = TrimTextBlock(style_args_desc)
 791                     args_desc += '''<refsect1 id="%s.style-property-details" role="style_properties_details">
 792 <title role="style_properties_details.title">Style Property Details</title>
 793 %s
 794 </refsect1>
 795 ''' % (section_id, style_args_desc)
 796
 797                 hierarchy_str = AddTreeLineArt(hierarchy)
 798                 if hierarchy_str != '':
 799                     hierarchy_str = '''<refsect1 id="%s.object-hierarchy" role="object_hierarchy">
 800 <title role="object_hierarchy.title">Object Hierarchy</title>
 801 <screen>%s
 802 </screen>
 803 </refsect1>
 804 ''' % (section_id, hierarchy_str)
 805
 806                 interfaces = TrimTextBlock(interfaces)
 807                 if interfaces != '':
 808                     interfaces = '''<refsect1 id="%s.implemented-interfaces" role="impl_interfaces">
 809 <title role="impl_interfaces.title">Implemented Interfaces</title>
 810 %s
 811 </refsect1>
 812 ''' % (section_id, interfaces)
 813
 814                 implementations = TrimTextBlock(implementations)
 815                 if implementations != '':
 816                     implementations = '''<refsect1 id="%s.implementations" role="implementations">
 817 <title role="implementations.title">Known Implementations</title>
 818 %s
 819 </refsect1>
 820 ''' % (section_id, implementations)
 821
 822                 prerequisites = TrimTextBlock(prerequisites)
 823                 if prerequisites != '':
 824                     prerequisites = '''<refsect1 id="%s.prerequisites" role="prerequisites">
 825 <title role="prerequisites.title">Prerequisites</title>
 826 %s
 827 </refsect1>
 828 ''' % (section_id, prerequisites)
 829
 830                 derived = TrimTextBlock(derived)
 831                 if derived != '':
 832                     derived = '''<refsect1 id="%s.derived-interfaces" role="derived_interfaces">
 833 <title role="derived_interfaces.title">Known Derived Interfaces</title>
 834 %s
 835 </refsect1>
 836 ''' % (section_id, derived)
 837
 838                 functions_synop = re.sub(r'^\n*', '', functions_synop)
 839                 functions_synop = re.sub(r'\n+$', '\n', functions_synop)
 840                 if functions_synop != '':
 841                     functions_synop = '''<refsect1 id="%s.functions" role="functions_proto">
 842 <title role="functions_proto.title">Functions</title>
 843 <informaltable pgwide="1" frame="none">
 844 <tgroup cols="2">
 845 <colspec colname="functions_return" colwidth="150px"/>
 846 <colspec colname="functions_name"/>
 847 <tbody>
 848 %s
 849 </tbody>
 850 </tgroup>
 851 </informaltable>
 852 </refsect1>
 853 ''' % (section_id, functions_synop)
 854
 855                 other_synop = re.sub(r'^\n*', '', other_synop)
 856                 other_synop = re.sub(r'\n+$', '\n', other_synop)
 857                 if other_synop != '':
 858                     other_synop = '''<refsect1 id="%s.other" role="other_proto">
 859 <title role="other_proto.title">Types and Values</title>
 860 <informaltable role="enum_members_table" pgwide="1" frame="none">
 861 <tgroup cols="2">
 862 <colspec colname="name" colwidth="150px"/>
 863 <colspec colname="description"/>
 864 <tbody>
 865 %s
 866 </tbody>
 867 </tgroup>
 868 </informaltable>
 869 </refsect1>
 870 ''' % (section_id, other_synop)
 871
 872                 file_changed = OutputDBFile(filename, title, section_id,
 873                                             section_includes,
 874                                             functions_synop, other_synop,
 875                                             functions_details, other_details,
 876                                             signals_synop, signals_desc,
 877                                             args_synop, args_desc,
 878                                             hierarchy_str, interfaces,
 879                                             implementations,
 880                                             prerequisites, derived,
 881                                             file_objects)
 882                 if file_changed:
 883                     changed = True
 884
 885             title = ''
 886             section_id = ''
 887             subsection = ''
 888             in_section = 0
 889             section_includes = ''
 890             functions_synop = ''
 891             other_synop = ''
 892             functions_details = ''
 893             other_details = ''
 894             signals_synop = ''
 895             signals_desc = ''
 896             args_synop = ''
 897             child_args_synop = ''
 898             style_args_synop = ''
 899             args_desc = ''
 900             child_args_desc = ''
 901             style_args_desc = ''
 902             hierarchy_str = ''
 903             hierarchy = []
 904             interfaces = ''
 905             implementations = ''
 906             prerequisites = ''
 907             derived = ''
 908
 909         elif m5:
 910             symbol = m5.group(1)
 911             logging.info('  Symbol: "%s" in subsection: "%s"', symbol, subsection)
 912
 913             # check for duplicate entries
 914             if symbol not in symbol_def_line:
 915                 declaration = Declarations.get(symbol)
 916                 ## FIXME: with this we'll output empty declaration
 917                 if declaration is not None:
 918                     if CheckIsObject(symbol):
 919                         file_objects.append(symbol)
 920
 921                     # We don't want standard macros/functions of GObjects,
 922                     # or private declarations.
 923                     if subsection != "Standard" and subsection != "Private":
 924                         synop, desc = OutputDeclaration(symbol, declaration)
 925                         type = DeclarationTypes[symbol]
 926
 927                         if type == 'FUNCTION' or type == 'USER_FUNCTION':
 928                             functions_synop += synop
 929                             functions_details += desc
 930                         elif type == 'MACRO' and re.search(symbol + r'\(', declaration):
 931                             functions_synop += synop
 932                             functions_details += desc
 933                         else:
 934                             other_synop += synop
 935                             other_details += desc
 936
 937                     sig_synop, sig_desc = GetSignals(symbol)
 938                     arg_synop, child_arg_synop, style_arg_synop, arg_desc, child_arg_desc, style_arg_desc = GetArgs(symbol)
 939                     ifaces = GetInterfaces(symbol)
 940                     impls = GetImplementations(symbol)
 941                     prereqs = GetPrerequisites(symbol)
 942                     der = GetDerived(symbol)
 943                     hierarchy = GetHierarchy(symbol, hierarchy)
 944
 945                     signals_synop += sig_synop
 946                     signals_desc += sig_desc
 947                     args_synop += arg_synop
 948                     child_args_synop += child_arg_synop
 949                     style_args_synop += style_arg_synop
 950                     args_desc += arg_desc
 951                     child_args_desc += child_arg_desc
 952                     style_args_desc += style_arg_desc
 953                     interfaces += ifaces
 954                     implementations += impls
 955                     prerequisites += prereqs
 956                     derived += der
 957
 958                     # Note that the declaration has been output.
 959                     DeclarationOutput[symbol] = True
 960                 elif subsection != "Standard" and subsection != "Private":
 961                     UndeclaredSymbols[symbol] = True
 962                     common.LogWarning(file, line_number, "No declaration found for %s." % symbol)
 963
 964                 num_symbols += 1
 965                 symbol_def_line[symbol] = line_number
 966
 967                 if section_id == '':
 968                     if title == '' and filename == '':
 969                         common.LogWarning(file, line_number, "Section has no title and no file.")
 970
 971                     # FIXME: one of those would be enough
 972                     # filename should be an internal detail for gtk-doc
 973                     if title == '':
 974                         title = filename
 975                     elif filename == '':
 976                         filename = title
 977
 978                     filename = filename.replace(' ', '_')
 979
 980                     section_id = SourceSymbolDocs.get(os.path.join(TMPL_DIR, filename + ":Section_Id"))
 981                     if section_id and section_id.strip() != '':
 982                         # Remove trailing blanks and use as is
 983                         section_id = section_id.rstrip()
 984                     elif CheckIsObject(title):
 985                         # GObjects use their class name as the ID.
 986                         section_id = common.CreateValidSGMLID(title)
 987                     else:
 988                         section_id = common.CreateValidSGMLID(MODULE + '-' + title)
 989
 990                 SymbolSection[symbol] = title
 991                 SymbolSectionId[symbol] = section_id
 992
 993             else:
 994                 common.LogWarning(file, line_number, "Double symbol entry for %s. "
 995                                   "Previous occurrence on line %d." % (symbol, symbol_def_line[symbol]))
 996     INPUT.close()
 997
 998     OutputMissingDocumentation()
 999     OutputUndeclaredSymbols()
1000     OutputUnusedSymbols()
1001
1002     if OUTPUT_ALL_SYMBOLS:
1003         OutputAllSymbols()
1004
1005     if OUTPUT_SYMBOLS_WITHOUT_SINCE:
1006         OutputSymbolsWithoutSince()
1007
1008     for filename in EXPAND_CONTENT_FILES.split():
1009         file_changed = OutputExtraFile(filename)
1010         if file_changed:
1011             changed = True
1012
1013     OutputBook(book_top, book_bottom)
1014
1015     logging.info("All files created: %d", changed)
1016     return changed
1017
1018 #############################################################################
1019 # Function    : OutputIndex
1020 # Description : This writes an indexlist that can be included into the main-
1021 #               document into an <index> tag.
1022 #############################################################################
1023
1024 def OutputIndex(basename, apiindex):
1025     old_index = os.path.join(DB_OUTPUT_DIR, basename + '.xml')
1026     new_index = os.path.join(DB_OUTPUT_DIR, basename + '.new')
1027     lastletter = " "
1028     divopen = 0
1029     symbol = None
1030     short_symbol = None
1031
1032     OUTPUT = open(new_index, 'w')
1033
1034     OUTPUT.write(MakeDocHeader("indexdiv") + "\n<indexdiv id=\"%s\">\n" % basename)
1035
1036     logging.info("generate %s index (%d entries) with namespace %s", basename, len(apiindex), NAME_SPACE)
1037
1038     # do a case insensitive sort while chopping off the prefix
1039     mapped_keys = [
1040         {
1041             'original': x,
1042             'short': re.sub(r'^' + NAME_SPACE + r'\_?(.*)', r'\1', x.upper(), flags=re.I),
1043         } for x in apiindex.keys()]
1044     sorted_keys = sorted(mapped_keys, key=lambda d: (d['short'], d['original']))
1045
1046     for key in sorted_keys:
1047         symbol = key['original']
1048         short = key['short']
1049         if short != '':
1050             short_symbol = short
1051         else:
1052             short_symbol = symbol
1053
1054         # generate a short symbol description
1055         symbol_desc = ''
1056         symbol_section = ''
1057         symbol_section_id = ''
1058         symbol_type = ''
1059         if symbol in DeclarationTypes:
1060             symbol_type = DeclarationTypes[symbol].lower()
1061
1062         if symbol_type == '':
1063             logging.info("trying symbol %s", symbol)
1064             m1 = re.search(r'(.*)::(.*)', symbol)
1065             m2 = re.search(r'(.*):(.*)', symbol)
1066             if m1:
1067                 oname = m1.group(1)
1068                 osym = m1.group(2)
1069                 logging.info("  trying object signal %s:%s in %d signals", oname, osym, len(SignalNames))
1070                 for name in SignalNames:
1071                     logging.info("    " + name)
1072                     if name == osym:
1073                         symbol_type = "object signal"
1074                         if oname in SymbolSection:
1075                             symbol_section = SymbolSection[oname]
1076                             symbol_section_id = SymbolSectionId[oname]
1077                         break
1078             elif m2:
1079                 oname = m2.group(1)
1080                 osym = m2.group(2)
1081                 logging.info("  trying object property %s::%s in %d properties", oname, osym, len(ArgNames))
1082                 for name in ArgNames:
1083                     logging.info("    " + name)
1084                     if name == osym:
1085                         symbol_type = "object property"
1086                         if oname in SymbolSection:
1087                             symbol_section = SymbolSection[oname]
1088                             symbol_section_id = SymbolSectionId[oname]
1089                         break
1090         else:
1091             if symbol in SymbolSection:
1092                 symbol_section = SymbolSection[symbol]
1093                 symbol_section_id = SymbolSectionId[symbol]
1094
1095         if symbol_type != '':
1096             symbol_desc = ", " + symbol_type
1097             if symbol_section != '':
1098                 symbol_desc += " in <link linkend=\"%s\">%s</link>" % (symbol_section_id, symbol_section)
1099                 #symbol_desc +=" in " + ExpandAbbreviations(symbol, "#symbol_section")
1100
1101         curletter = short_symbol[0].upper()
1102         ixid =  apiindex[symbol]
1103
1104         logging.info("  add symbol %s with %s to index in section '%s' (derived from %s)", symbol, ixid, curletter, short_symbol)
1105
1106         if curletter != lastletter:
1107             lastletter = curletter
1108
1109             if divopen:
1110                 OUTPUT.write("</indexdiv>\n")
1111
1112             OUTPUT.write("<indexdiv><title>%s</title>\n" % curletter)
1113             divopen = True
1114
1115         OUTPUT.write('<indexentry><primaryie linkends="%s"><link linkend="%s">%s</link>%s</primaryie></indexentry>\n' % (ixid, ixid, symbol, symbol_desc))
1116
1117     if divopen:
1118         OUTPUT.write("</indexdiv>\n")
1119
1120     OUTPUT.write("</indexdiv>\n")
1121     OUTPUT.close()
1122
1123     common.UpdateFileIfChanged(old_index, new_index, 0)
1124
1125
1126
1127 #############################################################################
1128 # Function    : OutputIndexFull
1129 # Description : This writes the full api indexlist that can be included into the
1130 #               main document into an <index> tag.
1131 #############################################################################
1132
1133 def OutputIndexFull():
1134     OutputIndex("api-index-full", IndexEntriesFull)
1135
1136
1137 #############################################################################
1138 # Function    : OutputDeprecatedIndex
1139 # Description : This writes the deprecated api indexlist that can be included
1140 #               into the main document into an <index> tag.
1141 #############################################################################
1142
1143 def OutputDeprecatedIndex():
1144     OutputIndex("api-index-deprecated", IndexEntriesDeprecated)
1145
1146
1147 #############################################################################
1148 # Function    : OutputSinceIndexes
1149 # Description : This writes the 'since' api indexlists that can be included into
1150 #               the main document into an <index> tag.
1151 #############################################################################
1152
1153 def OutputSinceIndexes():
1154     sinces = set(Since.values())
1155
1156     for version in sinces:
1157         logging.info("Since : [%s]", version)
1158         index = {x:IndexEntriesSince[x] for x in IndexEntriesSince.keys() if Since[x] == version}
1159
1160         OutputIndex("api-index-" + version, index)
1161
1162
1163 #############################################################################
1164 # Function    : OutputAnnotationGlossary
1165 # Description : This writes a glossary of the used annotation terms into a
1166 #               separate glossary file that can be included into the main
1167 #               document.
1168 #############################################################################
1169
1170 def OutputAnnotationGlossary():
1171     old_glossary = os.path.join(DB_OUTPUT_DIR, "annotation-glossary.xml")
1172     new_glossary = os.path.join(DB_OUTPUT_DIR, "annotation-glossary.new")
1173     lastletter = " "
1174     divopen = False
1175
1176     # if there are no annotations used return
1177     if not AnnotationsUsed:
1178         return
1179
1180     # add acronyms that are referenced from acronym text
1181     rerun = True
1182     while rerun:
1183         rerun = False
1184         for annotation in AnnotationsUsed:
1185             if annotation not in AnnotationDefinition:
1186                 continue
1187             m = re.search(r'<acronym>([\w ]+)<\/acronym>', AnnotationDefinition[annotation])
1188             if m and m.group(1) not in AnnotationsUsed:
1189                 AnnotationsUsed[m.group(1)] = 1
1190                 rerun = True
1191                 break
1192
1193     OUTPUT = open(new_glossary, 'w')
1194
1195     OUTPUT.write('''%s
1196 <glossary id="annotation-glossary">
1197   <title>Annotation Glossary</title>
1198 ''' % MakeDocHeader("glossary"))
1199
1200     for annotation in sorted(AnnotationsUsed.keys(), key=str.lower):
1201         if annotation in AnnotationDefinition:
1202             definition = AnnotationDefinition[annotation]
1203             curletter = annotation[0].upper()
1204
1205             if curletter != lastletter:
1206                 lastletter = curletter
1207
1208                 if divopen:
1209                     OUTPUT.write("</glossdiv>\n")
1210
1211                 OUTPUT.write("<glossdiv><title>%s</title>\n" % curletter)
1212                 divopen = True
1213
1214             OUTPUT.write('''    <glossentry>
1215       <glossterm><anchor id="annotation-glossterm-%s"/>%s</glossterm>
1216       <glossdef>
1217         <para>%s</para>
1218       </glossdef>
1219     </glossentry>
1220 ''' % (annotation, annotation, definition))
1221
1222     if divopen:
1223         OUTPUT.write("</glossdiv>\n")
1224
1225     OUTPUT.write("</glossary>\n")
1226     OUTPUT.close()
1227
1228     common.UpdateFileIfChanged(old_glossary, new_glossary, 0)
1229
1230
1231 #############################################################################
1232 # Function    : ReadKnownSymbols
1233 # Description : This collects the names of non-private symbols from the
1234 #               $MODULE-sections.txt file.
1235 # Arguments   : $file - the $MODULE-sections.txt file which contains all of
1236 #                the functions/macros/structs etc. being documented, organised
1237 #                into sections and subsections.
1238 #############################################################################
1239
1240 def ReadKnownSymbols(file):
1241
1242     subsection = ''
1243
1244     logging.info("Reading: %s", file)
1245     INPUT = open(file)
1246
1247     for line in INPUT:
1248         if line.startswith('#'):
1249             continue
1250
1251         if line.startswith('<SECTION>'):
1252             subsection = ''
1253             continue
1254
1255         m = re.search(r'^<SUBSECTION\s*(.*)>', line, flags=re.I)
1256         if m:
1257             subsection = m.group(1)
1258             continue
1259
1260         if line.startswith('<SUBSECTION>'):
1261             continue
1262
1263         if re.search(r'^<TITLE>(.*)<\/TITLE>', line):
1264             continue
1265
1266         m = re.search(r'^<FILE>(.*)<\/FILE>', line)
1267         if m:
1268             KnownSymbols[os.path.join(TMPL_DIR, m.group(1) + ":Long_Description")] = 1
1269             KnownSymbols[os.path.join(TMPL_DIR, m.group(1) + ":Short_Description")] = 1
1270             continue
1271
1272         m = re.search(r'^<INCLUDE>(.*)<\/INCLUDE>', line)
1273         if m:
1274             continue
1275
1276         m = re.search(r'^<\/SECTION>', line)
1277         if m:
1278             continue
1279
1280         m = re.search(r'^(\S+)', line)
1281         if m:
1282             symbol = m.group(1)
1283             if subsection != "Standard" and subsection != "Private":
1284                 KnownSymbols[symbol] = 1
1285             else:
1286                 KnownSymbols[symbol] = 0
1287     INPUT.close()
1288
1289 #############################################################################
1290 # Function    : OutputDeclaration
1291 # Description : Returns the synopsis and detailed description DocBook
1292 #                describing one function/macro etc.
1293 # Arguments   : $symbol - the name of the function/macro begin described.
1294 #                $declaration - the declaration of the function/macro.
1295 #############################################################################
1296
1297 def OutputDeclaration(symbol, declaration):
1298     dtype = DeclarationTypes[symbol]
1299     if dtype == 'MACRO':
1300         return OutputMacro(symbol, declaration)
1301     elif dtype == 'TYPEDEF':
1302         return OutputTypedef(symbol, declaration)
1303     elif dtype == 'STRUCT':
1304         return OutputStruct(symbol, declaration)
1305     elif dtype == 'ENUM':
1306         return OutputEnum(symbol, declaration)
1307     elif dtype == 'UNION':
1308         return OutputUnion(symbol, declaration)
1309     elif dtype == 'VARIABLE':
1310         return OutputVariable(symbol, declaration)
1311     elif dtype == 'FUNCTION':
1312         return OutputFunction(symbol, declaration, dtype)
1313     elif dtype == 'USER_FUNCTION':
1314         return OutputFunction(symbol, declaration, dtype)
1315     else:
1316         sys.exit("Unknown symbol type " + dtype)
1317
1318
1319 #############################################################################
1320 # Function    : OutputSymbolTraits
1321 # Description : Returns the Since and StabilityLevel paragraphs for a symbol.
1322 # Arguments   : $symbol - the name of the function/macro begin described.
1323 #############################################################################
1324
1325 def OutputSymbolTraits(symbol):
1326     desc = ''
1327
1328     if symbol in Since:
1329         link_id = "api-index-" + Since[symbol]
1330         desc += "<para role=\"since\">Since: <link linkend=\"%s\">%s</link></para>" % (link_id, Since[symbol])
1331
1332     if symbol in StabilityLevel:
1333         stability = StabilityLevel[symbol]
1334         AnnotationsUsed[stability] = True
1335         desc += "<para role=\"stability\">Stability Level: <acronym>%s</acronym></para>" % stability
1336     return desc
1337
1338
1339 #############################################################################
1340 # Function    : Output{Symbol,Section}ExtraLinks
1341 # Description : Returns extralinks for the symbol (if enabled).
1342 # Arguments   : $symbol - the name of the function/macro begin described.
1343 #############################################################################
1344
1345 def uri_escape(text):
1346     if text is None:
1347         return None
1348
1349     # Build a char to hex map
1350     escapes = {}
1351     for i in range(256):
1352         escapes[chr(i)] = "%%%02X" % i
1353
1354     # Default unsafe characters.  RFC 2732 ^(uric - reserved)
1355     def do_escape(char):
1356         return escapes[char]
1357     text = re.sub(r"([^A-Za-z0-9\-_.!~*'()]", do_escape, text)
1358
1359     return text
1360
1361 def OutputSymbolExtraLinks(symbol):
1362     desc = ''
1363
1364     if False: # NEW FEATURE: needs configurability
1365         sstr = uri_escape(symbol)
1366         mstr = uri_escape(MODULE)
1367         desc += '''<ulink role="extralinks" url="http://www.google.com/codesearch?q=%s">code search</ulink>
1368 <ulink role="extralinks" url="http://library.gnome.org/edit?module=%s&amp;symbol=%s">edit documentation</ulink>
1369 ''' % (sstr, mstr, sstr)
1370
1371     return desc
1372
1373
1374 def OutputSectionExtraLinks(symbol, docsymbol):
1375     desc = ''
1376
1377     if False: # NEW FEATURE: needs configurability
1378         sstr = uri_escape(symbol)
1379         mstr = uri_escape(MODULE)
1380         dsstr = uri_escape(docsymbol)
1381         desc += '''<ulink role="extralinks" url="http://www.google.com/codesearch?q=%s">code search</ulink>
1382 <ulink role="extralinks" url="http://library.gnome.org/edit?module=%s&amp;symbol=%s">edit documentation</ulink>
1383 ''' % (sstr, mstr, dsstr)
1384     return desc
1385
1386
1387 #############################################################################
1388 # Function    : OutputMacro
1389 # Description : Returns the synopsis and detailed description of a macro.
1390 # Arguments   : $symbol - the macro.
1391 #                $declaration - the declaration of the macro.
1392 #############################################################################
1393
1394 def OutputMacro(symbol, declaration):
1395     sid = common.CreateValidSGMLID(symbol)
1396     condition = MakeConditionDescription(symbol)
1397     synop = "<row><entry role=\"define_keyword\">#define</entry><entry role=\"function_name\"><link linkend=\"%s\">%s</link>" % (sid, symbol)
1398
1399     fields = common.ParseMacroDeclaration(declaration, CreateValidSGML)
1400     title = symbol
1401     if len(fields) > 0:
1402         title += '()'
1403
1404     desc = '<refsect2 id="%s" role="macro"%s>\n<title>%s</title>\n' % (sid, condition, title)
1405     desc += MakeIndexterms(symbol, sid)
1406     desc += "\n"
1407     desc += OutputSymbolExtraLinks(symbol)
1408
1409     if len(fields) > 0:
1410         synop += "<phrase role=\"c_punctuation\">()</phrase>"
1411
1412     synop += "</entry></row>\n"
1413
1414     # Don't output the macro definition if is is a conditional macro or it
1415     # looks like a function, i.e. starts with "g_" or "_?gnome_", or it is
1416     # longer than 2 lines, otherwise we get lots of complicated macros like
1417     # g_assert.
1418     if symbol not in DeclarationConditional and not symbol.startswith('g_') \
1419         and not re.search(r'^_?gnome_', symbol) and declaration.count('\n') < 2:
1420         decl_out = CreateValidSGML(declaration)
1421         desc += "<programlisting language=\"C\">%s</programlisting>\n" % decl_out
1422     else:
1423         desc += "<programlisting language=\"C\">" + MakeReturnField("#define") + symbol
1424         m = re.search(r'^\s*#\s*define\s+\w+(\([^\)]*\))', declaration)
1425         if m:
1426             args = m.group(1)
1427             pad = ' ' * (RETURN_TYPE_FIELD_WIDTH - len("#define "))
1428             # Align each line so that if should all line up OK.
1429             args = args.replace('\n', '\n' + pad)
1430             desc += CreateValidSGML(args)
1431
1432         desc += "</programlisting>\n"
1433
1434     desc += MakeDeprecationNote(symbol)
1435
1436     parameters = OutputParamDescriptions("MACRO", symbol, fields)
1437
1438     if symbol in SymbolDocs:
1439         symbol_docs = ConvertMarkDown(symbol, SymbolDocs[symbol])
1440         desc += symbol_docs
1441
1442
1443     desc += parameters
1444     desc += OutputSymbolTraits(symbol)
1445     desc += "</refsect2>\n"
1446     return (synop, desc)
1447
1448
1449 #############################################################################
1450 # Function    : OutputTypedef
1451 # Description : Returns the synopsis and detailed description of a typedef.
1452 # Arguments   : $symbol - the typedef.
1453 #                $declaration - the declaration of the typedef,
1454 #                  e.g. 'typedef unsigned int guint;'
1455 #############################################################################
1456
1457 def OutputTypedef(symbol, declaration):
1458     sid = common.CreateValidSGMLID(symbol)
1459     condition = MakeConditionDescription(symbol)
1460     desc = "<refsect2 id=\"%s\" role=\"typedef\"%s>\n<title>%s</title>\n" % (sid, condition, symbol)
1461     synop = "<row><entry role=\"typedef_keyword\">typedef</entry><entry role=\"function_name\"><link linkend=\"%s\">%s</link></entry></row>\n" % (sid, symbol)
1462
1463     desc += MakeIndexterms(symbol, sid)
1464     desc += "\n"
1465     desc += OutputSymbolExtraLinks(symbol)
1466
1467     if  symbol in DeclarationConditional:
1468         decl_out = CreateValidSGML(declaration)
1469         desc += "<programlisting language=\"C\">%s</programlisting>\n" % decl_out
1470
1471
1472     desc += MakeDeprecationNote(symbol)
1473
1474     if symbol in SymbolDocs:
1475         desc += ConvertMarkDown(symbol, SymbolDocs[symbol])
1476
1477     desc += OutputSymbolTraits(symbol)
1478     desc += "</refsect2>\n"
1479     return (synop, desc)
1480
1481
1482
1483 #############################################################################
1484 # Function    : OutputStruct
1485 # Description : Returns the synopsis and detailed description of a struct.
1486 #                We check if it is a object struct, and if so we only output
1487 #                parts of it that are noted as public fields.
1488 #                We also use a different IDs for object structs, since the
1489 #                original ID is used for the entire RefEntry.
1490 # Arguments   : $symbol - the struct.
1491 #                $declaration - the declaration of the struct.
1492 #############################################################################
1493
1494 def OutputStruct(symbol, declaration):
1495
1496     is_gtype = False
1497     default_to_public = True
1498     if CheckIsObject(symbol):
1499         logging.info("Found struct gtype: %s", symbol)
1500         is_gtype = True
1501         default_to_public = ObjectRoots[symbol] == 'GBoxed'
1502
1503     sid = None
1504     condition = None
1505     if is_gtype:
1506         sid = common.CreateValidSGMLID(symbol + "_struct")
1507         condition = MakeConditionDescription(symbol + "_struct")
1508     else:
1509         sid = common.CreateValidSGMLID(symbol)
1510         condition = MakeConditionDescription(symbol)
1511
1512     # Determine if it is a simple struct or it also has a typedef.
1513     has_typedef = False
1514     if symbol in StructHasTypedef or re.search(r'^\s*typedef\s+', declaration):
1515         has_typedef = True
1516
1517     type_output = None
1518     desc = None
1519     if has_typedef:
1520         # For structs with typedefs we just output the struct name.
1521         type_output = ''
1522         desc = "<refsect2 id=\"%s\" role=\"struct\"%s>\n<title>%s</title>\n" % (sid, condition, symbol)
1523     else:
1524         type_output = "struct"
1525         desc = "<refsect2 id=\"%s\" role=\"struct\"%s>\n<title>struct %s</title>\n" % (sid, condition, symbol)
1526
1527     synop = "<row><entry role=\"datatype_keyword\">%s</entry><entry role=\"function_name\"><link linkend=\"%s\">%s</link></entry></row>\n" % (type_output, sid, symbol)
1528
1529     desc += MakeIndexterms(symbol, sid)
1530     desc += "\n"
1531     desc += OutputSymbolExtraLinks(symbol)
1532
1533     # Form a pretty-printed, private-data-removed form of the declaration
1534
1535     decl_out = ''
1536     if re.search(r'^\s*$', declaration):
1537         logging.info("Found opaque struct: %s", symbol)
1538         decl_out = "typedef struct _%s %s;" % (symbol, symbol)
1539     elif re.search(r'^\s*struct\s+\w+\s*;\s*$', declaration):
1540         logging.info("Found opaque struct: %s", symbol)
1541         decl_out = "struct %s;" % symbol
1542     else:
1543         m = re.search(r'^\s*(typedef\s+)?struct\s*\w*\s*(?:\/\*.*\*\/)?\s*{(.*)}\s*\w*\s*;\s*$', declaration, flags=re.S)
1544         if m:
1545             struct_contents = m.group(2)
1546
1547             public = default_to_public
1548             new_declaration = ''
1549
1550             for decl_line  in struct_contents.splitlines():
1551                 logging.info("Struct line: %s", decl_line)
1552                 m2 = re.search(r'/\*\s*<\s*public\s*>\s*\*/', decl_line)
1553                 m3 = re.search(r'/\*\s*<\s*(private|protected)\s*>\s*\*/', decl_line)
1554                 if m2:
1555                     public = True
1556                 elif m3:
1557                     public = False
1558                 elif public:
1559                     new_declaration += decl_line + "\n"
1560
1561             if new_declaration:
1562                 # Strip any blank lines off the ends.
1563                 new_declaration = re.sub(r'^\s*\n', '', new_declaration)
1564                 new_declaration = re.sub(r'\n\s*$', r'\n', new_declaration)
1565
1566                 if has_typedef:
1567                     decl_out = "typedef struct {\n%s} %s;\n" % (new_declaration, symbol)
1568                 else:
1569                     decl_out = "struct %s {\n%s};\n" % (symbol, new_declaration)
1570
1571         else:
1572             common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1573                               "Couldn't parse struct:\n%s" % declaration)
1574
1575         # If we couldn't parse the struct or it was all private, output an
1576         # empty struct declaration.
1577         if decl_out == '':
1578             if has_typedef:
1579                 decl_out = "typedef struct _%s %s;" % (symbol, symbol)
1580             else:
1581                 decl_out = "struct %s;" % symbol
1582
1583     decl_out = CreateValidSGML(decl_out)
1584     desc += "<programlisting language=\"C\">%s</programlisting>\n" % decl_out
1585
1586     desc += MakeDeprecationNote(symbol)
1587
1588     if symbol in SymbolDocs:
1589         desc += ConvertMarkDown(symbol, SymbolDocs[symbol])
1590
1591     # Create a table of fields and descriptions
1592
1593     # FIXME: Inserting &#160's into the produced type declarations here would
1594     #        improve the output in most situations ... except for function
1595     #        members of structs!
1596     def pfunc(*args):
1597         return '<structfield id="%s">%s</structfield>' % (common.CreateValidSGMLID(sid + '.' + args[0]), args[0])
1598     fields = common.ParseStructDeclaration(declaration, not default_to_public, 0, MakeXRef, pfunc)
1599     params = SymbolParams.get(symbol)
1600
1601     # If no parameters are filled in, we don't generate the description
1602     # table, for backwards compatibility.
1603     found = False
1604     if params:
1605         found = next((True for p in params.values() if p.strip() != ''), False)
1606
1607     if found:
1608         field_descrs = params
1609         missing_parameters = ''
1610         unused_parameters = ''
1611         sid = common.CreateValidSGMLID(symbol + ".members")
1612
1613         desc += '''<refsect3 id="%s" role="struct_members">\n<title>Members</title>
1614 <informaltable role="struct_members_table" pgwide="1" frame="none">
1615 <tgroup cols="3">
1616 <colspec colname="struct_members_name" colwidth="300px"/>
1617 <colspec colname="struct_members_description"/>
1618 <colspec colname="struct_members_annotations" colwidth="200px"/>
1619 <tbody>
1620 ''' % sid
1621
1622         for field_name, text in fields.iteritems():
1623             param_annotations = ''
1624
1625             desc += "<row role=\"member\"><entry role=\"struct_member_name\"><para>%s</para></entry>\n" % text
1626             if field_name in field_descrs:
1627                 (field_descr, param_annotations) = ExpandAnnotation(symbol, field_descrs[field_name])
1628                 field_descr = ConvertMarkDown(symbol, field_descr)
1629                 # trim
1630                 field_descr = re.sub(r'^(\s|\n)+', '', field_descr, flags=re.M|re.S)
1631                 field_descr = re.sub(r'(\s|\n)+$', '', field_descr, flags=re.M|re.S)
1632                 desc += "<entry role=\"struct_member_description\">%s</entry>\n<entry role=\"struct_member_annotations\">%s</entry>\n" % (field_descr, param_annotations)
1633                 del field_descrs[field_name]
1634             else:
1635                 common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1636                                   "Field description for %s::%s is missing in source code comment block." % (symbol, field_name))
1637                 if missing_parameters != '':
1638                     missing_parameters += ", " + field_name
1639                 else:
1640                     missing_parameters = field_name
1641
1642                 desc += "<entry /><entry />\n"
1643
1644             desc += "</row>\n"
1645
1646         desc += "</tbody></tgroup></informaltable>\n</refsect3>\n"
1647         for field_name in field_descrs:
1648             # Documenting those standard fields is not required anymore, but
1649             # we don't want to warn if they are documented anyway.
1650             m = re.search(r'(g_iface|parent_instance|parent_class)', field_name)
1651             if m:
1652                 continue
1653
1654             common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1655                               "Field description for %s::%s is not used from source code comment block." % (symbol, field_name))
1656             if unused_parameters != '':
1657                 unused_parameters += ", " + field_name
1658             else:
1659                 unused_parameters = field_name
1660
1661         # remember missing/unused parameters (needed in tmpl-free build)
1662         if missing_parameters != '' and (symbol not in AllIncompleteSymbols):
1663             AllIncompleteSymbols[symbol] = missing_parameters
1664
1665         if unused_parameters != '' and (symbol not in AllUnusedSymbols):
1666             AllUnusedSymbols[symbol] = unused_parameters
1667     else:
1668         if fields:
1669             if symbol not in AllIncompleteSymbols:
1670                 AllIncompleteSymbols[symbol] = "<items>"
1671                 common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1672                                   "Field descriptions for struct %s are missing in source code comment block." % symbol)
1673                 logging.info("Remaining structs fields: " + ':'.join(fields) + "\n")
1674
1675     desc += OutputSymbolTraits(symbol)
1676     desc += "</refsect2>\n"
1677     return (synop, desc)
1678
1679
1680
1681 #############################################################################
1682 # Function    : OutputUnion
1683 # Description : Returns the synopsis and detailed description of a union.
1684 # Arguments   : $symbol - the union.
1685 #                $declaration - the declaration of the union.
1686 #############################################################################
1687
1688 def OutputUnion(symbol, declaration):
1689
1690     is_gtype = False
1691     if CheckIsObject(symbol):
1692         logging.info("Found union gtype: %s", symbol)
1693         is_gtype = True
1694
1695     sid = None
1696     condition = None
1697     if is_gtype:
1698         sid = common.CreateValidSGMLID(symbol + "_union")
1699         condition = MakeConditionDescription(symbol + "_union")
1700     else:
1701         sid = common.CreateValidSGMLID(symbol)
1702         condition = MakeConditionDescription(symbol)
1703
1704     # Determine if it is a simple struct or it also has a typedef.
1705     has_typedef = False
1706     if symbol in StructHasTypedef or re.search(r'^\s*typedef\s+', declaration):
1707         has_typedef = True
1708
1709     type_output = None
1710     desc = None
1711     if has_typedef:
1712         # For unions with typedefs we just output the union name.
1713         type_output = ''
1714         desc = "<refsect2 id=\"%s\" role=\"union\"%s>\n<title>%s</title>\n" % (sid, condition, symbol)
1715     else:
1716         type_output = "union"
1717         desc = "<refsect2 id=\"%s\" role=\"union\"%s>\n<title>union %s</title>\n" % (sid, condition, symbol)
1718
1719     synop = "<row><entry role=\"datatype_keyword\">%s</entry><entry role=\"function_name\"><link linkend=\"%s\">%s</link></entry></row>\n" % (type_output, sid, symbol)
1720
1721     desc += MakeIndexterms(symbol, sid)
1722     desc += "\n"
1723     desc += OutputSymbolExtraLinks(symbol)
1724     desc += MakeDeprecationNote(symbol)
1725
1726     if symbol in SymbolDocs:
1727         desc += ConvertMarkDown(symbol, SymbolDocs[symbol])
1728
1729     # Create a table of fields and descriptions
1730
1731     # FIXME: Inserting &#160's into the produced type declarations here would
1732     #        improve the output in most situations ... except for function
1733     #        members of structs!
1734     def pfunc(*args):
1735         return '<structfield id="%s">%s</structfield>' % (common.CreateValidSGMLID(sid + '.' + args[0]), args[0])
1736     fields = common.ParseStructDeclaration(declaration, 0, 0, MakeXRef, pfunc)
1737     params = SymbolParams.get(symbol)
1738
1739     # If no parameters are filled in, we don't generate the description
1740     # table, for backwards compatibility
1741     found = False
1742     if params:
1743         found = next((True for p in params.values() if p.strip() != ''), False)
1744
1745     logging.debug('Union %s has %d entries, found=%d, has_typedef=%d', symbol, len(fields), found, has_typedef)
1746
1747     if found:
1748         field_descrs = params
1749         missing_parameters = ''
1750         unused_parameters = ''
1751         sid = common.CreateValidSGMLID('%s.members' % symbol)
1752
1753         desc += '''<refsect3 id="%s" role="union_members">\n<title>Members</title>
1754 <informaltable role="union_members_table" pgwide="1" frame="none">
1755 <tgroup cols="3">
1756 <colspec colname="union_members_name" colwidth="300px"/>
1757 <colspec colname="union_members_description"/>
1758 <colspec colname="union_members_annotations" colwidth="200px"/>
1759 <tbody>
1760 ''' % sid
1761
1762         for field_name, text in fields.iteritems():
1763             param_annotations = ''
1764
1765             desc += "<row><entry role=\"union_member_name\"><para>%s</para></entry>\n" % text
1766             if field_name in field_descrs:
1767                 (field_descr, param_annotations) = ExpandAnnotation(symbol, field_descrs[field_name])
1768                 field_descr = ConvertMarkDown(symbol, field_descr)
1769
1770                 # trim
1771                 field_descr = re.sub(r'^(\s|\n)+', '', field_descr, flags=re.M|re.S)
1772                 field_descr = re.sub(r'(\s|\n)+$', '', field_descr, flags=re.M|re.S)
1773                 desc += "<entry role=\"union_member_description\">%s</entry>\n<entry role=\"union_member_annotations\">%s</entry>\n" % (field_descr, param_annotations)
1774                 del field_descrs[field_name]
1775             else:
1776                 common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1777                                   "Field description for %s::%s is missing in source code comment block." % (symbol, field_name))
1778                 if missing_parameters != '':
1779                     missing_parameters += ", " + field_name
1780                 else:
1781                     missing_parameters = field_name
1782
1783                 desc += "<entry /><entry />\n"
1784
1785             desc += "</row>\n"
1786
1787         desc += "</tbody></tgroup></informaltable>\n</refsect3>"
1788         for field_name in field_descrs:
1789             common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1790                               "Field description for %s::%s is not used from source code comment block." % (symbol, field_name))
1791             if unused_parameters != '':
1792                 unused_parameters += ", " + field_name
1793             else:
1794                 unused_parameters = field_name
1795
1796         # remember missing/unused parameters (needed in tmpl-free build)
1797         if missing_parameters != '' and (symbol not in AllIncompleteSymbols):
1798             AllIncompleteSymbols[symbol] = missing_parameters
1799
1800         if unused_parameters != '' and (symbol not in AllUnusedSymbols):
1801             AllUnusedSymbols[symbol] = unused_parameters
1802     else:
1803         if len(fields) > 0:
1804             if symbol not in AllIncompleteSymbols:
1805                 AllIncompleteSymbols[symbol] = "<items>"
1806                 common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1807                                   "Field descriptions for union %s are missing in source code comment block." % symbol)
1808                 logging.info("Remaining union fields: " + ':'.join(fields) + "\n")
1809
1810     desc += OutputSymbolTraits(symbol)
1811     desc += "</refsect2>\n"
1812     return (synop, desc)
1813
1814 #############################################################################
1815 # Function    : OutputEnum
1816 # Description : Returns the synopsis and detailed description of a enum.
1817 # Arguments   : $symbol - the enum.
1818 #                $declaration - the declaration of the enum.
1819 #############################################################################
1820
1821 def OutputEnum(symbol, declaration):
1822     is_gtype = False
1823     if CheckIsObject(symbol):
1824         logging.info("Found enum gtype: %s", symbol)
1825         is_gtype = True
1826
1827     sid = None
1828     condition = None
1829     if is_gtype:
1830         sid = common.CreateValidSGMLID(symbol + "_enum")
1831         condition = MakeConditionDescription(symbol + "_enum")
1832     else:
1833         sid = common.CreateValidSGMLID(symbol)
1834         condition = MakeConditionDescription(symbol)
1835
1836     synop = "<row><entry role=\"datatype_keyword\">enum</entry><entry role=\"function_name\"><link linkend=\"%s\">%s</link></entry></row>\n" % (sid, symbol)
1837     desc = "<refsect2 id=\"%s\" role=\"enum\"%s>\n<title>enum %s</title>\n" % (sid, condition, symbol)
1838
1839     desc += MakeIndexterms(symbol, sid)
1840     desc += "\n"
1841     desc += OutputSymbolExtraLinks(symbol)
1842     desc += MakeDeprecationNote(symbol)
1843
1844     if symbol in SymbolDocs:
1845         desc += ConvertMarkDown(symbol, SymbolDocs[symbol])
1846
1847     # Create a table of fields and descriptions
1848
1849     fields = common.ParseEnumDeclaration(declaration)
1850     params = SymbolParams.get(symbol)
1851
1852     # If nothing at all is documented log a single summary warning at the end.
1853     # Otherwise, warn about each undocumented item.
1854
1855     found = False
1856     if params:
1857         found = next((True for p in params.values() if p.strip() != ''), False)
1858         field_descrs = params
1859     else:
1860         field_descrs = {}
1861
1862     missing_parameters = ''
1863     unused_parameters = ''
1864
1865     sid = common.CreateValidSGMLID("%s.members" % symbol)
1866     desc += '''<refsect3 id="%s" role="enum_members">\n<title>Members</title>
1867 <informaltable role="enum_members_table" pgwide="1" frame="none">
1868 <tgroup cols="3">
1869 <colspec colname="enum_members_name" colwidth="300px"/>
1870 <colspec colname="enum_members_description"/>
1871 <colspec colname="enum_members_annotations" colwidth="200px"/>
1872 <tbody>
1873 ''' % sid
1874
1875     for field_name in fields:
1876         field_descr = field_descrs.get(field_name)
1877         param_annotations = ''
1878
1879         sid = common.CreateValidSGMLID(field_name)
1880         condition = MakeConditionDescription(field_name)
1881         desc += "<row role=\"constant\"><entry role=\"enum_member_name\"><para id=\"%s\">%s</para></entry>\n" % (sid, field_name)
1882         if field_descr:
1883             field_descr, param_annotations = ExpandAnnotation(symbol, field_descr)
1884             field_descr = ConvertMarkDown(symbol, field_descr)
1885             desc += "<entry role=\"enum_member_description\">%s</entry>\n<entry role=\"enum_member_annotations\">%s</entry>\n" % (field_descr, param_annotations)
1886             del field_descrs[field_name]
1887         else:
1888             if found:
1889                 common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1890                                   "Value description for %s::%s is missing in source code comment block." % (symbol, field_name))
1891                 if missing_parameters != '':
1892                     missing_parameters += ", " + field_name
1893                 else:
1894                     missing_parameters = field_name
1895             desc += "<entry /><entry />\n"
1896         desc += "</row>\n"
1897
1898     desc += "</tbody></tgroup></informaltable>\n</refsect3>"
1899     for field_name in field_descrs:
1900         common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1901                           "Value description for %s::%s is not used from source code comment block." % (symbol, field_name))
1902         if unused_parameters != '':
1903             unused_parameters += ", " + field_name
1904         else:
1905             unused_parameters = field_name
1906
1907     # remember missing/unused parameters (needed in tmpl-free build)
1908     if missing_parameters != '' and (symbol not in AllIncompleteSymbols):
1909         AllIncompleteSymbols[symbol] = missing_parameters
1910
1911     if unused_parameters != '' and (symbol not in AllUnusedSymbols):
1912         AllUnusedSymbols[symbol] = unused_parameters
1913
1914     if not found:
1915         if len(fields) > 0:
1916             if symbol not in AllIncompleteSymbols:
1917                 AllIncompleteSymbols[symbol] = "<items>"
1918                 common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1919                                   "Value descriptions for %s are missing in source code comment block." % symbol)
1920
1921     desc += OutputSymbolTraits(symbol)
1922     desc += "</refsect2>\n"
1923     return (synop, desc)
1924
1925
1926
1927 #############################################################################
1928 # Function    : OutputVariable
1929 # Description : Returns the synopsis and detailed description of a variable.
1930 # Arguments   : $symbol - the extern'ed variable.
1931 #                $declaration - the declaration of the variable.
1932 #############################################################################
1933
1934 def OutputVariable(symbol, declaration):
1935     sid = common.CreateValidSGMLID(symbol)
1936     condition = MakeConditionDescription(symbol)
1937
1938     logging.info("ouputing variable: '%s' '%s'", symbol, declaration)
1939
1940     type_output = None
1941     m1 = re.search(r'^\s*extern\s+((const\s+|signed\s+|unsigned\s+|long\s+|short\s+)*\w+)(\s+\*+|\*+|\s)(\s*)(const\s+)*([A-Za-z]\w*)\s*;', declaration)
1942     m2 = re.search(r'\s*((const\s+|signed\s+|unsigned\s+|long\s+|short\s+)*\w+)(\s+\*+|\*+|\s)(\s*)(const\s+)*([A-Za-z]\w*)\s*=', declaration)
1943     if m1:
1944         mod1 = m1.group(1) or ''
1945         ptr = m1.group(3) or ''
1946         space = m1.group(4) or ''
1947         mod2 = m1.group(5) or ''
1948         type_output = "extern %s%s%s%s" % (mod1, ptr, space, mod2)
1949     elif m2:
1950         mod1 = m2.group(1) or ''
1951         ptr = m2.group(3) or ''
1952         space = m2.group(4) or ''
1953         mod2 = m2.group(5) or ''
1954         type_output = '%s%s%s%s' % (mod1, ptr, space, mod2)
1955     else:
1956         type_output = "extern"
1957
1958     synop = "<row><entry role=\"variable_type\">%s</entry><entry role=\"function_name\"><link linkend=\"%s\">%s</link></entry></row>\n" % (type_output, sid, symbol)
1959
1960     desc = "<refsect2 id=\"%s\" role=\"variable\"%s>\n<title>%s</title>\n" % (sid, condition, symbol)
1961
1962     desc += MakeIndexterms(symbol, sid)
1963     desc += "\n"
1964     desc += OutputSymbolExtraLinks(symbol)
1965
1966     decl_out = CreateValidSGML(declaration)
1967     desc += "<programlisting language=\"C\">%s</programlisting>\n" % decl_out
1968
1969     desc += MakeDeprecationNote(symbol)
1970
1971     if symbol in SymbolDocs:
1972         desc += ConvertMarkDown(symbol, SymbolDocs[symbol])
1973
1974     if symbol in SymbolAnnotations:
1975         param_desc = SymbolAnnotations[symbol]
1976         param_annotations = ''
1977         (param_desc, param_annotations) = ExpandAnnotation(symbol, param_desc)
1978         if param_annotations != '':
1979             desc += "\n<para>%s</para>" % param_annotations
1980
1981
1982
1983     desc += OutputSymbolTraits(symbol)
1984     desc += "</refsect2>\n"
1985     return (synop, desc)
1986
1987
1988
1989 #############################################################################
1990 # Function    : OutputFunction
1991 # Description : Returns the synopsis and detailed description of a function.
1992 # Arguments   : $symbol - the function.
1993 #                $declaration - the declaration of the function.
1994 #############################################################################
1995
1996 def OutputFunction(symbol, declaration, symbol_type):
1997     sid = common.CreateValidSGMLID(symbol)
1998     condition = MakeConditionDescription(symbol)
1999
2000     # Take out the return type
2001     #                      $1                                                                                       $2   $3
2002     regex = r'<RETURNS>\s*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|long\s+|short\s+|struct\s+|enum\s+)*)(\w+)(\s*\**\s*(?:const|G_CONST_RETURN)?\s*\**\s*(?:restrict)?\s*)<\/RETURNS>\n'
2003     m = re.search(regex, declaration)
2004     declaration = re.sub(regex, '', declaration)
2005     type_modifier = m.group(1) or ''
2006     type = m.group(2)
2007     pointer = m.group(3)
2008     # Trim trailing spaces as we are going to pad to $RETURN_TYPE_FIELD_WIDTH below anyway
2009     pointer = pointer.rstrip()
2010     xref = MakeXRef(type, tagify(type, "returnvalue"))
2011     start = ''
2012     #if ($symbol_type == 'USER_FUNCTION')
2013     #    $start = "typedef "
2014     #
2015
2016     # We output const rather than G_CONST_RETURN.
2017     type_modifier = re.sub(r'G_CONST_RETURN', 'const', type_modifier)
2018     pointer = re.sub(r'G_CONST_RETURN', 'const', pointer)
2019     pointer = re.sub(r'^\s+', '&#160;', pointer)
2020
2021     ret_type_output = "%s%s%s%s\n" % (start, type_modifier, xref, pointer)
2022
2023     indent_len = len(symbol) + 2
2024     char1 = char2 = char3 = ''
2025     if symbol_type == 'USER_FUNCTION':
2026         indent_len += 3
2027         char1 = "<phrase role=\"c_punctuation\">(</phrase>"
2028         char2 = "*"
2029         char3 = "<phrase role=\"c_punctuation\">)</phrase>"
2030
2031     symbol_output = "%s<link linkend=\"%s\">%s%s</link>%s" % (char1, sid, char2, symbol, char3)
2032     if indent_len < MAX_SYMBOL_FIELD_WIDTH:
2033         symbol_desc_output = "%s%s%s%s " % (char1, char2, symbol, char3)
2034     else:
2035         indent_len = MAX_SYMBOL_FIELD_WIDTH - 8
2036         symbol_desc_output = ('%s%s%s%s\n' % (char1, char2, symbol, char3)) + (' ' * (indent_len - 1))
2037
2038     synop = "<row><entry role=\"function_type\">%s</entry><entry role=\"function_name\">%s&#160;<phrase role=\"c_punctuation\">()</phrase></entry></row>\n" % (ret_type_output, symbol_output)
2039
2040     desc = "<refsect2 id=\"%s\" role=\"function\"%s>\n<title>%s&#160;()</title>\n" % (sid, condition, symbol)
2041
2042     desc += MakeIndexterms(symbol, sid)
2043     desc += "\n"
2044     desc += OutputSymbolExtraLinks(symbol)
2045
2046     desc += "<programlisting language=\"C\">%s%s(" % (ret_type_output, symbol_desc_output)
2047
2048     def tagfun(*args):
2049         return tagify(args[0], "parameter")
2050
2051     fields = common.ParseFunctionDeclaration(declaration, MakeXRef, tagfun)
2052
2053     first = True
2054     for field_name in fields.values():
2055         if first:
2056             desc += field_name
2057             first = False
2058         else:
2059             desc += ",\n" + (' ' * indent_len) + field_name
2060
2061     desc += ");</programlisting>\n"
2062
2063     desc += MakeDeprecationNote(symbol)
2064
2065     if symbol in SymbolDocs:
2066         desc += ConvertMarkDown(symbol, SymbolDocs[symbol])
2067
2068     if symbol in SymbolAnnotations:
2069         param_desc = SymbolAnnotations[symbol]
2070         (param_desc, param_annotations) = ExpandAnnotation(symbol, param_desc)
2071         if param_annotations != '':
2072             desc += "\n<para>%s</para>" % param_annotations
2073
2074     desc += OutputParamDescriptions("FUNCTION", symbol, fields.keys())
2075     desc += OutputSymbolTraits(symbol)
2076     desc += "</refsect2>\n"
2077     return (synop, desc)
2078
2079 #############################################################################
2080 # Function    : OutputParamDescriptions
2081 # Description : Returns the DocBook output describing the parameters of a
2082 #                function, macro or signal handler.
2083 # Arguments   : $symbol_type - 'FUNCTION', 'MACRO' or 'SIGNAL'. Signal
2084 #                  handlers have an implicit user_data parameter last.
2085 #                $symbol - the name of the function/macro being described.
2086 #               @fields - parsed fields from the declaration, used to determine
2087 #                  undocumented/unused entries
2088 #############################################################################
2089
2090 def OutputParamDescriptions(symbol_type, symbol, fields):
2091     output = ''
2092     num_params = 0
2093     field_descrs = None
2094
2095     if fields:
2096         field_descrs = [f for f in fields if f not in ['void', 'Returns']]
2097     else:
2098         field_descrs = []
2099
2100     params = SymbolParams.get(symbol)
2101     logging.info("param_desc(%s, %s) = %s", symbol_type, symbol, str(params))
2102     # This might be an empty dict, but for SIGNALS we append the user_data docs.
2103     # TODO(ensonic): maybe create that docstring in GetSignals()
2104     if params is not None:
2105         returns = ''
2106         params_desc = ''
2107         missing_parameters = ''
2108         unused_parameters = ''
2109
2110         for param_name, param_desc in params.iteritems():
2111             (param_desc, param_annotations) = ExpandAnnotation(symbol, param_desc)
2112             param_desc = ConvertMarkDown(symbol, param_desc)
2113             # trim
2114             param_desc = re.sub(r'^(\s|\n)+', '', param_desc, flags=re.M|re.S)
2115             param_desc = re.sub(r'(\s|\n)+$', '', param_desc, flags=re.M|re.S)
2116             if param_name == "Returns":
2117                 returns = param_desc
2118                 if param_annotations != '':
2119                     returns += "\n<para>%s</para>" % param_annotations
2120
2121                 elif param_name == "void":
2122                     # FIXME: &common.LogWarning()?
2123                     logging.info("!!!! void in params for %s?\n", symbol)
2124             else:
2125                 if fields:
2126                     if param_name not in field_descrs:
2127                         common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
2128                                           "Parameter description for %s::%s is not used from source code comment block." % (symbol, param_name))
2129                         if unused_parameters != '':
2130                             unused_parameters += ", " + param_name
2131                         else:
2132                             unused_parameters = param_name
2133                     else:
2134                         field_descrs.remove(param_name)
2135
2136                 if param_desc != '':
2137                     params_desc += "<row><entry role=\"parameter_name\"><para>%s</para></entry>\n<entry role=\"parameter_description\">%s</entry>\n<entry role=\"parameter_annotations\">%s</entry></row>\n" % (param_name, param_desc, param_annotations)
2138                     num_params += 1
2139
2140         for param_name in field_descrs:
2141             common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
2142                               "Parameter description for %s::%s is missing in source code comment block." % (symbol, param_name))
2143             if missing_parameters != '':
2144                 missing_parameters += ", " + param_name
2145             else:
2146                 missing_parameters = param_name
2147
2148         # Signals have an implicit user_data parameter which we describe.
2149         if symbol_type == "SIGNAL":
2150             params_desc += "<row><entry role=\"parameter_name\"><simpara>user_data</simpara></entry>\n<entry role=\"parameter_description\"><simpara>user data set when the signal handler was connected.</simpara></entry>\n<entry role=\"parameter_annotations\"></entry></row>\n"
2151
2152
2153         # Start a table if we need one.
2154         if params_desc != '':
2155             sid = common.CreateValidSGMLID("%s.parameters" % symbol)
2156
2157             output += '''<refsect3 id="%s" role="parameters">\n<title>Parameters</title>
2158 <informaltable role="parameters_table" pgwide="1" frame="none">
2159 <tgroup cols="3">
2160 <colspec colname="parameters_name" colwidth="150px"/>
2161 <colspec colname="parameters_description"/>
2162 <colspec colname="parameters_annotations" colwidth="200px"/>
2163 <tbody>
2164 ''' % sid
2165             output += params_desc
2166             output += "</tbody></tgroup></informaltable>\n</refsect3>"
2167
2168         # Output the returns info last
2169         if returns != '':
2170             sid = common.CreateValidSGMLID("%s.returns" % symbol)
2171
2172             output += '''<refsect3 id="%s" role=\"returns\">\n<title>Returns</title>
2173 ''' % sid
2174             output += returns
2175             output += "\n</refsect3>"
2176
2177
2178         # remember missing/unused parameters (needed in tmpl-free build)
2179         if missing_parameters != '' and (symbol not in AllIncompleteSymbols):
2180             AllIncompleteSymbols[symbol] = missing_parameters
2181
2182         if unused_parameters != '' and (symbol not in AllUnusedSymbols):
2183             AllUnusedSymbols[symbol] = unused_parameters
2184
2185     if num_params == 0 and fields and field_descrs:
2186         if symbol not in AllIncompleteSymbols:
2187             AllIncompleteSymbols[symbol] = "<parameters>"
2188     return output
2189
2190
2191
2192 #############################################################################
2193 # Function    : ParseStabilityLevel
2194 # Description : Parses a stability level and outputs a warning if it isn't
2195 #               valid.
2196 # Arguments   : $stability - the stability text.
2197 #                $file, $line - context for error message
2198 #                $message - description of where the level is from, to use in
2199 #               any error message.
2200 # Returns     : The parsed stability level string.
2201 #############################################################################
2202
2203 def ParseStabilityLevel(stability, file, line, message):
2204
2205     stability = stability.strip()
2206     sl = stability.lower()
2207
2208     if sl == 'stable':
2209         stability = "Stable"
2210     elif stability == 'unstable':
2211         stability = "Unstable"
2212     elif stability == 'private':
2213         stability = "Private"
2214     else:
2215         common.LogWarning(file, line, "%s is %s." % (message, stability) +\
2216             "It should be one of these: Stable, Unstable, or Private.")
2217     return stability
2218
2219
2220
2221 #############################################################################
2222 # Function    : OutputDBFile
2223 # Description : Outputs the final DocBook file for one section.
2224 # Arguments   : $file - the name of the file.
2225 #               $title - the title from the $MODULE-sections.txt file, which
2226 #                 will be overridden by the title in the template file.
2227 #               $section_id - the id to use for the toplevel tag.
2228 #               $includes - comma-separates list of include files added at top of
2229 #                 synopsis, with '<' '>' around them (if not already enclosed in '').
2230 #               $functions_synop - reference to the DocBook for the Functions Synopsis part.
2231 #               $other_synop - reference to the DocBook for the Types and Values Synopsis part.
2232 #               $functions_details - reference to the DocBook for the Functions Details part.
2233 #               $other_details - reference to the DocBook for the Types and Values Details part.
2234 #               $signal_synop - reference to the DocBook for the Signal Synopsis part
2235 #               $signal_desc - reference to the DocBook for the Signal Description part
2236 #               $args_synop - reference to the DocBook for the Arg Synopsis part
2237 #               $args_desc - reference to the DocBook for the Arg Description part
2238 #               $hierarchy - reference to the DocBook for the Object Hierarchy part
2239 #               $interfaces - reference to the DocBook for the Interfaces part
2240 #               $implementations - reference to the DocBook for the Known Implementations part
2241 #               $prerequisites - reference to the DocBook for the Prerequisites part
2242 #               $derived - reference to the DocBook for the Derived Interfaces part
2243 #               $file_objects - reference to an array of objects in this file
2244 #############################################################################
2245
2246 def OutputDBFile(file, title, section_id, includes, functions_synop, other_synop, functions_details, other_details, signals_synop, signals_desc, args_synop, args_desc, hierarchy, interfaces, implementations, prerequisites, derived, file_objects):
2247
2248     logging.info("Output docbook for file %s with title '%s'", file, title)
2249
2250     # The edited title overrides the one from the sections file.
2251     new_title = SymbolDocs.get(os.path.join(TMPL_DIR, file + ":Title"))
2252     if new_title and not new_title.strip() == '':
2253         title = new_title
2254         logging.info("Found title: %s", title)
2255
2256     short_desc = SymbolDocs.get(os.path.join(TMPL_DIR, file + ":Short_Description"))
2257     if not short_desc or short_desc.strip() == '':
2258         short_desc = ''
2259     else:
2260         # Don't use ConvertMarkDown here for now since we don't want blocks
2261         short_desc = ExpandAbbreviations(title + ":Short_description", short_desc)
2262         logging.info("Found short_desc: %s", short_desc)
2263
2264     long_desc = SymbolDocs.get(os.path.join(TMPL_DIR, file + ":Long_Description"))
2265     if not long_desc or long_desc.strip() == '':
2266         long_desc = ''
2267     else:
2268         long_desc = ConvertMarkDown(title + ":Long_description", long_desc)
2269         logging.info("Found long_desc: %s", long_desc)
2270
2271     see_also = SymbolDocs.get(os.path.join(TMPL_DIR, file + ":See_Also"))
2272     if not see_also or re.search(r'^\s*(<para>)?\s*(</para>)?\s*$', see_also):
2273         see_also = ''
2274     else:
2275         see_also = ConvertMarkDown(title + ":See_Also", see_also)
2276         logging.info("Found see_also: %s", see_also)
2277
2278     if see_also:
2279         see_also = "<refsect1 id=\"%s.see-also\">\n<title>See Also</title>\n%s\n</refsect1>\n" % (section_id, see_also)
2280
2281     stability = SymbolDocs.get(os.path.join(TMPL_DIR, file + ":Stability_Level"))
2282     if not stability or re.search(r'^\s*$', stability):
2283         stability = ''
2284     else:
2285         line_number = GetSymbolSourceLine(os.path.join(TMPL_DIR, file + ":Stability_Level"))
2286         stability = ParseStabilityLevel(stability, file, line_number, "Section stability level")
2287         logging.info("Found stability: %s", stability)
2288
2289     if stability:
2290         AnnotationsUsed[stability] = 1
2291         stability = "<refsect1 id=\"%s.stability-level\">\n<title>Stability Level</title>\n<acronym>%s</acronym>, unless otherwise indicated\n</refsect1>\n" % (section_id, stability)
2292     elif DEFAULT_STABILITY:
2293         AnnotationsUsed[DEFAULT_STABILITY] = 1
2294         stability = "<refsect1 id=\"%s.stability-level\">\n<title>Stability Level</title>\n<acronym>%s</acronym>, unless otherwise indicated\n</refsect1>\n" % (section_id, DEFAULT_STABILITY)
2295
2296     image = SymbolDocs.get(os.path.join(TMPL_DIR, file + ":Image"))
2297     if not image or re.search(r'^\s*$', image):
2298         image = ''
2299     else:
2300         image = image.strip()
2301
2302         format = None
2303
2304         il = image.lower()
2305         if re.search(r'jpe?g$', il):
2306             format = "format='JPEG'"
2307         elif il.endswith('png'):
2308             format = "format='PNG'"
2309         elif il.endswith('svg'):
2310             format = "format='SVG'"
2311         else:
2312             format = ''
2313
2314         image = "  <inlinegraphic fileref='%s' %s/>\n" % (image, format)
2315
2316     include_output = ''
2317     if includes:
2318         include_output += "<refsect1 id=\"%s.includes\"><title>Includes</title><synopsis>" % section_id
2319         for include in includes.split(','):
2320             if re.search(r'^\".+\"$', include):
2321                 include_output += "#include %s\n" % include
2322             else:
2323                 include = re.sub(r'^\s+|\s+$', '', include, flags=re.S)
2324                 include_output += "#include &lt;%s&gt;\n" % include
2325
2326         include_output += "</synopsis></refsect1>\n"
2327
2328     extralinks = OutputSectionExtraLinks(title, "Section:%s" % file)
2329
2330     old_db_file = os.path.join(DB_OUTPUT_DIR, file + '.xml')
2331     new_db_file = os.path.join(DB_OUTPUT_DIR, file + '.xml.new')
2332
2333     OUTPUT = open(new_db_file, 'w')
2334
2335     object_anchors = ''
2336     for fobject in file_objects:
2337         if fobject == section_id:
2338             continue
2339         sid = common.CreateValidSGMLID(fobject)
2340         logging.info("Adding anchor for %s\n", fobject)
2341         object_anchors += "<anchor id=\"%s\"/>" % sid
2342
2343     # Make sure we produce valid docbook
2344     if not functions_details:
2345         functions_details = "<para />"
2346
2347     # We used to output this, but is messes up our common.UpdateFileIfChanged code
2348     # since it changes every day (and it is only used in the man pages):
2349     # "<refentry id="$section_id" revision="$mday $month $year">"
2350
2351     OUTPUT.write(REFENTRY.substitute({
2352             'args_desc': args_desc,
2353             'args_synop': args_synop,
2354             'derived': derived,
2355             'extralinks': extralinks,
2356             'functions_details': functions_details,
2357             'functions_synop': functions_synop,
2358             'header': MakeDocHeader('refentry'),
2359             'hierarchy': hierarchy,
2360             'image': image,
2361             'include_output': include_output,
2362             'interfaces': interfaces,
2363             'implementations': implementations,
2364             'long_desc': long_desc,
2365             'object_anchors': object_anchors,
2366             'other_details': other_details,
2367             'other_synop': other_synop,
2368             'prerequisites': prerequisites,
2369             'section_id': section_id,
2370             'see_also': see_also,
2371             'signals_desc': signals_desc,
2372             'signals_synop': signals_synop,
2373             'short_desc': short_desc,
2374             'stability': stability,
2375             'title': title,
2376             'MODULE': MODULE.upper(),
2377         }))
2378     OUTPUT.close()
2379
2380     return common.UpdateFileIfChanged(old_db_file, new_db_file, 0)
2381
2382
2383 #############################################################################
2384 # Function    : OutputProgramDBFile
2385 # Description : Outputs the final DocBook file for one program.
2386 # Arguments   : $file - the name of the file.
2387 #               $section_id - the id to use for the toplevel tag.
2388 #############################################################################
2389
2390 def OutputProgramDBFile(program, section_id):
2391     logging.info("Output program docbook for %s", program)
2392
2393     short_desc = SourceSymbolDocs.get(os.path.join(TMPL_DIR, program + ":Short_Description"))
2394     if not short_desc or short_desc.strip() == '':
2395         short_desc = ''
2396     else:
2397         # Don't use ConvertMarkDown here for now since we don't want blocks
2398         short_desc = ExpandAbbreviations(program, short_desc)
2399         logging.info("Found short_desc: %s", short_desc)
2400
2401     synopsis = SourceSymbolDocs.get(os.path.join(TMPL_DIR, program + ":Synopsis"))
2402     if synopsis and synopsis.strip() != '':
2403         items = synopsis.split(' ')
2404         for i in range(0, len(items)):
2405             parameter = items[i]
2406             choice = "plain"
2407             rep = ''
2408
2409             # first parameter is the command name
2410             if i == 0:
2411                 synopsis = "<command>%s</command>\n" % parameter
2412                 continue
2413
2414             # square brackets indicate optional parameters, curly brackets
2415             # indicate required parameters ("plain" parameters are also
2416             # mandatory, but do not get extra decoration)
2417             m1 = re.search(r'^\[(.+?)\]$', parameter)
2418             m2 = re.search(r'^\{(.+?)\}$', parameter)
2419             if m1:
2420                 choice = "opt"
2421                 parameter = m1.group(1)
2422             elif m2:
2423                 choice = "req"
2424                 parameter = m2.group(1)
2425
2426             # parameters ending in "..." are repeatable
2427             if parameter.endswith('...'):
2428                 rep = ' rep=\"repeat\"'
2429                 parameter = parameter[:-3]
2430
2431             # italic parameters are replaceable parameters
2432             parameter = re.sub(r'\*(.+?)\*', r'<replaceable>\1</replaceable>', parameter)
2433
2434             synopsis += "<arg choice=\"%s\"%s>" % (choice, rep)
2435             synopsis += parameter
2436             synopsis += "</arg>\n"
2437
2438         logging.info("Found synopsis: %s", synopsis)
2439     else:
2440         synopsis = "<command>%s</command>" % program
2441
2442     long_desc = SourceSymbolDocs.get(os.path.join(TMPL_DIR, program + ":Long_Description"))
2443     if not long_desc or long_desc.strip() == '':
2444         long_desc = ''
2445     else:
2446         long_desc = ConvertMarkDown("%s:Long_description" % program, long_desc)
2447         logging.info("Found long_desc: %s", long_desc)
2448
2449     options = ''
2450     o = os.path.join(TMPL_DIR, program + ":Options")
2451     if o in SourceSymbolDocs:
2452         opts = SourceSymbolDocs[o].split('\t')
2453
2454         logging.info('options: %d, %s', len(opts), str(opts))
2455
2456         options = "<refsect1>\n<title>Options</title>\n<variablelist>\n"
2457         for k in range(0, len(opts), 2):
2458             opt_desc = opts[k+1]
2459
2460             opt_desc = re.sub(r'\*(.+?)\*', r'<replaceable>\1</replaceable>', opt_desc)
2461
2462             options += "<varlistentry>\n<term>"
2463             opt_names = opts[k].split(',')
2464             for i in range(len(opt_names)):
2465                 prefix = ', ' if i > 0 else ''
2466                 # italic parameters are replaceable parameters
2467                 opt_name = re.sub(r'\*(.+?)\*', r'<replaceable>\1</replaceable>', opt_names[i])
2468
2469                 options += "%s<option>%s</option>\n" % (prefix, opt_name)
2470
2471             options += "</term>\n"
2472             options += "<listitem><para>%s</para></listitem>\n" % opt_desc
2473             options += "</varlistentry>\n"
2474
2475         options += "</variablelist></refsect1>\n"
2476
2477     exit_status = SourceSymbolDocs.get(os.path.join(TMPL_DIR, program + ":Returns"))
2478     if exit_status and exit_status != '':
2479         exit_status = ConvertMarkDown("%s:Returns" % program, exit_status)
2480         exit_status = "<refsect1 id=\"%s.exit-status\">\n<title>Exit Status</title>\n%s\n</refsect1>\n" % (section_id, exit_status)
2481     else:
2482         exit_status = ''
2483
2484     see_also = SourceSymbolDocs.get(os.path.join(TMPL_DIR, program + ":See_Also"))
2485     if not see_also or re.search(r'^\s*(<para>)?\s*(</para>)?\s*$', see_also):
2486         see_also = ''
2487     else:
2488         see_also = ConvertMarkDown("%s:See_Also" % program, see_also)
2489         logging.info("Found see_also: %s", see_also)
2490
2491     if see_also:
2492         see_also = "<refsect1 id=\"%s.see-also\">\n<title>See Also</title>\n%s\n</refsect1>\n" % (section_id, see_also)
2493
2494     old_db_file = os.path.join(DB_OUTPUT_DIR, program + ".xml")
2495     new_db_file = os.path.join(DB_OUTPUT_DIR, program + ".xml.new")
2496
2497     OUTPUT = open(new_db_file, 'w')
2498
2499     OUTPUT.write('''%s
2500 <refentry id="%s">
2501 <refmeta>
2502 <refentrytitle role="top_of_page" id="%s.top_of_page">%s</refentrytitle>
2503 <manvolnum>1</manvolnum>
2504 <refmiscinfo>User Commands</refmiscinfo>
2505 </refmeta>
2506 <refnamediv>
2507 <refname>%s</refname>
2508 <refpurpose>%s</refpurpose>
2509 </refnamediv>
2510 <refsynopsisdiv>
2511 <cmdsynopsis>%s</cmdsynopsis>
2512 </refsynopsisdiv>
2513 <refsect1 id="%s.description" role="desc">
2514 <title role="desc.title">Description</title>
2515 %s
2516 </refsect1>
2517 %s%s%s
2518 </refentry>
2519 ''' % (MakeDocHeader("refentry"), section_id, section_id, program, program, short_desc, synopsis, section_id, long_desc, options, exit_status, see_also))
2520     OUTPUT.close()
2521
2522     return common.UpdateFileIfChanged(old_db_file, new_db_file, 0)
2523
2524
2525 #############################################################################
2526 # Function    : OutputExtraFile
2527 # Description : Copies an "extra" DocBook file into the output directory,
2528 #               expanding abbreviations
2529 # Arguments   : $file - the source file.
2530 #############################################################################
2531 def OutputExtraFile(file):
2532
2533     basename = re.sub(r'^.*/', '', file)
2534
2535     old_db_file = os.path.join(DB_OUTPUT_DIR, basename)
2536     new_db_file = os.path.join(DB_OUTPUT_DIR, basename + ".new")
2537
2538     contents = open(file).read()
2539
2540     OUTPUT = open(new_db_file, 'w')
2541
2542     OUTPUT.write(ExpandAbbreviations(basename + " file", contents))
2543     OUTPUT.close()
2544
2545     return common.UpdateFileIfChanged(old_db_file, new_db_file, 0)
2546
2547 #############################################################################
2548 # Function    : OutputBook
2549 # Description : Outputs the entities that need to be included into the
2550 #                main docbook file for the module.
2551 # Arguments   : $book_top - the declarations of the entities, which are added
2552 #                  at the top of the main docbook file.
2553 #                $book_bottom - the references to the entities, which are
2554 #                  added in the main docbook file at the desired position.
2555 #############################################################################
2556
2557 def OutputBook(book_top, book_bottom):
2558
2559     old_file = os.path.join(DB_OUTPUT_DIR, MODULE + "-doc.top")
2560     new_file = os.path.join(DB_OUTPUT_DIR, MODULE + "-doc.top.new")
2561
2562     OUTPUT = open(new_file, 'w')
2563     OUTPUT.write(book_top)
2564     OUTPUT.close()
2565
2566     common.UpdateFileIfChanged(old_file, new_file, 0)
2567
2568
2569     old_file = os.path.join(DB_OUTPUT_DIR, MODULE + "-doc.bottom")
2570     new_file = os.path.join(DB_OUTPUT_DIR, MODULE + "-doc.bottom.new")
2571
2572     OUTPUT = open(new_file, 'w')
2573     OUTPUT.write(book_bottom)
2574     OUTPUT.close()
2575
2576     common.UpdateFileIfChanged(old_file, new_file, 0)
2577
2578
2579     # If the main docbook file hasn't been created yet, we create it here.
2580     # The user can tweak it later.
2581     if MAIN_SGML_FILE and not os.path.exists(MAIN_SGML_FILE):
2582         OUTPUT = open(MAIN_SGML_FILE, 'w')
2583
2584         OUTPUT.write('''%s
2585 <book id="index">
2586   <bookinfo>
2587     <title>&package_name; Reference Manual</title>
2588     <releaseinfo>
2589       for &package_string;.
2590       The latest version of this documentation can be found on-line at
2591       <ulink role="online-location" url="http://[SERVER]/&package_name;/index.html">http://[SERVER]/&package_name;/</ulink>.
2592     </releaseinfo>
2593   </bookinfo>
2594
2595   <chapter>
2596     <title>[Insert title here]</title>
2597     %s
2598   </chapter>
2599 ''' % (MakeDocHeader("book"), book_bottom))
2600         if os.path.exists(OBJECT_TREE_FILE):
2601             OUTPUT.write('''  <chapter id="object-tree">
2602     <title>Object Hierarchy</title>
2603     <xi:include href="xml/tree_index.sgml"/>
2604   </chapter>
2605 ''')
2606         else:
2607             OUTPUT.write('''  <!-- enable this when you use gobject types
2608   <chapter id="object-tree">
2609     <title>Object Hierarchy</title>
2610     <xi:include href="xml/tree_index.sgml"/>
2611   </chapter>
2612   -->
2613 ''')
2614
2615         OUTPUT.write('''  <index id="api-index-full">
2616     <title>API Index</title>
2617     <xi:include href="xml/api-index-full.xml"><xi:fallback /></xi:include>
2618   </index>
2619   <index id="deprecated-api-index" role="deprecated">
2620     <title>Index of deprecated API</title>
2621     <xi:include href="xml/api-index-deprecated.xml"><xi:fallback /></xi:include>
2622   </index>
2623 ''')
2624         if AnnotationsUsed:
2625             OUTPUT.write('''  <xi:include href="xml/annotation-glossary.xml"><xi:fallback /></xi:include>
2626 ''')
2627         else:
2628             OUTPUT.write('''  <!-- enable this when you use gobject introspection annotations
2629   <xi:include href="xml/annotation-glossary.xml"><xi:fallback /></xi:include>
2630   -->
2631 ''')
2632
2633         OUTPUT.write('''</book>
2634 ''')
2635
2636         OUTPUT.close()
2637
2638 #############################################################################
2639 # Function    : CreateValidSGML
2640 # Description : This turns any chars which are used in SGML into entities,
2641 #                e.g. '<' into '&lt;'
2642 # Arguments   : $text - the text to turn into proper SGML.
2643 #############################################################################
2644
2645 def CreateValidSGML(text):
2646     text = re.sub(r'&', r'&amp;', text)        # Do this first, or the others get messed up.
2647     text = re.sub(r'<', r'&lt;', text)
2648     text = re.sub(r'>', r'&gt;', text)
2649     # browsers render single tabs inconsistently
2650     text = re.sub(r'([^\s])\t([^\s])', r'\1&#160;\2', text)
2651     return text
2652
2653
2654 #############################################################################
2655 # Function    : ConvertSGMLChars
2656 # Description : This is used for text in source code comment blocks, to turn
2657 #               chars which are used in SGML into entities, e.g. '<' into
2658 #               '&lt;'. Depending on $INLINE_MARKUP_MODE, this is done
2659 #               unconditionally or only if the character doesn't seem to be
2660 #               part of an SGML construct (tag or entity reference).
2661 # Arguments   : $text - the text to turn into proper SGML.
2662 #############################################################################
2663
2664 def ConvertSGMLChars(symbol, text):
2665
2666     if INLINE_MARKUP_MODE:
2667         # For the XML/SGML mode only convert to entities outside CDATA sections.
2668         return ModifyXMLElements(text, symbol,
2669                                  "<!\\[CDATA\\[|<programlisting[^>]*>",
2670                                  ConvertSGMLCharsEndTag,
2671                                  ConvertSGMLCharsCallback)
2672     # For the simple non-sgml mode, convert to entities everywhere.
2673
2674     # First, convert freestanding & to &amp
2675     text = re.sub(r'&(?![a-zA-Z#]+;)', r'&amp;', text)
2676     text = re.sub(r'<', r'&lt;', text)
2677     # Allow '>' at beginning of string for blockquote markdown
2678     text = re.sub(r'''(?<=[^\w\n"'\/-])>''', r'&gt;', text)
2679
2680     return text
2681
2682 def ConvertSGMLCharsEndTag(t):
2683     if t == '<![CDATA[':
2684         return "]]>"
2685     return "</programlisting>"
2686
2687 def ConvertSGMLCharsCallback(text, symbol, tag):
2688     if re.search(r'^<programlisting', tag):
2689         # We can handle <programlisting> specially here.
2690         return ModifyXMLElements(text, symbol,
2691                                  "<!\\[CDATA\\[",
2692                                  ConvertSGMLCharsEndTag,
2693                                  ConvertSGMLCharsCallback2)
2694     elif tag == '':
2695         # If we're not in CDATA convert to entities.
2696         text = re.sub(r'&(?![a-zA-Z#]+;)', r'&amp;', text)        # Do this first, or the others get messed up.
2697         text = re.sub(r'<(?![a-zA-Z\/!])', r'&lt;', text)
2698         # Allow '>' at beginning of string for blockquote markdown
2699         text = re.sub(r'''(?<=[^\w\n"'\/-])>''', r'&gt;', text)
2700
2701         # Handle "#include <xxxxx>"
2702         text = re.sub(r'#include(\s+)<([^>]+)>', r'#include\1&lt;\2&gt;', text)
2703
2704     return text
2705
2706
2707 def ConvertSGMLCharsCallback2(text, symbol, tag):
2708
2709     # If we're not in CDATA convert to entities.
2710     # We could handle <programlisting> differently, though I'm not sure it helps.
2711     if tag == '':
2712         # replace only if its not a tag
2713         text = re.sub(r'&(?![a-zA-Z#]+;)', r'&amp;', text)        # Do this first, or the others get messed up.
2714         text = re.sub(r'<(?![a-zA-Z\/!])', r'&lt;', text)
2715         text = re.sub(r'''(?<![a-zA-Z0-9"'\/-])>''', r'&gt;', text)
2716         # Handle "#include <xxxxx>"
2717         text = re.sub(r'/#include(\s+)<([^>]+)>', r'#include\1&lt;\2&gt;', text)
2718
2719     return text
2720
2721
2722 #############################################################################
2723 # Function    : ExpandAnnotation
2724 # Description : This turns annotations into acronym tags.
2725 # Arguments   : $symbol - the symbol being documented, for error messages.
2726 #                $text - the text to expand.
2727 #############################################################################
2728 def ExpandAnnotation(symbol, param_desc):
2729     param_annotations = ''
2730
2731     # look for annotations at the start of the comment part
2732     # function level annotations don't end with a colon ':'
2733     m = re.search(r'^\s*\((.*?)\)(:|$)', param_desc)
2734     if m:
2735         param_desc = param_desc[m.end():]
2736
2737         annotations = re.split(r'\)\s*\(', m.group(1))
2738         logging.info("annotations for %s: '%s'\n", symbol, m.group(1))
2739         for annotation in annotations:
2740             # need to search for the longest key-match in %AnnotationDefinition
2741             match_length = 0
2742             match_annotation = ''
2743
2744             for annotationdef in AnnotationDefinition:
2745                 if annotation.startswith(annotationdef):
2746                     if len(annotationdef) > match_length:
2747                         match_length = len(annotationdef)
2748                         match_annotation = annotationdef
2749
2750             annotation_extra = ''
2751             if match_annotation != '':
2752                 m = re.search(match_annotation + r'\s+(.*)', annotation)
2753                 if m:
2754                     annotation_extra = " " + m.group(1)
2755
2756                 AnnotationsUsed[match_annotation] = 1
2757                 param_annotations += "[<acronym>%s</acronym>%s]" % (match_annotation, annotation_extra)
2758             else:
2759                 common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
2760                                   "unknown annotation \"%s\" in documentation for %s." % (annotation, symbol))
2761                 param_annotations += "[%s]" % annotation
2762
2763
2764         param_desc = param_desc.strip()
2765         m = re.search(r'^(.*?)\.*\s*$', param_desc, flags=re.S)
2766         param_desc = m.group(1) + '. '
2767
2768     if param_annotations != '':
2769         param_annotations = "<emphasis role=\"annotation\">%s</emphasis>" % param_annotations
2770
2771     return (param_desc, param_annotations)
2772
2773
2774 #############################################################################
2775 # Function    : ExpandAbbreviations
2776 # Description : This turns the abbreviations function(), macro(), @param,
2777 #                %constant, and #symbol into appropriate DocBook markup.
2778 #               CDATA sections and <programlisting> parts are skipped.
2779 # Arguments   : $symbol - the symbol being documented, for error messages.
2780 #                $text - the text to expand.
2781 #############################################################################
2782
2783 def ExpandAbbreviations(symbol, text):
2784     # Note: This is a fallback and normally done in the markdown parser
2785
2786     # Convert "|[" and "]|" into the start and end of program listing examples.
2787     # Support \[<!-- language="C" --> modifiers
2788     text = re.sub(r'\|\[<!-- language="([^"]+)" -->', r'<informalexample><programlisting language="\1"><![CDATA[', text)
2789     text = re.sub(r'\|\[', r'<informalexample><programlisting><![CDATA[', text)
2790     text = re.sub(r'\]\|', r']]></programlisting></informalexample>', text)
2791
2792     # keep CDATA unmodified, preserve ulink tags (ideally we preseve all tags
2793     # as such)
2794     return ModifyXMLElements(text, symbol,
2795                              "<!\\[CDATA\\[|<ulink[^>]*>|<programlisting[^>]*>|<!DOCTYPE",
2796                              ExpandAbbreviationsEndTag,
2797                              ExpandAbbreviationsCallback)
2798
2799
2800 # Returns the end tag (as a regexp) corresponding to the given start tag.
2801 def ExpandAbbreviationsEndTag(start_tag):
2802     if start_tag == r'<!\[CDATA\[':
2803         return "]]>"
2804     if start_tag == "<!DOCTYPE":
2805         return '>'
2806     m = re.search(r'<(\w+)', start_tag)
2807     return "</%s>" % m.group(1)
2808
2809 # Called inside or outside each CDATA or <programlisting> section.
2810 def ExpandAbbreviationsCallback(text, symbol, tag):
2811
2812     if tag.startswith(r'^<programlisting'):
2813         # Handle any embedded CDATA sections.
2814         return ModifyXMLElements(text, symbol,
2815                                  "<!\\[CDATA\\[",
2816                                  ExpandAbbreviationsEndTag,
2817                                  ExpandAbbreviationsCallback2)
2818     elif tag == '':
2819         # NOTE: this is a fallback. It is normally done by the Markdown parser.
2820
2821         # We are outside any CDATA or <programlisting> sections, so we expand
2822         # any gtk-doc abbreviations.
2823
2824         # Convert '@param()'
2825         # FIXME: we could make those also links ($symbol.$2), but that would be less
2826         # useful as the link target is a few lines up or down
2827         text = re.sub(r'(\A|[^\\])\@(\w+((\.|->)\w+)*)\s*\(\)', r'\1<parameter>\2()<\/parameter>', text)
2828
2829         # Convert 'function()' or 'macro()'.
2830         # if there is abc_*_def() we don't want to make a link to _def()
2831         # FIXME: also handle abc(def(....)) : but that would need to be done recursively :/
2832         def f1(m):
2833             return m.group(1) + MakeXRef(m.group(2), tagify(m.group(2) + "()", "function"))
2834         text = re.sub(r'([^\*.\w])(\w+)\s*\(\)', f1, text)
2835         # handle #Object.func()
2836         text = re.sub(r'(\A|[^\\])#([\w\-:\.]+[\w]+)\s*\(\)', f1, text)
2837
2838         # Convert '@param', but not '\@param'.
2839         text = re.sub(r'(\A|[^\\])\@(\w+((\.|->)\w+)*)', r'\1<parameter>\2<\/parameter>', text)
2840         text = re.sub(r'/\\\@', r'\@', text)
2841
2842         # Convert '%constant', but not '\%constant'.
2843         # Also allow negative numbers, e.g. %-1.
2844         def f2(m):
2845             return m.group(1) + MakeXRef(m.group(2), tagify(m.group(2), "literal"))
2846         text = re.sub(r'(\A|[^\\])\%(-?\w+)', f2, text)
2847         text = re.sub(r'\\\%', r'\%', text)
2848
2849         # Convert '#symbol', but not '\#symbol'.
2850         def f3(m):
2851             return m.group(1) + MakeHashXRef(m.group(2), "type")
2852         text = re.sub(r'(\A|[^\\])#([\w\-:\.]+[\w]+)', f3, text)
2853         text = re.sub(r'\\#', '#', text)
2854
2855     return text
2856
2857
2858 # This is called inside a <programlisting>
2859 def ExpandAbbreviationsCallback2(text, symbol, tag):
2860     if tag == '':
2861         # We are inside a <programlisting> but outside any CDATA sections,
2862         # so we expand any gtk-doc abbreviations.
2863         # FIXME: why is this different from &ExpandAbbreviationsCallback(),
2864         #        why not just call it
2865         text = re.sub(r'#(\w+)', lambda m: '%s;' % MakeHashXRef(m.group(1), ''), text)
2866     elif tag == "<![CDATA[":
2867         # NOTE: this is a fallback. It is normally done by the Markdown parser.
2868         text = ReplaceEntities(text, symbol)
2869
2870
2871     return text
2872
2873
2874 def MakeHashXRef(symbol, tag):
2875     text = symbol
2876
2877     # Check for things like '#include', '#define', and skip them.
2878     if PreProcessorDirectives.get(symbol):
2879         return "#%s" % symbol
2880
2881     # Get rid of special suffixes ('-struct','-enum').
2882     text = re.sub(r'-struct$', '', text)
2883     text = re.sub(r'-enum$', '', text)
2884
2885     # If the symbol is in the form "Object::signal", then change the symbol to
2886     # "Object-signal" and use "signal" as the text.
2887     if '::' in symbol:
2888         o, s = symbol.split('::', 1)
2889         symbol = '%s-%s' % (o, s)
2890         text = '“' + s + '”'
2891
2892     # If the symbol is in the form "Object:property", then change the symbol to
2893     # "Object--property" and use "property" as the text.
2894     if ':' in symbol:
2895         o, p = symbol.split(':', 1)
2896         symbol = '%s--%s' % (o, p)
2897         text = '“' + p + '”'
2898
2899
2900     if tag != '':
2901         text = tagify(text, tag)
2902
2903
2904     return MakeXRef(symbol, text)
2905
2906
2907
2908 #############################################################################
2909 # Function    : ModifyXMLElements
2910 # Description : Looks for given XML element tags within the text, and calls
2911 #               the callback on pieces of text inside & outside those elements.
2912 #               Used for special handling of text inside things like CDATA
2913 #               and <programlisting>.
2914 # Arguments   : $text - the text.
2915 #               $symbol - the symbol currently being documented (only used for
2916 #                      error messages).
2917 #               $start_tag_regexp - the regular expression to match start tags.
2918 #                      e.g. "<!\\[CDATA\\[|<programlisting[^>]*>" to match
2919 #                      CDATA sections or programlisting elements.
2920 #               $end_tag_func - function which is passed the matched start tag
2921 #                      and should return the appropriate end tag string regexp.
2922 #               $callback - callback called with each part of the text. It is
2923 #                      called with a piece of text, the symbol being
2924 #                      documented, and the matched start tag or '' if the text
2925 #                      is outside the XML elements being matched.
2926 #############################################################################
2927 def ModifyXMLElements(text, symbol, start_tag_regexp, end_tag_func, callback):
2928     before_tag = start_tag = end_tag_regexp = end_tag = None
2929     result = ''
2930
2931     logging.debug('symbol: %s text: [%s]', symbol, text)
2932
2933     m = re.search(start_tag_regexp, text, flags=re.S)
2934     while m:
2935         before_tag = text[:m.start()] # Prematch for last successful match string
2936         start_tag = m.group(0)        # Last successful match
2937         text = text[m.end():]         # Postmatch for last successful match string
2938
2939         logging.debug('symbol: %s matched start %s: text: [%s]', symbol, start_tag, text)
2940
2941         result += callback(before_tag, symbol, '')
2942         result += start_tag
2943
2944         # get the matching end-tag for current tag
2945         end_tag_regexp = end_tag_func(start_tag)
2946
2947         m2 = re.search(end_tag_regexp, text, flags=re.S)
2948         if m2:
2949             before_tag = text[:m2.start()]
2950             end_tag = m2.group(0)
2951             text = text[m2.end():]
2952
2953             logging.debug('symbol: %s matched end %s: text: [%s]', symbol, end_tag, text)
2954
2955             result += callback(before_tag, symbol, start_tag)
2956             result += end_tag
2957         else:
2958             common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
2959                               "Can't find tag end: %s in docs for: %s." % (end_tag_regexp, symbol))
2960             # Just assume it is all inside the tag.
2961             result += callback(text, symbol, start_tag)
2962             text = ''
2963         m = re.search(start_tag_regexp, text, flags=re.S)
2964
2965     # Handle any remaining text outside the tags.
2966     result += callback(text, symbol, '')
2967
2968     return result
2969
2970
2971 def noop(*args):
2972     return args[0]
2973
2974
2975 # Adds a tag around some text.
2976 # e.g tagify("Text", "literal") => "<literal>Text</literal>".
2977 def tagify(text, elem):
2978     return '<' + elem + '>' + text + '</' + elem + '>'
2979
2980 #############################################################################
2981 # Function    : MakeDocHeader
2982 # Description : Builds a docbook header for the given tag
2983 # Arguments   : $tag - doctype tag
2984 #############################################################################
2985
2986 def MakeDocHeader(tag):
2987     header = doctype_header
2988     header = re.sub(r'<!DOCTYPE \w+', r'<!DOCTYPE ' + tag, header)
2989
2990     # fix the path for book since this is one level up
2991     if tag == 'book':
2992         header = re.sub(r'<!ENTITY % gtkdocentities SYSTEM "../([a-zA-Z./]+)">', r'<!ENTITY % gtkdocentities SYSTEM "\1">', header)
2993     return header
2994
2995
2996
2997 #############################################################################
2998 # Function    : MakeXRef
2999 # Description : This returns a cross-reference link to the given symbol.
3000 #                Though it doesn't try to do this for a few standard C types
3001 #                that it        knows won't be in the documentation.
3002 # Arguments   : $symbol - the symbol to try to create a XRef to.
3003 #               $text - text text to put inside the XRef, defaults to $symbol
3004 #############################################################################
3005
3006 def MakeXRef(symbol, text=None):
3007     symbol = symbol.strip()
3008
3009     if not text:
3010         text = symbol
3011
3012         # Get rid of special suffixes ('-struct','-enum').
3013         text = re.sub(r'-struct$', '', text)
3014         text = re.sub(r'-enum$', '', text)
3015
3016     if ' ' in symbol:
3017         return text
3018
3019     logging.info("Getting type link for %s -> %s", symbol, text)
3020
3021     symbol_id = common.CreateValidSGMLID(symbol)
3022     return "<link linkend=\"%s\">%s</link>" % (symbol_id, text)
3023
3024 #############################################################################
3025 # Function    : MakeIndexterms
3026 # Description : This returns a indexterm elements for the given symbol
3027 # Arguments   : $symbol - the symbol to create indexterms for
3028 #############################################################################
3029
3030 def MakeIndexterms(symbol, sid):
3031     terms = ''
3032     sortas = ''
3033
3034     # make the index useful, by ommiting the namespace when sorting
3035     if NAME_SPACE != '':
3036         m = re.search(r'^%s\_?(.*)' % NAME_SPACE, symbol, flags=re.I)
3037         if m:
3038             sortas = ' sortas="%s"' % m.group(1)
3039
3040     if symbol in Deprecated:
3041         terms += "<indexterm zone=\"%s\" role=\"deprecated\"><primary%s>%s</primary></indexterm>" % (sid, sortas, symbol)
3042         IndexEntriesDeprecated[symbol] = sid
3043         IndexEntriesFull[symbol] = sid
3044     if symbol in Since:
3045         since = Since[symbol].strip()
3046         if since != '':
3047             terms += "<indexterm zone=\"%s\" role=\"%s\"><primary%s>%s</primary></indexterm>" % (sid, since, sortas, symbol)
3048         IndexEntriesSince[symbol] = sid
3049         IndexEntriesFull[symbol] = sid
3050     if terms == '':
3051         terms += "<indexterm zone=\"%s\"><primary%s>%s</primary></indexterm>" % (sid, sortas, symbol)
3052         IndexEntriesFull[symbol] = sid
3053     return terms
3054
3055
3056 #############################################################################
3057 # Function    : MakeDeprecationNote
3058 # Description : This returns a deprecation warning for the given symbol.
3059 # Arguments   : $symbol - the symbol to try to create a warning for.
3060 #############################################################################
3061
3062 def MakeDeprecationNote(symbol):
3063     desc = ''
3064     if symbol in Deprecated:
3065         desc += "<warning><para><literal>%s</literal> " % symbol
3066
3067         note = Deprecated[symbol]
3068
3069         m = re.search(r'^\s*([0-9\.]+)\s*:?', note)
3070         if m:
3071             desc += "has been deprecated since version %s and should not be used in newly-written code.</para>" % m.group(1)
3072         else:
3073             desc += "is deprecated and should not be used in newly-written code.</para>"
3074
3075         note = re.sub(r'^\s*([0-9\.]+)\s*:?\s*', '', note)
3076         note = note.strip()
3077
3078         if note != '':
3079             note = ConvertMarkDown(symbol, note)
3080             desc += " " + note
3081
3082         desc += "</warning>\n"
3083
3084     return desc
3085
3086
3087 #############################################################################
3088 # Function    : MakeConditionDescription
3089 # Description : This returns a sumary of conditions for the given symbol.
3090 # Arguments   : $symbol - the symbol to try to create the sumary.
3091 #############################################################################
3092
3093 def MakeConditionDescription(symbol):
3094     desc = ''
3095     if symbol in Deprecated:
3096         if desc != '':
3097             desc += "|"
3098         m = re.search(r'^\s*(.*?)\s*$', Deprecated[symbol])
3099         if m:
3100             desc += "deprecated:%s" % m.group(1)
3101         else:
3102             desc += "deprecated"
3103
3104     if symbol in Since:
3105         if desc != '':
3106             desc += "|"
3107         m = re.search(r'^\s*(.*?)\s*$', Since[symbol])
3108         if m:
3109             desc += "since:%s" % m.group(1)
3110         else:
3111             desc += "since"
3112
3113     if symbol in StabilityLevel:
3114         if desc != '':
3115             desc += "|"
3116
3117         desc += "stability:" + StabilityLevel[symbol]
3118
3119     if desc != '':
3120         cond = re.sub(r'"', r'&quot;', desc)
3121         desc = ' condition=\"%s\"' % cond
3122         logging.info("condition for '%s' = '%s'", symbol, desc)
3123
3124     return desc
3125
3126
3127 #############################################################################
3128 # Function    : GetHierarchy
3129 # Description : Returns the DocBook output describing the ancestors and
3130 #               immediate children of a GObject subclass. It uses the
3131 #               global @Objects and @ObjectLevels arrays to walk the tree.
3132 #
3133 # Arguments   : $object - the GtkObject subclass.
3134 #               @hierarchy - previous hierarchy
3135 #############################################################################
3136
3137 def GetHierarchy(gobject, hierarchy):
3138     # Find object in the objects array.
3139     found = False
3140     children = []
3141     level = 0
3142     j = 0
3143     for i in range(len(Objects)):
3144         if found:
3145             if ObjectLevels[i] <= level:
3146                 break
3147
3148             elif ObjectLevels[i] == level + 1:
3149                 children.append(Objects[i])
3150
3151         elif Objects[i] == gobject:
3152             found = True
3153             j = i
3154             level = ObjectLevels[i]
3155
3156     if not found:
3157         return hierarchy
3158
3159     logging.info("=== Hierachy for: %s (%d existing entries) ===", gobject, len(hierarchy))
3160
3161     # Walk up the hierarchy, pushing ancestors onto the ancestors array.
3162     ancestors = [gobject]
3163     logging.info("Level: %s", level)
3164     while level > 1:
3165         j -= 1
3166         if ObjectLevels[j] < level:
3167             ancestors.append(Objects[j])
3168             level = ObjectLevels[j]
3169             logging.info("Level: %s", level)
3170
3171     # Output the ancestors, indented and with links.
3172     logging.info('%d ancestors', len(ancestors))
3173     last_index = 0
3174     level = 1
3175     for i in range(len(ancestors) - 1, -1, -1):
3176         ancestor = ancestors[i]
3177         ancestor_id = common.CreateValidSGMLID(ancestor)
3178         indent = ' ' * (level * 4)
3179         # Don't add a link to the current object, i.e. when i == 0.
3180         if i > 0:
3181             entry_text = indent + "<link linkend=\"%s\">%s</link>" % (ancestor_id, ancestor)
3182             alt_text = indent + ancestor
3183         else:
3184             entry_text = indent + ancestor
3185             alt_text = indent + "<link linkend=\"%s\">%s</link>" % (ancestor_id, ancestor)
3186
3187         logging.info("Checking for '%s' or '%s'", entry_text, alt_text)
3188         # Check if we already have this object
3189         index = -1
3190         for j in range(len(hierarchy)):
3191             if hierarchy[j] == entry_text or (hierarchy[j] == alt_text):
3192                 index = j
3193                 break
3194         if index == -1:
3195             # We have a new entry, find insert position in alphabetical order
3196             found = False
3197             for j in range(last_index, len(hierarchy)):
3198                 if not re.search(r'^' + indent, hierarchy[j]):
3199                     last_index = j
3200                     found = True
3201                     break
3202                 elif re.search(r'^%s[^ ]' % indent, hierarchy[j]):
3203                     stripped_text = hierarchy[j]
3204                     if r'<link linkend' not in entry_text:
3205                         stripped_text = re.sub(r'<link linkend="[A-Za-z]*">', '', stripped_text)
3206                         stripped_text = re.sub(r'</link>', '', stripped_text)
3207
3208                     if entry_text < stripped_text:
3209                         last_index = j
3210                         found = True
3211                         break
3212
3213             # Append to bottom
3214             if not found:
3215                 last_index = len(hierarchy)
3216
3217             logging.debug('insert at %d: %s', last_index, entry_text)
3218             hierarchy.insert(last_index, entry_text)
3219             last_index += 1
3220         else:
3221             # Already have this one, make sure we use the not linked version
3222             if r'<link linkend' not in entry_text:
3223                 hierarchy[j] = entry_text
3224
3225             # Remember index as base insert point
3226             last_index = index + 1
3227
3228         level += 1
3229
3230     # Output the children, indented and with links.
3231     logging.info('%d children', len(children))
3232     for i in range(len(children)):
3233         sid = common.CreateValidSGMLID(children[i])
3234         indented_text = ' ' * (level * 4) + "<link linkend=\"%s\">%s</link>" % (sid, children[i])
3235         logging.debug('insert at %d: %s', last_index, indented_text)
3236         hierarchy.insert(last_index, indented_text)
3237         last_index += 1
3238     return hierarchy
3239
3240
3241 #############################################################################
3242 # Function    : GetInterfaces
3243 # Description : Returns the DocBook output describing the interfaces
3244 #               implemented by a class. It uses the global %Interfaces hash.
3245 # Arguments   : $object - the GtkObject subclass.
3246 #############################################################################
3247
3248 def GetInterfaces(gobject):
3249     text = ''
3250
3251     # Find object in the objects array.
3252     if gobject in Interfaces:
3253         ifaces = Interfaces[gobject].split()
3254         text = '''<para>
3255 %s implements
3256 ''' % gobject
3257         count = len(ifaces)
3258         for i in range(count):
3259             sid = common.CreateValidSGMLID(ifaces[i])
3260             text += " <link linkend=\"%s\">%s</link>" % (sid, ifaces[i])
3261             if i < count - 2:
3262                 text += ', '
3263             elif i < count - 1:
3264                 text += ' and '
3265             else:
3266                 text += '.'
3267         text += '</para>\n'
3268     return text
3269
3270
3271 #############################################################################
3272 # Function    : GetImplementations
3273 # Description : Returns the DocBook output describing the implementations
3274 #               of an interface. It uses the global %Interfaces hash.
3275 # Arguments   : $object - the GtkObject subclass.
3276 #############################################################################
3277
3278 def GetImplementations(gobject):
3279     text = ''
3280
3281     impls = []
3282     for key in Interfaces:
3283         if re.search(r'\b%s\b' % gobject, Interfaces[key]):
3284             impls.append(key)
3285
3286     count = len(impls)
3287     if count > 0:
3288         impls.sort()
3289         text = '''<para>
3290 %s is implemented by
3291 ''' % gobject
3292         for i in range(count):
3293             sid = common.CreateValidSGMLID(impls[i])
3294             text += " <link linkend=\"%s\">%s</link>" % (sid, impls[i])
3295             if i < count - 2:
3296                 text += ', '
3297             elif i < count - 1:
3298                 text += ' and '
3299             else:
3300                 text += '.'
3301         text += '</para>\n'
3302     return text
3303
3304
3305 #############################################################################
3306 # Function    : GetPrerequisites
3307 # Description : Returns the DocBook output describing the prerequisites
3308 #               of an interface. It uses the global %Prerequisites hash.
3309 # Arguments   : $iface - the interface.
3310 #############################################################################
3311
3312 def GetPrerequisites(iface):
3313     text = ''
3314
3315     if iface in Prerequisites:
3316         text = '''<para>
3317 %s requires
3318 ''' % iface
3319         prereqs = Prerequisites[iface].split()
3320         count = len(prereqs)
3321         for i in range(count):
3322             sid = common.CreateValidSGMLID(prereqs[i])
3323             text += " <link linkend=\"%s\">%s</link>" % (sid, prereqs[i])
3324             if i < count - 2:
3325                 text += ', '
3326             elif i < count - 1:
3327                 text += ' and '
3328             else:
3329                 text += '.'
3330         text += '</para>\n'
3331     return text
3332
3333
3334 #############################################################################
3335 # Function    : GetDerived
3336 # Description : Returns the DocBook output describing the derived interfaces
3337 #               of an interface. It uses the global %Prerequisites hash.
3338 # Arguments   : $iface - the interface.
3339 #############################################################################
3340
3341 def GetDerived(iface):
3342     text = ''
3343
3344     derived = []
3345     for key in Prerequisites:
3346         if re.search(r'\b%s\b' % iface, Prerequisites[key]):
3347             derived.append(key)
3348
3349     count = len(derived)
3350     if count > 0:
3351         derived.sort()
3352         text = '''<para>
3353 %s is required by
3354 ''' % iface
3355         for i in range(count):
3356             sid = common.CreateValidSGMLID(derived[i])
3357             text += " <link linkend=\"%s\">%s</link>" % (sid, derived[i])
3358             if i < count - 2:
3359                 text += ', '
3360             elif i < count - 1:
3361                 text += ' and '
3362             else:
3363                 text += '.'
3364         text += '</para>\n'
3365     return text
3366
3367
3368
3369 #############################################################################
3370 # Function    : GetSignals
3371 # Description : Returns the synopsis and detailed description DocBook output
3372 #                for the signal handlers of a given GtkObject subclass.
3373 # Arguments   : $object - the GtkObject subclass, e.g. 'GtkButton'.
3374 #############################################################################
3375
3376 def GetSignals(gobject):
3377     synop = ''
3378     desc = ''
3379
3380     for i in range(len(SignalObjects)):
3381         if SignalObjects[i] == gobject:
3382             logging.info("Found signal: %s", SignalNames[i])
3383             name = SignalNames[i]
3384             symbol = '%s::%s' % (gobject, name)
3385             sid = common.CreateValidSGMLID('%s-%s' % (gobject, name))
3386
3387             desc += "<refsect2 id=\"%s\" role=\"signal\"><title>The <literal>“%s”</literal> signal</title>\n" % (sid, name)
3388             desc += MakeIndexterms(symbol, sid)
3389             desc += "\n"
3390             desc += OutputSymbolExtraLinks(symbol)
3391
3392             desc += "<programlisting language=\"C\">"
3393
3394             m = re.search(r'\s*(const\s+)?(\w+)\s*(\**)', SignalReturns[i])
3395             type_modifier = m.group(1) or ''
3396             gtype = m.group(2)
3397             pointer = m.group(3)
3398             xref = MakeXRef(gtype, tagify(gtype, "returnvalue"))
3399
3400             ret_type_output = '%s%s%s' % (type_modifier, xref, pointer)
3401             callback_name = "user_function"
3402             desc += '%s\n%s (' % (ret_type_output, callback_name)
3403
3404             indentation = ' ' * (len(callback_name) + 2)
3405
3406             sourceparams = SourceSymbolParams.get(symbol)
3407             params = SignalPrototypes[i].splitlines()
3408             type_len = len("gpointer")
3409             name_len = len("user_data")
3410             # do two passes, the first one is to calculate padding
3411             for l in range(2):
3412                 for j in range(len(params)):
3413                     param_name = None
3414                     # allow alphanumerics, '_', '[' & ']' in param names
3415                     m = re.search(r'^\s*(\w+)\s*(\**)\s*([\w\[\]]+)\s*$', params[j])
3416                     if m:
3417                         gtype = m.group(1)
3418                         pointer = m.group(2)
3419                         if sourceparams:
3420                             param_name = sourceparams.keys()[j]
3421                             logging.info('from sourceparams: "%s" (%d: %s)', param_name, j, params[j])
3422                         else:
3423                             param_name = m.group(3)
3424                             logging.info('from params: "%s" (%d: %s)', param_name, j, params[j])
3425
3426                         if not param_name:
3427                             param_name = "arg%d" % j
3428
3429                         if l == 0:
3430                             if len(gtype) + len(pointer) > type_len:
3431                                 type_len = len(gtype) + len(pointer)
3432                             if len(param_name) > name_len:
3433                                 name_len = len(param_name)
3434                         else:
3435                             logging.info("signal arg[%d]: '%s'", j, param_name)
3436                             xref = MakeXRef(gtype, tagify(gtype, "type"))
3437                             pad = ' ' * (type_len - len(gtype) - len(pointer))
3438                             desc += '%s%s %s%s,\n' % (xref, pad, pointer, param_name)
3439                             desc += indentation
3440
3441                     else:
3442                         common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
3443                                           "Can't parse arg: %s\nArgs:%s" % (params[j], SignalPrototypes[i]))
3444
3445             xref = MakeXRef("gpointer", tagify("gpointer", "type"))
3446             pad = ' ' * (type_len - len("gpointer"))
3447             desc += '%s%s user_data)' % (xref, pad)
3448             desc += "</programlisting>\n"
3449
3450             flags = SignalFlags[i]
3451             flags_string = ''
3452             if flags:
3453                 if 'f' in flags:
3454                     flags_string = "<link linkend=\"G-SIGNAL-RUN-FIRST:CAPS\">Run First</link>"
3455
3456                 elif 'l' in flags:
3457                     flags_string = "<link linkend=\"G-SIGNAL-RUN-LAST:CAPS\">Run Last</link>"
3458
3459                 elif 'c' in flags:
3460                     flags_string = "<link linkend=\"G-SIGNAL-RUN-CLEANUP:CAPS\">Cleanup</link>"
3461                     flags_string = "Cleanup"
3462
3463                 if 'r' in flags:
3464                     if flags_string:
3465                         flags_string += " / "
3466                     flags_string = "<link linkend=\"G-SIGNAL-NO-RECURSE:CAPS\">No Recursion</link>"
3467
3468                 if 'd' in flags:
3469                     if flags_string:
3470                         flags_string += " / "
3471                     flags_string = "<link linkend=\"G-SIGNAL-DETAILED:CAPS\">Has Details</link>"
3472
3473                 if 'a' in flags:
3474                     if flags_string:
3475                         flags_string += " / "
3476                     flags_string = "<link linkend=\"G-SIGNAL-ACTION:CAPS\">Action</link>"
3477
3478                 if 'h' in flags:
3479                     if flags_string:
3480                         flags_string += " / "
3481                     flags_string = "<link linkend=\"G-SIGNAL-NO-HOOKS:CAPS\">No Hooks</link>"
3482
3483             synop += "<row><entry role=\"signal_type\">%s</entry><entry role=\"signal_name\"><link linkend=\"%s\">%s</link></entry><entry role=\"signal_flags\">%s</entry></row>\n" % (ret_type_output, sid, name, flags_string)
3484
3485             parameters = OutputParamDescriptions("SIGNAL", symbol, None)
3486             logging.info("formatted signal params: '%s' -> '%s'", symbol, parameters)
3487
3488             AllSymbols[symbol] = 1
3489             if symbol in SymbolDocs:
3490                 symbol_docs = ConvertMarkDown(symbol, SymbolDocs[symbol])
3491
3492                 desc += symbol_docs
3493
3494                 if not IsEmptyDoc(SymbolDocs[symbol]):
3495                     AllDocumentedSymbols[symbol] = 1
3496
3497             if symbol in SymbolAnnotations:
3498                 param_desc = SymbolAnnotations[symbol]
3499                 (param_desc, param_annotations) = ExpandAnnotation(symbol, param_desc)
3500                 if param_annotations != '':
3501                     desc += "\n<para>%s</para>" % param_annotations
3502
3503             desc += MakeDeprecationNote(symbol)
3504
3505             desc += parameters
3506             if flags_string:
3507                 desc += "<para>Flags: %s</para>\n" % flags_string
3508
3509             desc += OutputSymbolTraits(symbol)
3510             desc += "</refsect2>"
3511
3512     return (synop, desc)
3513
3514 #############################################################################
3515 # Function    : GetArgs
3516 # Description : Returns the synopsis and detailed description DocBook output
3517 #                for the Args of a given GtkObject subclass.
3518 # Arguments   : $object - the GtkObject subclass, e.g. 'GtkButton'.
3519 #############################################################################
3520
3521 def GetArgs(gobject):
3522     synop = ''
3523     desc = ''
3524     child_synop = ''
3525     child_desc = ''
3526     style_synop = ''
3527     style_desc = ''
3528
3529     for i in range(len(ArgObjects)):
3530         if ArgObjects[i] == gobject:
3531             logging.info("Found arg: %s", ArgNames[i])
3532             name = ArgNames[i]
3533             flags = ArgFlags[i]
3534             flags_string = ''
3535             kind = ''
3536             id_sep = ''
3537
3538             if 'c' in flags:
3539                 kind = "child property"
3540                 id_sep = "c-"
3541             elif 's' in flags:
3542                 kind = "style property"
3543                 id_sep = "s-"
3544             else:
3545                 kind = "property"
3546
3547
3548             # Remember only one colon so we don't clash with signals.
3549             symbol = '%s:%s' % (gobject, name)
3550             # use two dashes and ev. an extra separator here for the same reason.
3551             sid = common.CreateValidSGMLID('%s--%s%s' % (gobject, id_sep, name))
3552
3553             atype = ArgTypes[i]
3554             type_output = None
3555             arange = ArgRanges[i]
3556             range_output = CreateValidSGML(arange)
3557             default = ArgDefaults[i]
3558             default_output = CreateValidSGML(default)
3559
3560             if atype == "GtkString":
3561                 atype = "char&#160;*"
3562
3563             if atype == "GtkSignal":
3564                 atype = "GtkSignalFunc, gpointer"
3565                 type_output = MakeXRef("GtkSignalFunc") + ", " + MakeXRef("gpointer")
3566             elif re.search(r'^(\w+)\*$', atype):
3567                 m = re.search(r'^(\w+)\*$', atype)
3568                 type_output = MakeXRef(m.group(1), tagify(m.group(1), "type")) + "&#160;*"
3569             else:
3570                 type_output = MakeXRef(atype, tagify(atype, "type"))
3571
3572             if 'r' in flags:
3573                 flags_string = "Read"
3574
3575             if 'w' in flags:
3576                 if flags_string:
3577                     flags_string += " / "
3578                 flags_string += "Write"
3579
3580             if 'x' in flags:
3581                 if flags_string:
3582                     flags_string += " / "
3583                 flags_string += "Construct"
3584
3585             if 'X' in flags:
3586                 if flags_string:
3587                     flags_string += " / "
3588                 flags_string += "Construct Only"
3589
3590
3591             AllSymbols[symbol] = 1
3592             blurb = ''
3593             if symbol in SymbolDocs and not IsEmptyDoc(SymbolDocs[symbol]):
3594                 blurb = ConvertMarkDown(symbol, SymbolDocs[symbol])
3595                 logging.info(".. [%s][%s]", SymbolDocs[symbol], blurb)
3596                 AllDocumentedSymbols[symbol] = 1
3597
3598             else:
3599                 if ArgBlurbs[i] != '':
3600                     blurb = "<para>" + CreateValidSGML(ArgBlurbs[i]) + "</para>"
3601                     AllDocumentedSymbols[symbol] = 1
3602                 else:
3603                     # FIXME: print a warning?
3604                     logging.info(".. no description")
3605
3606             pad1 = ''
3607             if len(name) < 24:
3608                 pad1 = " " * (24 - len(name))
3609
3610
3611             arg_synop = "<row><entry role=\"property_type\">%s</entry><entry role=\"property_name\"><link linkend=\"%s\">%s</link></entry><entry role=\"property_flags\">%s</entry></row>\n" % (type_output, sid, name, flags_string)
3612             arg_desc = "<refsect2 id=\"%s\" role=\"property\"><title>The <literal>“%s”</literal> %s</title>\n" % (sid, name, kind)
3613             arg_desc += MakeIndexterms(symbol, sid)
3614             arg_desc += "\n"
3615             arg_desc += OutputSymbolExtraLinks(symbol)
3616
3617             arg_desc += "<programlisting>  “%s”%s %s</programlisting>\n" % (name, pad1, type_output)
3618             arg_desc += blurb
3619             if symbol in SymbolAnnotations:
3620                 param_desc = SymbolAnnotations[symbol]
3621                 (param_desc, param_annotations) = ExpandAnnotation(symbol, param_desc)
3622                 if param_annotations != '':
3623                     arg_desc += "\n<para>%s</para>" % param_annotations
3624
3625             arg_desc += MakeDeprecationNote(symbol)
3626
3627             if flags_string:
3628                 arg_desc += "<para>Flags: %s</para>\n" % flags_string
3629
3630             if arange != '':
3631                 arg_desc += "<para>Allowed values: %s</para>\n" % range_output
3632
3633             if default != '':
3634                 arg_desc += "<para>Default value: %s</para>\n" % default_output
3635
3636             arg_desc += OutputSymbolTraits(symbol)
3637             arg_desc += "</refsect2>\n"
3638
3639             if 'c' in flags:
3640                 child_synop += arg_synop
3641                 child_desc += arg_desc
3642
3643             elif 's' in flags:
3644                 style_synop += arg_synop
3645                 style_desc += arg_desc
3646
3647             else:
3648                 synop += arg_synop
3649                 desc += arg_desc
3650
3651     return (synop, child_synop, style_synop, desc, child_desc, style_desc)
3652
3653
3654
3655 #############################################################################
3656 # Function    : ReadSourceDocumentation
3657 # Description : This reads in the documentation embedded in comment blocks
3658 #                in the source code (for Gnome).
3659 #
3660 #                Parameter descriptions override any in the template files.
3661 #                Function descriptions are placed before any description from
3662 #                the template files.
3663 #
3664 #                It recursively descends the source directory looking for .c
3665 #                files and scans them looking for specially-formatted comment
3666 #                blocks.
3667 #
3668 # Arguments   : $source_dir - the directory to scan.
3669 #############m###############################################################
3670
3671 def ReadSourceDocumentation(source_dir):
3672
3673     # prepend entries from @SOURCE_DIR
3674     for sdir in SOURCE_DIRS:
3675         # Check if the filename is in the ignore list.
3676         m1 = re.search(r'^%s/(.*)$' % re.escape(sdir), source_dir)
3677         if m1:
3678             m2 = re.search(r'(\s|^)%s(\s|$)' % re.escape(m1.group(1)), IGNORE_FILES)
3679             if m2:
3680                 logging.info("Skipping source directory: %s", source_dir)
3681                 return
3682             else:
3683                 logging.info("No match for: %s", (m1.group(1) or source_dir))
3684
3685     logging.info("Scanning source directory: %s", source_dir)
3686
3687     # This array holds any subdirectories found.
3688     subdirs = []
3689
3690     suffix_list = SOURCE_SUFFIXES.split(',')
3691
3692     for ifile in os.listdir(source_dir):
3693         logging.info("... : %s", ifile)
3694         if ifile.startswith('.'):
3695             continue
3696         fname = os.path.join(source_dir, ifile)
3697         if os.path.isdir(fname):
3698             subdirs.append(fname)
3699         elif SOURCE_SUFFIXES:
3700             for suffix in suffix_list:
3701                 if ifile.endswith(suffix):
3702                     ScanSourceFile(fname)
3703         elif re.search(r'\.[ch]$', ifile):
3704             ScanSourceFile(fname)
3705
3706     # Now recursively scan the subdirectories.
3707     for sdir in subdirs:
3708         ReadSourceDocumentation(sdir)
3709
3710 #############################################################################
3711 # Function    : ScanSourceFile
3712 # Description : Scans one source file looking for specially-formatted comment
3713 #                blocks. Later &MergeSourceDocumentation is used to merge any
3714 #                documentation found with the documentation already read in
3715 #                from the template files.
3716 #
3717 # Arguments   : $file - the file to scan.
3718 #############################################################################
3719
3720 def ScanSourceFile(ifile):
3721
3722     # prepend entries from @SOURCE_DIR
3723     for idir in SOURCE_DIRS:
3724         # Check if the filename is in the ignore list.
3725         m1 = re.search(r'^%s/(.*)$' % re.escape(idir), ifile)
3726         if m1:
3727             m2 = re.search(r'(\s|^)%s(\s|$)' % re.escape(m1.group(1)), IGNORE_FILES)
3728             if m2:
3729                 logging.info("Skipping source file: ", ifile)
3730                 return
3731
3732     m = re.search(r'^.*[\/\\]([^\/\\]*)$', ifile)
3733     if m:
3734         basename = m.group(1)
3735     else:
3736         common.LogWarning(ifile, 1, "Can't find basename for this filename.")
3737         basename = ifile
3738
3739
3740     # Check if the basename is in the list of files to ignore.
3741     if re.search(r'(\s|^)%s(\s|$)' % re.escape(basename), IGNORE_FILES):
3742         logging.info("Skipping source file: %s", ifile)
3743         return
3744
3745
3746     logging.info("Scanning source file: %s", ifile)
3747
3748     SRCFILE = open(ifile)
3749     in_comment_block = False
3750     symbol = None
3751     in_part = ''
3752     description = ''
3753     return_desc = ''
3754     since_desc = stability_desc = deprecated_desc = ''
3755     params = OrderedDict()
3756     param_name = None
3757     line_number = 0
3758     for line in SRCFILE:
3759         line_number += 1
3760         # Look for the start of a comment block.
3761         if not in_comment_block:
3762             if re.search(r'^\s*/\*.*\*/', line):
3763                 #one-line comment - not gtkdoc
3764                 pass
3765             elif re.search(r'^\s*/\*\*\s', line):
3766                 logging.info("Found comment block start")
3767
3768                 in_comment_block = True
3769
3770                 # Reset all the symbol data.
3771                 symbol = ''
3772                 in_part = ''
3773                 description = ''
3774                 return_desc = ''
3775                 since_desc = ''
3776                 deprecated_desc = ''
3777                 stability_desc = ''
3778                 params = OrderedDict()
3779                 param_name = None
3780
3781             continue
3782
3783         # We're in a comment block. Check if we've found the end of it.
3784         if re.search(r'^\s*\*+/', line):
3785             if not symbol:
3786                 # maybe its not even meant to be a gtk-doc comment?
3787                 common.LogWarning(ifile, line_number, "Symbol name not found at the start of the comment block.")
3788             else:
3789                 # Add the return value description onto the end of the params.
3790                 if return_desc:
3791                     # TODO(ensonic): check for duplicated Return docs
3792                     # common.LogWarning(file, line_number, "Multiple Returns for %s." % symbol)
3793                     params['Returns'] = return_desc
3794
3795                 # Convert special characters
3796                 description = ConvertSGMLChars(symbol, description)
3797                 for (param_name, param_desc) in params.iteritems():
3798                     params[param_name] = ConvertSGMLChars(symbol, param_desc)
3799
3800                 # Handle Section docs
3801                 m = re.search(r'SECTION:\s*(.*)', symbol)
3802                 m2 = re.search(r'PROGRAM:\s*(.*)', symbol)
3803                 if m:
3804                     real_symbol = m.group(1)
3805                     long_descr = os.path.join(TMPL_DIR, real_symbol + ":Long_Description")
3806
3807                     if KnownSymbols:
3808                         if long_descr not in KnownSymbols or KnownSymbols[long_descr] != 1:
3809                             common.LogWarning(ifile, line_number, "Section %s is not defined in the %s-sections.txt file." % (real_symbol, MODULE))
3810
3811                     logging.info("SECTION DOCS found in source for : '%s'", real_symbol)
3812                     for param_name, param_desc in params.iteritems():
3813                         param_name = param_name.lower()
3814                         logging.info("   '" + param_name + "'")
3815                         key = None
3816                         if param_name == "short_description":
3817                             key = os.path.join(TMPL_DIR, real_symbol + ":Short_Description")
3818                         elif param_name == "see_also":
3819                             key = os.path.join(TMPL_DIR, real_symbol + ":See_Also")
3820                         elif param_name == "title":
3821                             key = os.path.join(TMPL_DIR, real_symbol + ":Title")
3822                         elif param_name == "stability":
3823                             key = os.path.join(TMPL_DIR, real_symbol + ":Stability_Level")
3824                         elif param_name == "section_id":
3825                             key = os.path.join(TMPL_DIR, real_symbol + ":Section_Id")
3826                         elif param_name == "include":
3827                             key = os.path.join(TMPL_DIR, real_symbol + ":Include")
3828                         elif param_name == "image":
3829                             key = os.path.join(TMPL_DIR, real_symbol + ":Image")
3830
3831                         if key:
3832                             SourceSymbolDocs[key] = param_desc
3833                             SourceSymbolSourceFile[key] = ifile
3834                             SourceSymbolSourceLine[key] = line_number
3835
3836                     SourceSymbolDocs[long_descr] = description
3837                     SourceSymbolSourceFile[long_descr] = ifile
3838                     SourceSymbolSourceLine[long_descr] = line_number
3839                     #$SourceSymbolTypes{$symbol} = "SECTION"
3840                 elif m2:
3841                     real_symbol = m2.group(1)
3842                     key = None
3843                     section_id = None
3844
3845                     logging.info("PROGRAM DOCS found in source for '%s'", real_symbol)
3846                     for param_name, param_desc in params.iteritems():
3847                         param_name = param_name.lower()
3848                         logging.info("PROGRAM key %s: '%s'", real_symbol, param_name)
3849                         key = None
3850                         if param_name == "short_description":
3851                             key = os.path.join(TMPL_DIR, real_symbol + ":Short_Description")
3852                         elif param_name == "see_also":
3853                             key = os.path.join(TMPL_DIR, real_symbol + ":See_Also")
3854                         elif param_name == "section_id":
3855                             key = os.path.join(TMPL_DIR, real_symbol + ":Section_Id")
3856                         elif param_name == "synopsis":
3857                             key = os.path.join(TMPL_DIR, real_symbol + ":Synopsis")
3858                         elif param_name == "returns":
3859                             key = os.path.join(TMPL_DIR, real_symbol + ":Returns")
3860                         elif re.search(r'^(-.*)', param_name):
3861                             logging.info("PROGRAM opts: '%s': '%s'", param_name, param_desc)
3862                             key = os.path.join(TMPL_DIR, real_symbol + ":Options")
3863                             opts = []
3864                             opts_str = SourceSymbolDocs.get(key)
3865                             if opts_str:
3866                                 opts = opts_str.split('\t')
3867                             opts.append(param_name)
3868                             opts.append(param_desc)
3869
3870                             logging.info("Setting options for symbol: %s: '%s'", real_symbol, '\t'.join(opts))
3871                             SourceSymbolDocs[key] = '\t'.join(opts)
3872                             continue
3873
3874                         if key:
3875                             logging.info("PROGRAM value %s: '%s'", real_symbol, param_desc.rstrip())
3876                             SourceSymbolDocs[key] = param_desc.rstrip()
3877                             SourceSymbolSourceFile[key] = ifile
3878                             SourceSymbolSourceLine[key] = line_number
3879
3880                     long_descr = os.path.join(TMPL_DIR, real_symbol + ":Long_Description")
3881                     SourceSymbolDocs[long_descr] = description
3882                     SourceSymbolSourceFile[long_descr] = ifile
3883                     SourceSymbolSourceLine[long_descr] = line_number
3884
3885                     section_id = SourceSymbolDocs.get(os.path.join(TMPL_DIR, real_symbol + ":Section_Id"))
3886                     if section_id and section_id.strip() != '':
3887                         # Remove trailing blanks and use as is
3888                         section_id = section_id.rstrip()
3889                     else:
3890                         section_id = common.CreateValidSGMLID('%s-%s' % (MODULE, real_symbol))
3891                     OutputProgramDBFile(real_symbol, section_id)
3892
3893                 else:
3894                     logging.info("SYMBOL DOCS found in source for : '%s' %d, %s", symbol, len(description), str(params))
3895                     SourceSymbolDocs[symbol] = description
3896                     SourceSymbolParams[symbol] = params
3897                     # FIXME $SourceSymbolTypes{$symbol} = "STRUCT,SIGNAL,ARG,FUNCTION,MACRO"
3898                     #if (defined $DeclarationTypes{$symbol})
3899                     #    $SourceSymbolTypes{$symbol} = $DeclarationTypes{$symbol
3900                     #
3901                     SourceSymbolSourceFile[symbol] = ifile
3902                     SourceSymbolSourceLine[symbol] = line_number
3903
3904                 if since_desc:
3905                     arr = since_desc.splitlines()
3906                     since_desc = arr[0].strip()
3907                     extra_lines = arr[1:]
3908                     logging.info("Since(%s) : [%s]", symbol, since_desc)
3909                     Since[symbol] = ConvertSGMLChars(symbol, since_desc)
3910                     if len(extra_lines) > 1:
3911                         common.LogWarning(ifile, line_number, "multi-line since docs found")
3912
3913                 if stability_desc:
3914                     stability_desc = ParseStabilityLevel(stability_desc, ifile, line_number, "Stability level for %s" % symbol)
3915                     StabilityLevel[symbol] = ConvertSGMLChars(symbol, stability_desc)
3916
3917                 if deprecated_desc:
3918                     if symbol not in Deprecated:
3919                         # don't warn for signals and properties
3920                         #if ($symbol !~ m/::?(.*)/)
3921                         if symbol in DeclarationTypes:
3922                             common.LogWarning(ifile, line_number,
3923                                               "%s is deprecated in the inline comments, but no deprecation guards were found around the declaration. (See the --deprecated-guards option for gtkdoc-scan.)" % symbol)
3924
3925                     Deprecated[symbol] = ConvertSGMLChars(symbol, deprecated_desc)
3926
3927             in_comment_block = False
3928             continue
3929
3930         # Get rid of ' * ' at start of every line in the comment block.
3931         line = re.sub(r'^\s*\*\s?', '', line)
3932         # But make sure we don't get rid of the newline at the end.
3933         if not line.endswith('\n'):
3934             line = line + "\n"
3935
3936         logging.info("scanning : %s", line)
3937
3938         # If we haven't found the symbol name yet, look for it.
3939         if not symbol:
3940             m1 = re.search(r'^\s*(SECTION:\s*\S+)', line)
3941             m2 = re.search(r'^\s*(PROGRAM:\s*\S+)', line)
3942             m3 = re.search(r'^\s*([\w:-]*\w)\s*:?\s*(\(.+?\)\s*)*$', line)
3943             if m1:
3944                 symbol = m1.group(1)
3945                 logging.info("SECTION DOCS found in source for : '%s'", symbol)
3946             elif m2:
3947                 symbol = m2.group(1)
3948                 logging.info("PROGRAM DOCS found in source for : '%s'", symbol)
3949             elif m3:
3950                 symbol = m3.group(1)
3951                 annotation = m3.group(2)
3952                 logging.info("SYMBOL DOCS found in source for : '%s'", symbol)
3953                 if annotation:
3954                     annotation = annotation.strip()
3955                     if annotation != '':
3956                         SymbolAnnotations[symbol] = annotation
3957                         logging.info("remaining text for %s: '%s'", symbol, annotation)
3958
3959             continue
3960
3961         if in_part == "description":
3962             # Get rid of 'Description:'
3963             line = re.sub(r'^\s*Description:', '', line)
3964
3965         m1 = re.search(r'^\s*(returns|return\s+value):', line, flags=re.I)
3966         m2 = re.search(r'^\s*since:', line, flags=re.I)
3967         m3 = re.search(r'^\s*deprecated:', line, flags=re.I)
3968         m4 = re.search(r'^\s*stability:', line, flags=re.I)
3969
3970         if m1:
3971             # we're in param section and have not seen the blank line
3972             if in_part != '':
3973                 return_desc = line[m1.end():]
3974                 in_part = "return"
3975                 continue
3976
3977         if m2:
3978             # we're in param section and have not seen the blank line
3979             if in_part != "param":
3980                 since_desc = line[m2.end():]
3981                 in_part = "since"
3982                 continue
3983
3984         elif m3:
3985             # we're in param section and have not seen the blank line
3986             if in_part != "param":
3987                 deprecated_desc = line[m3.end():]
3988                 in_part = "deprecated"
3989                 continue
3990
3991         elif m4:
3992             stability_desc = line[m4.end():]
3993             in_part = "stability"
3994             continue
3995
3996         if in_part == "description":
3997             description += line
3998             continue
3999         elif in_part == "return":
4000             return_desc += line
4001             continue
4002         elif in_part == "since":
4003             since_desc += line
4004             continue
4005         elif in_part == "stability":
4006             stability_desc += line
4007             continue
4008         elif in_part == "deprecated":
4009             deprecated_desc += line
4010             continue
4011
4012         # We must be in the parameters. Check for the empty line below them.
4013         if re.search(r'^\s*$', line):
4014             in_part = "description"
4015             continue
4016
4017         # Look for a parameter name.
4018         m = re.search(r'^\s*@(.+?)\s*:\s*', line)
4019         if m:
4020             param_name = m.group(1)
4021             param_desc = line[m.end():]
4022
4023             logging.info("Found parameter: %s", param_name)
4024             # Allow varargs variations
4025             if re.search(r'^\.\.\.$', param_name):
4026                 param_name = "..."
4027
4028             logging.info("Found param for symbol %s : '%s'= '%s'", symbol, param_name, line)
4029
4030             params[param_name] = param_desc
4031             in_part = "param"
4032             continue
4033         elif in_part == '':
4034             logging.info("continuation for %s annotation '%s'", symbol, line)
4035             annotation = re.sub(r'^\s+|\s+$', '', line)
4036             if symbol in SymbolAnnotations:
4037                 SymbolAnnotations[symbol] += annotation
4038             else:
4039                 SymbolAnnotations[symbol] = annotation
4040             continue
4041
4042         # We must be in the middle of a parameter description, so add it on
4043         # to the last element in @params.
4044         if not param_name:
4045             common.LogWarning(file, line_number, "Parsing comment block file : parameter expected, but got '%s'" % line)
4046         else:
4047             params[param_name] += line
4048
4049     SRCFILE.close()
4050
4051
4052 #############################################################################
4053 # Function    : OutputMissingDocumentation
4054 # Description : Outputs report of documentation coverage to a file
4055 #
4056 # Arguments   : none
4057 #############################################################################
4058
4059 def OutputMissingDocumentation():
4060     old_undocumented_file = os.path.join(ROOT_DIR, MODULE + "-undocumented.txt")
4061     new_undocumented_file = os.path.join(ROOT_DIR, MODULE + "-undocumented.new")
4062
4063     n_documented = 0
4064     n_incomplete = 0
4065     total = 0
4066     symbol = None
4067     percent = None
4068     buffer = ''
4069     buffer_deprecated = ''
4070     buffer_descriptions = ''
4071
4072     UNDOCUMENTED = open(new_undocumented_file, 'w')
4073
4074     for symbol in sorted(AllSymbols.keys()):
4075         # FIXME: should we print common.LogWarnings for undocumented stuff?
4076         # DEBUG
4077         #my $ssfile = &GetSymbolSourceFile($symbol)
4078         #my $ssline = &GetSymbolSourceLine($symbol)
4079         #my $location = "defined at " . (defined($ssfile)?$ssfile:"?") . ":" . (defined($ssline)?$ssline:"0") . "\n"
4080         # DEBUG
4081         m = re.search(r':(Title|Long_Description|Short_Description|See_Also|Stability_Level|Include|Section_Id|Image)', symbol)
4082         m2 = re.search(r':(Long_Description|Short_Description)', symbol)
4083         if not m:
4084             total += 1
4085             if symbol in AllDocumentedSymbols:
4086                 n_documented += 1
4087                 if symbol in AllIncompleteSymbols:
4088                     n_incomplete += 1
4089                     buffer += symbol + " (" + AllIncompleteSymbols[symbol] + ")\n"
4090                     #$buffer += "\t0: ".$location
4091
4092             elif symbol in Deprecated:
4093                 if symbol in AllIncompleteSymbols:
4094                     n_incomplete += 1
4095                     buffer_deprecated += symbol + " (" + AllIncompleteSymbols[symbol] + ")\n"
4096                     #$buffer += "\t1a: ".$location
4097                 else:
4098                     buffer_deprecated += symbol + "\n"
4099                     #$buffer += "\t1b: ".$location
4100
4101             else:
4102                 if symbol in AllIncompleteSymbols:
4103                     n_incomplete += 1
4104                     buffer += symbol + " (" + AllIncompleteSymbols[symbol] + ")\n"
4105                     #$buffer += "\t2a: ".$location
4106                 else:
4107                     buffer += symbol + "\n"
4108                     #$buffer += "\t2b: ".$location
4109
4110         elif m2:
4111             total += 1
4112             if symbol in SymbolDocs and len(SymbolDocs[symbol]) > 0\
4113                or symbol in AllDocumentedSymbols and len(AllDocumentedSymbols[symbol]) > 0:
4114                 n_documented += 1
4115             else:
4116                 # cut off the leading namespace ($TMPL_DIR)
4117                 m = re.search(r'^.*\/(.*)$', symbol)
4118                 buffer_descriptions += m.group(1) + "\n"
4119
4120     if total == 0:
4121         percent = 100
4122     else:
4123         percent = (n_documented / total) * 100.0
4124
4125
4126     UNDOCUMENTED.write("%.0f%% symbol docs coverage.\n"%percent)
4127     UNDOCUMENTED.write("%s symbols documented.\n" % n_documented)
4128     UNDOCUMENTED.write("%s symbols incomplete.\n" % n_incomplete)
4129     UNDOCUMENTED.write("%d not documented.\n" % (total - n_documented))
4130
4131     if buffer_deprecated != '':
4132         buffer += "\n" + buffer_deprecated
4133
4134     if buffer_descriptions != '':
4135         buffer += "\n" + buffer_descriptions
4136
4137     if buffer != '':
4138         UNDOCUMENTED.write("\n\n" + buffer)
4139
4140     UNDOCUMENTED.close()
4141
4142     return common.UpdateFileIfChanged(old_undocumented_file, new_undocumented_file, 0)
4143
4144 #    printf "%.0f%% symbol docs coverage", $percent
4145 #    print "($n_documented symbols documented, $n_incomplete symbols incomplete, " . ($total - $n_documented) . " not documented)\n"
4146 #    print "See $MODULE-undocumented.txt for a list of missing docs.\nThe doc coverage percentage doesn't include intro sections.\n"
4147
4148
4149
4150 #############################################################################
4151 # Function    : OutputUndeclaredSymbols
4152 # Description : Outputs symbols that are listed in the section file, but not
4153 #               declaration is found in the sources
4154 #
4155 # Arguments   : none
4156 #############################################################################
4157
4158 def OutputUndeclaredSymbols():
4159     old_undeclared_file = os.path.join(ROOT_DIR, MODULE + "-undeclared.txt")
4160     new_undeclared_file = os.path.join(ROOT_DIR, MODULE + "-undeclared.new")
4161
4162     UNDECLARED = open(new_undeclared_file, 'w')
4163
4164     if UndeclaredSymbols:
4165         UNDECLARED.write("\n".join(sorted(UndeclaredSymbols.keys())))
4166         UNDECLARED.write("\n")
4167         print("See %s-undeclared.txt for the list of undeclared symbols." % MODULE)
4168
4169     UNDECLARED.close()
4170
4171     return common.UpdateFileIfChanged(old_undeclared_file, new_undeclared_file, 0)
4172
4173
4174 #############################################################################
4175 # Function    : OutputUnusedSymbols
4176 # Description : Outputs symbols that are documented in comments, but not
4177 #               declared in the sources
4178 #
4179 # Arguments   : none
4180 #############################################################################
4181
4182 def OutputUnusedSymbols():
4183     num_unused = 0
4184     old_unused_file = os.path.join(ROOT_DIR, MODULE + "-unused.txt")
4185     new_unused_file = os.path.join(ROOT_DIR, MODULE + "-unused.new")
4186
4187     UNUSED = open(new_unused_file, 'w')
4188
4189     for symbol in sorted(Declarations.keys()):
4190         if not symbol in DeclarationOutput:
4191             UNUSED.write("%s\n" % symbol)
4192             num_unused += 1
4193
4194
4195     for symbol in sorted(AllUnusedSymbols.keys()):
4196         UNUSED.write(symbol + "(" + AllUnusedSymbols[symbol] + ")\n")
4197         num_unused += 1
4198
4199     UNUSED.close()
4200     if num_unused != 0:
4201         common.LogWarning(old_unused_file, 1, "%d unused declarations. They should be added to %s-sections.txt in the appropriate place." % (num_unused, MODULE))
4202
4203
4204     return common.UpdateFileIfChanged(old_unused_file, new_unused_file, 0)
4205
4206
4207
4208 #############################################################################
4209 # Function    : OutputAllSymbols
4210 # Description : Outputs list of all symbols to a file
4211 #
4212 # Arguments   : none
4213 #############################################################################
4214
4215 def OutputAllSymbols():
4216     SYMBOLS = open(os.path.join(ROOT_DIR, MODULE + "-symbols.txt"), 'w')
4217
4218     for symbol in sorted(AllSymbols.keys()):
4219         SYMBOLS.write(symbol + "\n")
4220     SYMBOLS.close()
4221
4222
4223 #############################################################################
4224 # Function    : OutputSymbolsWithoutSince
4225 # Description : Outputs list of all symbols without a since tag to a file
4226 #
4227 # Arguments   : none
4228 #############################################################################
4229
4230 def OutputSymbolsWithoutSince():
4231     SYMBOLS = open(os.path.join(ROOT_DIR, MODULE + "-nosince.txt"), 'w')
4232
4233     for symbol in sorted(SourceSymbolDocs.keys()):
4234         if symbol in Since:
4235             SYMBOLS.write(symbol + "\n")
4236     SYMBOLS.close()
4237
4238 #############################################################################
4239 # Function    : MergeSourceDocumentation
4240 # Description : This merges documentation read from a source file into the
4241 #                documentation read in from a template file.
4242 #
4243 #                Parameter descriptions override any in the template files.
4244 #                Function descriptions are placed before any description from
4245 #                the template files.
4246 #
4247 # Arguments   : none
4248 #############################################################################
4249
4250 def MergeSourceDocumentation():
4251     if SymbolDocs:
4252         Symbols = SymbolDocs.keys()
4253         logging.info("num existing entries: %d", len(Symbols))
4254     else:
4255         # filter scanned declarations, with what we suppress from -sections.txt
4256         tmp = {}
4257         for symbol in Declarations.keys():
4258             if symbol in KnownSymbols and KnownSymbols[symbol] == 1:
4259                 tmp[symbol] = 1
4260
4261         # , add the rest from -sections.txt
4262         for symbol in KnownSymbols.keys():
4263             if KnownSymbols[symbol] == 1:
4264                 tmp[symbol] = 1
4265
4266         # and add whats found in the source
4267         for symbol in SourceSymbolDocs.keys():
4268             tmp[symbol] = 1
4269
4270         Symbols = tmp.keys()
4271         logging.info("num source entries: %d", len(Symbols))
4272
4273     for symbol in Symbols:
4274         AllSymbols[symbol] = 1
4275
4276         have_tmpl_docs = 0
4277
4278         ## see if the symbol is documented in template
4279         tmpl_doc = SymbolDocs.get(symbol, '')
4280         check_tmpl_doc = tmpl_doc
4281         # remove all xml-tags and whitespaces
4282         check_tmpl_doc = re.sub(r'<.*?>', '', check_tmpl_doc)
4283         check_tmpl_doc = re.sub(r'\s', '', check_tmpl_doc)
4284         # anything left ?
4285         if check_tmpl_doc != '':
4286             have_tmpl_docs = 1
4287         else:
4288             # if the docs have just an empty para, don't merge that.
4289             check_tmpl_doc = re.sub(r'(\s|\n)', '', tmpl_doc, flags=re.M|re.S)
4290             if check_tmpl_doc == "<para></para>":
4291                 tmpl_doc = ''
4292
4293         if symbol in SourceSymbolDocs:
4294             stype = DeclarationTypes.get(symbol)
4295
4296             logging.info("merging [%s] from source", symbol)
4297
4298             item = "Parameter"
4299             if stype:
4300                 if stype == 'STRUCT':
4301                     item = "Field"
4302                 elif stype == 'ENUM':
4303                     item = "Value"
4304                 elif stype == 'UNION':
4305                     item = "Field"
4306             else:
4307                 stype = "SIGNAL"
4308
4309             src_doc = SourceSymbolDocs[symbol]
4310             # remove leading and training whitespaces
4311             src_doc = src_doc.strip()
4312
4313             # Don't output warnings for overridden titles as titles are
4314             # automatically generated in the -sections.txt file, and thus they
4315             # are often overridden.
4316             m = re.search(r':Title$', symbol)
4317             if have_tmpl_docs and not m:
4318                 # check if content is different
4319                 if tmpl_doc != src_doc:
4320                     #print "[$tmpl_doc] [$src_doc]\n"
4321                     common.LogWarning(SourceSymbolSourceFile[symbol], SourceSymbolSourceLine[symbol],
4322                                       "Documentation in template " + SymbolSourceFile[symbol] + ":" + SymbolSourceLine[symbol] + " for %s being overridden by inline comments." % symbol)
4323
4324             if src_doc != '':
4325                 AllDocumentedSymbols[symbol] = 1
4326
4327
4328             # Do not add <para> to nothing, it breaks missing docs checks.
4329             src_doc_para = ''
4330             if src_doc != '':
4331                 src_doc_para = src_doc
4332
4333
4334             m = re.search(TMPL_DIR + r'\/.+:Long_Description', symbol)
4335             m2 = re.search(TMPL_DIR + r'\/.+:.+', symbol)
4336             if m:
4337                 SymbolDocs[symbol] = '%s%s' % (src_doc_para, tmpl_doc)
4338             elif m2:
4339                 # For the title/summary/see also section docs we don't want to
4340                 # add any <para> tags.
4341                 SymbolDocs[symbol] = src_doc
4342             else:
4343                 SymbolDocs[symbol] = '%s%s' % (src_doc_para, tmpl_doc)
4344
4345             # merge parameters
4346             if re.search(r'.*::.*', symbol):
4347                 # For signals we prefer the param names from the source docs,
4348                 # since the ones from the templates are likely to contain the
4349                 # artificial argn names which are generated by gtkdoc-scangobj.
4350                 SymbolParams[symbol] = SourceSymbolParams[symbol]
4351                 # FIXME: we need to check for empty docs here as well!
4352             else:
4353                 # The templates contain the definitive parameter names and order,
4354                 # so we will not change that. We only override the actual text.
4355                 tmpl_params = SymbolParams.get(symbol)
4356                 if not tmpl_params:
4357                     logging.info("No merge needed for %s", symbol)
4358                     if symbol in SourceSymbolParams:
4359                         SymbolParams[symbol] = SourceSymbolParams[symbol]
4360                     #  FIXME: we still like to get the number of params and merge
4361                     #  1) we would noticed that params have been removed/renamed
4362                     #  2) we would catch undocumented params
4363                     #  params are not (yet) exported in -decl.txt so that we
4364                     #  could easily grab them :/
4365                 else:
4366                     params = SourceSymbolParams.get(symbol)
4367                     logging.info("Merge needed for %s, tmpl_params: %d, source_params: %d", symbol, len(tmpl_params), len(params))
4368                     for tmpl_param_name in tmpl_params.keys():
4369                         # Try to find the param in the source comment documentation.
4370                         found = False
4371                         logging.info("  try merge param %s", tmpl_param_name)
4372                         for (param_name, param_desc) in params.iteritems():
4373                             logging.info("    test param  %s", param_name)
4374                             # We accept changes in case, since the Gnome source
4375                             # docs contain a lot of these.
4376                             if param_name.lower() == tmpl_param_name.lower():
4377                                 found = True
4378
4379                                 # Override the description.
4380                                 tmpl_params[param_name] = param_desc
4381
4382                                 # Set the name to '' to mark it as used.
4383                                 params[param_name] = ''
4384                                 break
4385
4386                         # If it looks like the parameters are there, but not
4387                         # in the right place, try to explain a bit better.
4388                         if found and re.search(r'\@%s:' % tmpl_param_name, src_doc):
4389                             common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
4390                                               "Parameters for %s must start on the line immediately after the function or macro name." % symbol)
4391
4392                     # Now we output a warning if parameters have been described which
4393                     # do not exist.
4394                     for param_name in params.keys():
4395                         if param_name:
4396                             # the template builder cannot detect if a macro returns
4397                             # a result or not
4398                             if stype == "MACRO" and param_name == "Returns":
4399                                 # FIXME: do we need to add it then to tmpl_params[] ?
4400                                 logging.info("  adding Returns: to macro docs for %s.", symbol)
4401                                 tmpl_params['Returns'] = params[param_name]
4402                                 continue
4403
4404                             common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
4405                                               "%s described in source code comment block but does not exist. %s: %s %s: %s." % (item, stype, symbol, item, param_name))
4406
4407         else:
4408             if have_tmpl_docs:
4409                 AllDocumentedSymbols[symbol] = 1
4410                 logging.info("merging [%s] from template", symbol)
4411             else:
4412                 logging.info("[%s] undocumented", symbol)
4413
4414         # if this symbol is documented, check if docs are complete
4415         check_tmpl_doc = SymbolDocs.get(symbol, '')
4416         # remove all xml-tags and whitespaces
4417         check_tmpl_doc = re.sub(r'<.*?>', '', check_tmpl_doc)
4418         check_tmpl_doc = re.sub(r'\s', '', check_tmpl_doc)
4419         if check_tmpl_doc != '':
4420             tmpl_params = SymbolParams.get(symbol)
4421             if tmpl_params:
4422                 stype = DeclarationTypes.get(symbol)
4423
4424                 item = "Parameter"
4425                 if stype:
4426                     if stype == 'STRUCT':
4427                         item = "Field"
4428                     elif stype == 'ENUM':
4429                         item = "Value"
4430                     elif stype == 'UNION':
4431                         item = "Field"
4432
4433                 else:
4434                     stype = "SIGNAL"
4435                 logging.info("Check param docs for %s, tmpl_params: %s entries, type=%s", symbol, len(tmpl_params), stype)
4436
4437                 if len(tmpl_params) > 0:    # FIXME: we checked this already above
4438                     logging.info("tmpl_params: %s", str(tmpl_params))
4439                     for (tmpl_param_name, tmpl_param_desc) in tmpl_params.iteritems():
4440                         # Output a warning if the parameter is empty and
4441                         # remember for stats.
4442                         if tmpl_param_name != "void" and not re.search(r'\S', tmpl_param_desc):
4443                             if symbol in AllIncompleteSymbols[symbol]:
4444                                 AllIncompleteSymbols[symbol] += ", " + tmpl_param_name
4445                             else:
4446                                 AllIncompleteSymbols[symbol] = tmpl_param_name
4447
4448                             common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
4449                                               "%s description for %s::%s is missing in source code comment block." % (item, symbol, tmpl_param_name))
4450
4451                 else:
4452                     if len(tmpl_params) == 0:
4453                         AllIncompleteSymbols[symbol] = "<items>"
4454                         common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
4455                                           "%s descriptions for %s are missing in source code comment block." % (item, symbol))
4456
4457                     # $#$tmpl_params==-1 means we don't know about parameters
4458                     # this unfortunately does not tell if there should be some
4459     logging.info("num doc entries: %d", len(SymbolDocs))
4460
4461
4462 #############################################################################
4463 # Function    : IsEmptyDoc
4464 # Description : Check if a doc-string is empty. Its also regarded as empty if
4465 #               it only consist of whitespace or e.g. FIXME.
4466 # Arguments   : the doc-string
4467 #############################################################################
4468 def IsEmptyDoc(doc):
4469
4470     if re.search(r'^\s*$', doc):
4471         return True
4472
4473
4474     if re.search(r'^\s*<para>\s*(FIXME)?\s*<\/para>\s*$', doc):
4475         return True
4476
4477
4478     return False
4479
4480
4481 #############################################################################
4482 # Function    : ConvertMarkDown
4483 # Description : Converts mark down syntax to the respective docbook.
4484 #               http://de.wikipedia.org/wiki/Markdown
4485 #               Inspired by the design of ParseDown
4486 #               http://parsedown.org/
4487 #               Copyright (c) 2013 Emanuil Rusev, erusev.com
4488 # Arguments   : the symbol name, the doc-string
4489 #############################################################################
4490
4491 def ConvertMarkDown(symbol, text):
4492     text = MarkDownParse(text, symbol)
4493     return text
4494
4495
4496 # SUPPORTED MARKDOWN
4497 # ==================
4498 #
4499 # Atx-style Headers
4500 # -----------------
4501 #
4502 # # Header 1
4503 #
4504 # ## Header 2 ##
4505 #
4506 # Setext-style Headers
4507 # --------------------
4508 #
4509 # Header 1
4510 # ========
4511 #
4512 # Header 2
4513 # --------
4514 #
4515 # Ordered (unnested) Lists
4516 # ------------------------
4517 #
4518 # 1. item 1
4519 #
4520 # 1. item 2 with loooong
4521 #    description
4522 #
4523 # 3. item 3
4524 #
4525 # Note: we require a blank line above the list items
4526 #
4527
4528 # TODO(ensonic): it would be nice to add id parameters to the refsect2 elements
4529
4530 def MarkDownParseBlocks(lines, symbol, context):
4531   md_blocks = []
4532   md_block = {"type": ''}
4533
4534   logging.info("parsing %s lines", len(lines))
4535   for line in lines:
4536     logging.info("type='%s', int='%s', parsing '%s'", md_block["type"], md_block.get('interrupted'), line)
4537     first_char = None
4538     if line:
4539         first_char = line[0]
4540
4541     if md_block["type"] == "markup":
4542       if 'closed' not in md_block:
4543         if md_block["start"] in line:
4544           md_block["depth"] += 1
4545
4546         if md_block["end"] in line:
4547           if md_block["depth"] > 0:
4548             md_block["depth"] -= 1
4549           else:
4550             logging.info("closing tag '%s'", line)
4551             md_block["closed"] = 1
4552             # TODO(ensonic): reparse inner text with MarkDownParseLines?
4553
4554         md_block["text"] += "\n" + line
4555         logging.info("add to markup: '%s'", line)
4556         continue
4557
4558     deindented_line = line.lstrip()
4559
4560     if md_block["type"] == "heading":
4561       # a heading is ended by any level less than or equal
4562       if md_block["level"] == 1:
4563         heading_match = re.search(r'^[#][ \t]+(.+?)[ \t]*[#]*[ \t]*(?:{#([^}]+)})?[ \t]*$', line)
4564         if re.search(r'^={4,}[ \t]*$', line):
4565           text = md_block["lines"].pop()
4566           md_block.pop("interrupted", None)
4567           md_blocks.append(md_block)
4568           md_block = {'type': "heading",
4569                       'text': text,
4570                       'lines': [],
4571                       'level': 1,
4572                      }
4573           continue
4574         elif heading_match:
4575           md_block.pop("interrupted", None)
4576           md_blocks.append(md_block)
4577           md_block = {'type': "heading",
4578                       'text': heading_match.group(1),
4579                       'lines': [],
4580                       'level': 1,
4581                      }
4582           if heading_match.group(2):
4583               md_block['id'] = heading_match.group(2)
4584           continue
4585         else:
4586           # push lines into the block until the end is reached
4587           md_block["lines"].append(line)
4588           continue
4589
4590       else:
4591         heading_match = re.search(r'^([#]{1,2})[ \t]+(.+?)[ \t]*[#]*[ \t]*(?:{#([^}]+)})?[ \t]*$', line)
4592         if re.search(r'^[=]{4,}[ \t]*$', line):
4593           text = md_block["lines"].pop()
4594           md_block.pop("interrupted", None)
4595           md_blocks.append(md_block)
4596           md_block = {'type': "heading",
4597                       'text': text,
4598                       'lines': [],
4599                       'level': 1,
4600           }
4601           continue
4602         elif re.search(r'^[-]{4,}[ \t]*$', line):
4603           text = md_block["lines"].pop()
4604           md_block.pop("interrupted", None)
4605           md_blocks.append(md_block)
4606           md_block = {'type': "heading",
4607                       'text': text,
4608                       'lines': [],
4609                       'level': 2,
4610                       }
4611           continue
4612         elif heading_match:
4613           md_block.pop("interrupted", None)
4614           md_blocks.append(md_block)
4615           md_block = {'type': "heading",
4616                       'text': heading_match.group(2),
4617                       'lines': [],
4618                       'level': len(heading_match.group(1))
4619                       }
4620           if heading_match.group(3):
4621               md_block['id'] = heading_match.group(3)
4622           continue
4623         else:
4624           # push lines into the block until the end is reached
4625           md_block["lines"].append(line)
4626           continue
4627     elif md_block["type"] == "code":
4628       end_of_code_match = re.search(r'^[ \t]*\]\|(.*)', line)
4629       if end_of_code_match:
4630         md_blocks.append(md_block)
4631         md_block = {'type': "paragraph",
4632                     'text': end_of_code_match.group(1),
4633                     'lines': [],
4634                     }
4635       else:
4636         md_block["lines"].append(line)
4637       continue
4638
4639     if deindented_line == '':
4640       logging.info('setting "interrupted" due to empty line')
4641       md_block["interrupted"] = 1
4642       continue
4643
4644     if md_block["type"] == "quote":
4645       if 'interrupted' not in md_block:
4646         line = re.sub(r'^[ ]*>[ ]?', '', line)
4647         md_block["lines"].append(line)
4648         continue
4649
4650     elif md_block["type"] == "li":
4651       marker = md_block["marker"]
4652       marker_match = re.search(r'^([ ]{0,3})(%s)[ ](.*)' % marker, line)
4653       if marker_match:
4654         indentation = marker_match.group(1)
4655         if md_block["indentation"] != indentation:
4656           md_block["lines"].append(line)
4657         else:
4658           ordered = md_block["ordered"]
4659           md_block.pop('last', None)
4660           md_blocks.append(md_block)
4661           md_block = {'type': "li",
4662                       'ordered': ordered,
4663                       'indentation': indentation,
4664                       'marker': marker,
4665                       'last': 1,
4666                       'lines': [re.sub(r'^[ ]{0,4}', '', marker_match.group(3))],
4667                       }
4668         continue
4669
4670       if 'interrupted' in md_block:
4671         if first_char == " ":
4672           md_block["lines"].append('')
4673           line =  re.sub(r'^[ ]{0,4}', '', line)
4674           md_block["lines"].append(line)
4675           md_block.pop("interrupted", None)
4676           continue
4677       else:
4678         line = re.sub(r'^[ ]{0,4}', '', line)
4679         md_block["lines"].append(line)
4680         continue
4681
4682     # indentation sensitive types
4683     heading_match = re.search(r'^([#]{1,2})[ \t]+(.+?)[ \t]*[#]*[ \t]*(?:{#([^}]+)})?[ \t]*$', line)
4684     code_match = re.search(r'^[ \t]*\|\[[ ]*(?:<!-- language="([^"]+?)" -->)?', line)
4685     if heading_match:
4686       # atx heading (#)
4687       md_blocks.append(md_block)
4688       md_block = {'type': "heading",
4689                   'text': heading_match.group(2),
4690                   'lines': [],
4691                   'level': len(heading_match.group(1)),
4692                   }
4693       if heading_match.group(3):
4694           md_block['id'] = heading_match.group(3)
4695       continue
4696     elif re.search(r'^={4,}[ \t]*$', line):
4697       # setext heading (====)
4698
4699       if md_block["type"] == "paragraph" and "interrupted" in md_block:
4700         md_blocks.append(md_block.copy())
4701         md_block["type"] = "heading"
4702         md_block["lines"] = []
4703         md_block["level"] = 1
4704       continue
4705     elif re.search(r'^-{4,}[ \t]*$', line):
4706       # setext heading (-----)
4707
4708       if md_block["type"] == "paragraph" and "interrupted" in md_block:
4709         md_blocks.append(md_block.copy())
4710         md_block["type"] = "heading"
4711         md_block["lines"] = []
4712         md_block["level"] = 2
4713
4714       continue
4715     elif code_match:
4716       # code
4717       md_block["interrupted"] = 1
4718       md_blocks.append(md_block)
4719       md_block = {'type': "code",
4720                   'lines': [],
4721                   }
4722       if code_match.group(1):
4723           md_block['language'] = code_match.group(1)
4724       continue
4725
4726     # indentation insensitive types
4727     markup_match = re.search(r'^[ ]*<\??(\w+)[^>]*([\/\?])?[ \t]*>', line)
4728     li_match = re.search(r'^([ ]*)[*+-][ ](.*)', line)
4729     quote_match = re.search(r'^[ ]*>[ ]?(.*)', line)
4730     if re.search(r'^[ ]*<!DOCTYPE/', line):
4731       md_blocks.append(md_block)
4732       md_block = {'type'   : "markup",
4733                   'text'   : deindented_line,
4734                   'start'  : '<',
4735                   'end'    : '>',
4736                   'depth'  : 0,
4737                   }
4738
4739     elif markup_match:
4740       # markup, including <?xml version="1.0"?>
4741       tag = markup_match.group(1)
4742       is_self_closing = markup_match.group(2) is not None
4743
4744       # skip link markdown
4745       # TODO(ensonic): consider adding more uri schemes (ftp, ...)
4746       if re.search(r'https?', tag):
4747         logging.info("skipping link '%s'", tag)
4748       else:
4749         # for TEXT_LEVEL_ELEMENTS, we want to keep them as-is in the paragraph
4750         # instead of creation a markdown block.
4751         scanning_for_end_of_text_level_tag = (
4752             md_block["type"] == "paragraph" and
4753             'start' in md_block and
4754             'closed' not in md_block)
4755         logging.info("markup found '%s', scanning %s ?", tag, scanning_for_end_of_text_level_tag)
4756         if tag not in MD_TEXT_LEVEL_ELEMENTS and not scanning_for_end_of_text_level_tag:
4757           md_blocks.append(md_block)
4758
4759           if is_self_closing:
4760             logging.info("self-closing docbook '%s'", tag)
4761             md_block = {'type': "self-closing tag",
4762                         'text': deindented_line,
4763                         }
4764             is_self_closing = 0
4765             continue
4766
4767           logging.info("new markup '%s'", tag)
4768           md_block = {'type'   : "markup",
4769                       'text'   : deindented_line,
4770                       'start'  : '<' + tag + '>',
4771                       'end'    : '</' + tag + '>',
4772                       'depth'  : 0,
4773                       }
4774           if re.search(r'<\/%s>' % tag, deindented_line):
4775             md_block["closed"] = 1
4776
4777           continue
4778         else:
4779           if tag in MD_TEXT_LEVEL_ELEMENTS:
4780             logging.info("text level docbook '%s' in '%s' state", tag, md_block["type"])
4781             # TODO(ensonic): handle nesting
4782             if not scanning_for_end_of_text_level_tag:
4783               if not re.search(r'<\/%s>' % tag, deindented_line):
4784                 logging.info("new text level markup '%s'", tag)
4785                 md_block["start"] = '<' + tag + '>'
4786                 md_block["end"] = '</' + tag + '>'
4787                 md_block.pop("closed", None)
4788                 logging.info("scanning for end of '%s'", tag)
4789
4790             else:
4791               if md_block["end"] in deindented_line:
4792                 md_block["closed"] = 1
4793                 logging.info("found end of '%s'", tag)
4794     elif li_match:
4795       # li
4796       md_blocks.append(md_block)
4797       indentation = li_match.group(1)
4798       md_block = {'type': "li",
4799                   'ordered': 0,
4800                   'indentation': indentation,
4801                   'marker': "[*+-]",
4802                   'first': 1,
4803                   'last': 1,
4804                   'lines': [re.sub(r'^[ ]{0,4}', '', li_match.group(2))],
4805                   }
4806       continue
4807     elif quote_match:
4808       md_blocks.append(md_block)
4809       md_block = {'type': "quote",
4810                   'lines': [quote_match.group(1)],
4811                   }
4812       continue
4813
4814     # list item
4815     list_item_match = re.search(r'^([ ]{0,4})\d+[.][ ]+(.*)', line)
4816     if list_item_match:
4817       md_blocks.append(md_block)
4818       indentation = list_item_match.group(1)
4819       md_block = {'type': "li",
4820                   'ordered': 1,
4821                   'indentation': indentation,
4822                   'marker': "\\d+[.]",
4823                   'first': 1,
4824                   'last': 1,
4825                   'lines': [re.sub(r'^[ ]{0,4}', '', list_item_match.group(2))],
4826                   }
4827       continue
4828
4829     # paragraph
4830     if md_block["type"] == "paragraph":
4831       if "interrupted" in md_block:
4832         md_blocks.append(md_block)
4833         md_block = {'type': "paragraph",
4834                     'text': line,
4835                     }
4836         logging.info("new paragraph due to interrupted")
4837       else:
4838         md_block["text"] += "\n" + line
4839         logging.info("add to paragraph: '%s'", line)
4840
4841     else:
4842       md_blocks.append(md_block)
4843       md_block = {'type': "paragraph",
4844                   'text': line,
4845                   }
4846       logging.info("new paragraph due to different block type")
4847
4848   logging.info('done')
4849   md_blocks.append(md_block)
4850   md_blocks.pop(0)
4851
4852   return md_blocks
4853
4854
4855 def MarkDownParseSpanElementsInner(text, markersref):
4856   markup = ''
4857   markers = {i: 1 for i in markersref}
4858
4859   while text != '':
4860     closest_marker = ''
4861     closest_marker_position = -1
4862     text_marker = ''
4863     offset = 0
4864     markers_rest = []
4865
4866     for marker, use in markers.items():
4867       if not use:
4868         continue
4869
4870       marker_position = text.find(marker)
4871
4872       if marker_position < 0:
4873         markers[marker] = 0
4874         continue
4875
4876       if closest_marker == '' or marker_position < closest_marker_position:
4877         closest_marker = marker
4878         closest_marker_position = marker_position
4879
4880     if closest_marker_position >= 0:
4881       text_marker = text[closest_marker_position:]
4882
4883     if text_marker == '':
4884       markup += text
4885       text = ''
4886       continue # last
4887
4888     markup += text[:closest_marker_position]
4889     text = text[closest_marker_position:]
4890     markers_rest = {k: v for k, v in markers.items() if v and k != closest_marker}
4891
4892     if closest_marker == '![' or closest_marker == '[':
4893       element = None
4894
4895       ## FIXME: '(?R)' is a recursive subpattern
4896       # \[((?:[^][]|(?R))*)\]
4897       # match a [...] block with no ][ inside or this thing again
4898       # m = re.search(r'\[((?:[^][]|(?R))*)\]', text)
4899       m = re.search(  r'\[((?:[^][])*)\]', text)
4900       if ']' in text and m:
4901         element = {'!': text[0] == '!',
4902                    'a': m.group(1),
4903                    }
4904
4905         offset = len(m.group(0))
4906         if element['!']:
4907           offset += 1
4908         logging.debug("Recursive md-expr match: off=%d, text='%s', match='%s'", offset, text, m.group(1))
4909
4910         remaining_text = text[offset:]
4911         m2 = re.search(r'''^\([ ]*([^)'"]*?)(?:[ ]+['"](.+?)['"])?[ ]*\)''', remaining_text)
4912         m3 = re.search(r'^\s*\[([^\]<]*?)\]', remaining_text)
4913         if m2:
4914           element['»'] = m2.group(1)
4915           if m2.group(2):
4916             element['#'] = m2.group(2)
4917           offset += len(m2.group(0))
4918         elif m3:
4919           element['ref'] = m3.group(1)
4920           offset += len(m3.group(0))
4921         else:
4922           element = None
4923
4924       if element:
4925         if '»' in element:
4926           element['»'] = element['»'].replace('&', '&amp;').replace('<', '&lt;')
4927
4928         if element['!']:
4929           markup += '<inlinemediaobject><imageobject><imagedata fileref="' + element['»'] + '"></imagedata></imageobject>'
4930
4931           if 'a' in element:
4932             markup += "<textobject><phrase>" + element['a'] + "</phrase></textobject>"
4933
4934             markup += "</inlinemediaobject>"
4935         elif 'ref' in element:
4936           element['a'] = MarkDownParseSpanElementsInner(element['a'], markers_rest)
4937           markup += '<link linkend="' + element['ref'] + '"'
4938
4939           if '#' in element:
4940             # title attribute not supported
4941             pass
4942
4943           markup += '>' + element['a'] + "</link>"
4944         else:
4945           element['a'] = MarkDownParseSpanElementsInner(element['a'], markers_rest)
4946           markup += '<ulink url="' + element['»'] + '"'
4947
4948           if '#' in element:
4949             # title attribute not supported
4950             pass
4951
4952           markup += '>' + element['a'] + "</ulink>"
4953
4954       else:
4955         markup += closest_marker
4956         if closest_marker == '![':
4957           offset = 2
4958         else:
4959           offset = 1
4960
4961     elif closest_marker == '<':
4962       m4 = re.search(r'^<(https?:[\/]{2}[^\s]+?)>', text, flags=re.I)
4963       m5 = re.search(r'^<([A-Za-z0-9._-]+?@[A-Za-z0-9._-]+?)>', text)
4964       m6 = re.search(r'^<[^>]+?>', text)
4965       if m4:
4966         element_url = m4.group(1).replace('&', '&amp;').replace('<', '&lt;')
4967
4968         markup += '<ulink url="' + element_url + '">' + element_url + '</ulink>'
4969         offset = len(m4.group(0))
4970       elif m5:
4971         markup += "<ulink url=\"mailto:" + m5.group(1) + "\">" + m5.group(1) + "</ulink>"
4972         offset = len(m5.group(0))
4973       elif m6:
4974         markup += m6.group(0)
4975         offset = len(m6.group(0))
4976       else:
4977         markup += "&lt;"
4978         offset = 1
4979
4980     elif closest_marker == "\\":
4981       special_char = ''
4982       if len(text) > 1:
4983           special_char = text[1]
4984       if special_char in MD_ESCAPABLE_CHARS or special_char in MD_GTK_ESCAPABLE_CHARS:
4985         markup += special_char
4986         offset = 2
4987       else:
4988         markup += "\\"
4989         offset = 1
4990
4991     elif closest_marker == "`":
4992       m7 = re.search(r'^(`+)([^`]+?)\1(?!`)', text)
4993       if m7:
4994         element_text = m7.group(2)
4995         markup += "<literal>" + element_text + "</literal>"
4996         offset = len(m7.group(0))
4997       else:
4998         markup += "`"
4999         offset = 1
5000
5001     elif closest_marker == "@":
5002       # Convert '@param()'
5003       # FIXME: we could make those also links ($symbol.$2), but that would be less
5004       # useful as the link target is a few lines up or down
5005       m7 = re.search(r'^(\A|[^\\])\@(\w+((\.|->)\w+)*)\s*\(\)', text)
5006       m8 = re.search(r'^(\A|[^\\])\@(\w+((\.|->)\w+)*)', text)
5007       m9 = re.search(r'^\\\@', text)
5008       if m7:
5009         markup += m7.group(1) + "<parameter>" + m7.group(2) + "()</parameter>\n"
5010         offset = len(m7.group(0))
5011       elif m8:
5012         # Convert '@param', but not '\@param'.
5013         markup += m8.group(1) + "<parameter>" + m8.group(2) + "</parameter>\n"
5014         offset = len(m8.group(0))
5015       elif m9:
5016         markup += r"\@"
5017         offset = len(m9.group(0))
5018       else:
5019         markup += "@"
5020         offset = 1
5021
5022     elif closest_marker == '#':
5023       m10 = re.search(r'^(\A|[^\\])#([\w\-:\.]+[\w]+)\s*\(\)', text)
5024       m11 = re.search(r'^(\A|[^\\])#([\w\-:\.]+[\w]+)', text)
5025       m12 = re.search(r'^\\#', text)
5026       if m10:
5027         # handle #Object.func()
5028         markup += m10.group(1) + MakeXRef(m10.group(2), tagify(m10.group(2) + "()", "function"))
5029         offset = len(m10.group(0))
5030       elif m11:
5031         # Convert '#symbol', but not '\#symbol'.
5032         markup += m11.group(1) + MakeHashXRef(m11.group(2), "type")
5033         offset = len(m11.group(0))
5034       elif m12:
5035         markup += '#'
5036         offset = len(m12.group(0))
5037       else:
5038         markup += '#'
5039         offset = 1
5040
5041     elif closest_marker == "%":
5042       m12 = re.search(r'^(\A|[^\\])\%(-?\w+)', text)
5043       m13 = re.search(r'^\\%', text)
5044       if m12:
5045         # Convert '%constant', but not '\%constant'.
5046         # Also allow negative numbers, e.g. %-1.
5047         markup += m12.group(1) + MakeXRef(m12.group(2), tagify(m12.group(2), "literal"))
5048         offset = len(m12.group(0))
5049       elif m13:
5050         markup += r"\%"
5051         offset = len(m13.group(0))
5052       else:
5053         markup += "%"
5054         offset = 1
5055
5056     if offset > 0:
5057       text = text[offset:]
5058
5059   return markup
5060
5061
5062 def MarkDownParseSpanElements(text):
5063     markers = ["\\", '<', '![', '[', "`", '%', '#', '@']
5064
5065     text = MarkDownParseSpanElementsInner(text, markers)
5066
5067     # Convert 'function()' or 'macro()'.
5068     # if there is abc_*_def() we don't want to make a link to _def()
5069     # FIXME: also handle abc(def(....)) : but that would need to be done recursively :/
5070     def f(m):
5071         return m.group(1) + MakeXRef(m.group(2), tagify(m.group(2) + "()", "function"))
5072     text = re.sub(r'([^\*.\w])(\w+)\s*\(\)', f, text)
5073     return text
5074
5075
5076 def ReplaceEntities(text, symbol):
5077     entities = [["&lt;", '<'],
5078                 ["&gt;", '>'],
5079                 ["&ast;", '*'],
5080                 ["&num;", '#'],
5081                 ["&percnt;", '%'],
5082                 ["&colon;", ':'],
5083                 ["&quot;", '"'],
5084                 ["&apos;", "'"],
5085                 ["&nbsp;", ' '],
5086                 ["&amp;", '&'], # Do this last, or the others get messed up.
5087                ]
5088
5089     # Expand entities in <programlisting> even inside CDATA since
5090     # we changed the definition of |[ to add CDATA
5091     for i in entities:
5092         text = re.sub(i[0], i[1], text)
5093     return text
5094
5095
5096 def MarkDownOutputDocBook(blocksref, symbol, context):
5097   output = ''
5098   blocks = blocksref
5099
5100   for block in blocks:
5101     #$output += "\n<!-- beg type='" . $block->{"type"} . "'-->\n"
5102
5103     if block["type"] == "paragraph":
5104       text = MarkDownParseSpanElements(block["text"])
5105       if context == "li" and output == '':
5106         if 'interrupted' in block:
5107           output += "\n<para>%s</para>\n" % text
5108         else:
5109           output += "<para>%s</para>" % text
5110           if len(blocks) > 1:
5111             output += "\n"
5112       else:
5113         output += "<para>%s</para>\n" % text
5114
5115     elif block["type"] == "heading":
5116
5117       title = MarkDownParseSpanElements(block["text"])
5118
5119       if block["level"] == 1:
5120         tag = "refsect2"
5121       else:
5122         tag = "refsect3"
5123
5124       text = MarkDownParseLines(block["lines"], symbol, "heading")
5125       if 'id' in block:
5126         output += "<%s id=\"%s\">" % (tag, block["id"])
5127       else:
5128         output += "<%s>" % tag
5129
5130       output += "<title>%s</title>%s</%s>\n" % (title, text, tag)
5131     elif block["type"] == "li":
5132       tag = "itemizedlist"
5133
5134       if "first" in block:
5135         if block["ordered"]:
5136           tag = "orderedlist"
5137         output += "<%s>\n" % tag
5138
5139       if "interrupted" in block:
5140         block["lines"].append('')
5141
5142       text = MarkDownParseLines(block["lines"], symbol, "li")
5143       output += "<listitem>" + text + "</listitem>\n"
5144       if 'last' in block:
5145         if block["ordered"]:
5146           tag = "orderedlist"
5147         output += "</%s>\n" % tag
5148
5149     elif block["type"] == "quote":
5150       text = MarkDownParseLines(block["lines"], symbol, "quote")
5151       output += "<blockquote>\n%s</blockquote>\n" % text
5152     elif block["type"] == "code":
5153       tag = "programlisting"
5154
5155       if "language" in block:
5156         if block["language"] == "plain":
5157           output += "<informalexample><screen><![CDATA[\n"
5158           tag = "screen"
5159         else:
5160           output += "<informalexample><programlisting language=\"%s\"><![CDATA[\n" % block['language']
5161       else:
5162         output += "<informalexample><programlisting><![CDATA[\n"
5163
5164       logging.debug('listing for %s: [%s]', symbol, '\n'.join(block['lines']))
5165       for line in block["lines"]:
5166         output += ReplaceEntities(line, symbol) + "\n"
5167
5168       output += "]]></%s></informalexample>\n" % tag
5169     elif block["type"] == "markup":
5170       text = ExpandAbbreviations(symbol, block["text"])
5171       output += text + "\n"
5172     else:
5173       output += block["text"] + "\n"
5174
5175     #$output += "\n<!-- end type='" . $block->{"type"} . "'-->\n"
5176   return output
5177
5178
5179 def MarkDownParseLines(lines, symbol, context):
5180     logging.info('md parse: ctx=%s, [%s]', context, '\n'.join(lines))
5181     blocks = MarkDownParseBlocks(lines, symbol, context)
5182     output = MarkDownOutputDocBook(blocks, symbol, context)
5183     return output
5184
5185
5186 def MarkDownParse(text, symbol):
5187     return MarkDownParseLines(text.splitlines(), symbol, '')
5188
5189
5190 #############################################################################
5191 # Function    : ReadDeclarationsFile
5192 # Description : This reads in a file containing the function/macro/enum etc.
5193 #                declarations.
5194 #
5195 #                Note that in some cases there are several declarations with
5196 #                the same name, e.g. for conditional macros. In this case we
5197 #                set a flag in the %DeclarationConditional hash so the
5198 #                declaration is not shown in the docs.
5199 #
5200 #                If a macro and a function have the same name, e.g. for
5201 #                gtk_object_ref, the function declaration takes precedence.
5202 #
5203 #                Some opaque structs are just declared with 'typedef struct
5204 #                _name name;' in which case the declaration may be empty.
5205 #                The structure may have been found later in the header, so
5206 #                that overrides the empty declaration.
5207 #
5208 # Arguments   : $file - the declarations file to read
5209 #                $override - if declarations in this file should override
5210 #                        any current declaration.
5211 #############################################################################
5212
5213 def ReadDeclarationsFile(ifile, override):
5214
5215     if override == 0:
5216         global Declarations, DeclarationTypes, DeclarationConditional, DeclarationOutput
5217         Declarations = {}
5218         DeclarationTypes = {}
5219         DeclarationConditional = {}
5220         DeclarationOutput = {}
5221
5222     INPUT = open(ifile)
5223     declaration_type = ''
5224     declaration_name = None
5225     declaration = None
5226     is_deprecated = 0
5227     line_number = 0
5228     for line in INPUT:
5229         line_number += 1
5230         if not declaration_type:
5231             m1 = re.search(r'^<([^>]+)>', line)
5232             if m1:
5233                 declaration_type = m1.group(1)
5234                 declaration_name = ''
5235                 logging.info("Found declaration: %s", declaration_type)
5236                 declaration = ''
5237
5238         else:
5239             m2 = re.search(r'^<NAME>(.*)</NAME>', line)
5240             m3 = re.search(r'^<DEPRECATED/>', line)
5241             m4 = re.search(r'^</%s>' % declaration_type, line)
5242             if m2:
5243                 declaration_name = m2.group(1)
5244             elif m3:
5245                 is_deprecated = True
5246             elif m4:
5247                 logging.info("Found end of declaration: %s, %s", declaration_type, declaration_name)
5248                 # Check that the declaration has a name
5249                 if declaration_name == '':
5250                     common.LogWarning(ifile, line_number, declaration_type + " has no name.\n")
5251
5252                 # If the declaration is an empty typedef struct _XXX XXX
5253                 # set the flag to indicate the struct has a typedef.
5254                 if (declaration_type == 'STRUCT' or declaration_type == 'UNION') \
5255                     and re.search(r'^\s*$', declaration):
5256                     logging.info("Struct has typedef: %s", declaration_name)
5257                     StructHasTypedef[declaration_name] = 1
5258
5259                 # Check if the symbol is already defined.
5260                 if declaration_name in Declarations and override == 0:
5261                     # Function declarations take precedence.
5262                     if DeclarationTypes[declaration_name] == 'FUNCTION':
5263                         # Ignore it.
5264                         pass
5265                     elif declaration_type == 'FUNCTION':
5266                         if is_deprecated:
5267                             Deprecated[declaration_name] = ''
5268
5269                         Declarations[declaration_name] = declaration
5270                         DeclarationTypes[declaration_name] = declaration_type
5271                     elif DeclarationTypes[declaration_name] == declaration_type:
5272                         # If the existing declaration is empty, or is just a
5273                         # forward declaration of a struct, override it.
5274                         if declaration_type == 'STRUCT' or declaration_type == 'UNION':
5275                             if re.search(r'^\s*((struct|union)\s+\w+\s*;)?\s*$', Declarations[declaration_name]):
5276                                 if is_deprecated:
5277                                     Deprecated[declaration_name] = ''
5278                                 Declarations[declaration_name] = declaration
5279                             elif re.search(r'^\s*((struct|union)\s+\w+\s*;)?\s*$', declaration):
5280                                 # Ignore an empty or forward declaration.
5281                                 pass
5282                             else:
5283                                 common.LogWarning(ifile, line_number, "Structure %s has multiple definitions." % declaration_name)
5284
5285                         else:
5286                             # set flag in %DeclarationConditional hash for
5287                             # multiply defined macros/typedefs.
5288                             DeclarationConditional[declaration_name] = 1
5289
5290                     else:
5291                         common.LogWarning(ifile, line_number, declaration_name + " has multiple definitions.")
5292
5293                 else:
5294                     if is_deprecated:
5295                         Deprecated[declaration_name] = ''
5296
5297                     Declarations[declaration_name] = declaration
5298                     DeclarationTypes[declaration_name] = declaration_type
5299                     logging.info("added declaration: %s, %s, [%s]", declaration_type, declaration_name, declaration)
5300
5301                 declaration_type = ''
5302                 is_deprecated = False
5303             else:
5304                 declaration += line
5305     INPUT.close()
5306
5307
5308 #############################################################################
5309 # Function    : ReadSignalsFile
5310 # Description : This reads in an existing file which contains information on
5311 #                all GTK signals. It creates the arrays @SignalNames and
5312 #                @SignalPrototypes containing info on the signals. The first
5313 #                line of the SignalPrototype is the return type of the signal
5314 #                handler. The remaining lines are the parameters passed to it.
5315 #                The last parameter, "gpointer user_data" is always the same
5316 #                so is not included.
5317 # Arguments   : $file - the file containing the signal handler prototype
5318 #                        information.
5319 #############################################################################
5320
5321 def ReadSignalsFile(ifile):
5322
5323     in_signal = 0
5324     signal_object = None
5325     signal_name = None
5326     signal_returns = None
5327     signal_flags = None
5328     signal_prototype = None
5329
5330     # Reset the signal info.
5331     global SignalObjects, SignalNames, SignalReturns, SignalFlags, SignalPrototypes
5332     SignalObjects = []
5333     SignalNames = []
5334     SignalReturns = []
5335     SignalFlags = []
5336     SignalPrototypes = []
5337
5338     if not os.path.isfile(ifile):
5339         return
5340
5341     INPUT = open(ifile)
5342     line_number = 0
5343     for line in INPUT:
5344         line_number += 1
5345         if not in_signal:
5346             if re.search(r'^<SIGNAL>', line):
5347                 in_signal = 1
5348                 signal_object = ''
5349                 signal_name = ''
5350                 signal_returns = ''
5351                 signal_prototype = ''
5352
5353         else:
5354             m = re.search(r'^<NAME>(.*)<\/NAME>', line)
5355             m2 = re.search(r'^<RETURNS>(.*)<\/RETURNS>', line)
5356             m3 = re.search(r'^<FLAGS>(.*)<\/FLAGS>', line)
5357             if m:
5358                 signal_name = m.group(1)
5359                 m1_2 = re.search(r'^(.*)::(.*)$', signal_name)
5360                 if m1_2:
5361                     signal_object = m1_2.group(1)
5362                     signal_name = m1_2.group(2).replace('_', '-')
5363                     logging.info("Found signal: %s", signal_name)
5364                 else:
5365                     common.LogWarning(ifile, line_number, "Invalid signal name: %s." % signal_name)
5366
5367             elif m2:
5368                 signal_returns = m2.group(1)
5369             elif m3:
5370                 signal_flags = m3.group(1)
5371             elif re.search(r'^</SIGNAL>', line):
5372                 logging.info("Found end of signal: %s::%s\nReturns: %s\n%s", signal_object, signal_name, signal_returns, signal_prototype)
5373                 SignalObjects.append(signal_object)
5374                 SignalNames.append(signal_name)
5375                 SignalReturns.append(signal_returns)
5376                 SignalFlags.append(signal_flags)
5377                 SignalPrototypes.append(signal_prototype)
5378                 in_signal = False
5379             else:
5380                 signal_prototype += line
5381     INPUT.close()
5382
5383
5384
5385 #############################################################################
5386 # Function    : ReadTemplateFile
5387 # Description : This reads in the manually-edited documentation file
5388 #               corresponding to the file currently being created, so we can
5389 #               insert the documentation at the appropriate places.
5390 #               It outputs %SymbolTypes, %SymbolDocs and %SymbolParams, which
5391 #               is a hash of arrays.
5392 # Arguments   : $docsfile - the template file to read in.
5393 #               $skip_unused_params - 1 if the unused parameters should be
5394 #                 skipped.
5395 #############################################################################
5396
5397 def ReadTemplateFile(docsfile, skip_unused_params):
5398
5399     template = docsfile + ".sgml"
5400     if not os.path.isfile(template):
5401         logging.info("File doesn't exist: " + template)
5402         return 0
5403
5404
5405     # start with empty hashes, we merge the source comment for each file
5406     # afterwards
5407     global SymbolDocs, SymbolTypes, SymbolParams
5408     SymbolDocs = {}
5409     SymbolTypes = {}
5410     SymbolParams = {}
5411
5412     current_type = ''          # Type of symbol being read.
5413     current_symbol = ''        # Name of symbol being read.
5414     symbol_doc = ''            # Description of symbol being read.
5415     params = {}                # Parameter names and descriptions of current function/macro/function typedef.
5416     param_name = None          # Parameter name
5417     in_unused_params = 0       # True if we are reading in the unused params.
5418     in_deprecated = 0
5419     in_since = 0
5420     in_stability = 0
5421
5422     DOCS = open(template)
5423
5424     logging.info("reading template " + template)
5425
5426     line_number = 0
5427     for line in DOCS:
5428         line_number += 1
5429         m1 = re.search(r'^<!-- ##### ([A-Z_]+) (\S+) ##### -->', line)
5430         if m1:
5431             stype = m1.group(1)
5432             symbol = m1.group(2)
5433             if symbol == "Title" \
5434                 or symbol == "Short_Description" \
5435                 or symbol == "Long_Description" \
5436                 or symbol == "See_Also" \
5437                 or symbol == "Stability_Level" \
5438                 or symbol == "Include" \
5439                 or symbol == "Image":
5440
5441                 symbol = docsfile + ":" + symbol
5442
5443
5444             logging.info("Found symbol: " + symbol)
5445             # Remember file and line for the symbol
5446             SymbolSourceFile[symbol] = template
5447             SymbolSourceLine[symbol] = line_number
5448
5449             # Store previous symbol, but remove any trailing blank lines.
5450             if current_symbol != '':
5451                 symbol_doc = symbol_doc.rstrip()
5452                 SymbolTypes[current_symbol] = current_type
5453                 SymbolDocs[current_symbol] = symbol_doc
5454
5455                 # Check that the stability level is valid.
5456                 if StabilityLevel[current_symbol]:
5457                     StabilityLevel[current_symbol] = ParseStabilityLevel(StabilityLevel[current_symbol], template, line_number, "Stability level for " + current_symbol)
5458
5459                 if param_name:
5460                     SymbolParams[current_symbol] = params
5461                 else:
5462                     # Delete any existing params in case we are overriding a
5463                     # previously read template.
5464                     del SymbolParams[current_symbol]
5465
5466
5467             current_type = stype
5468             current_symbol = symbol
5469             in_unused_params = 0
5470             in_deprecated = 0
5471             in_since = 0
5472             in_stability = 0
5473             symbol_doc = ''
5474             params = {}
5475             param_name = None
5476
5477         elif re.search(r'^<!-- # Unused Parameters # -->', line):
5478             logging.info("Found unused parameters")
5479             in_unused_params = True
5480             continue
5481
5482         elif in_unused_params and skip_unused_params:
5483             # When outputting the DocBook we skip unused parameters.
5484             logging.info("Skipping unused param: " + line)
5485             continue
5486
5487         else:
5488             # Check if param found. Need to handle "..." and "format...".
5489             m2 = re.search(r'^\@([\w\.]+):\040?', line)
5490             if m2:
5491                 line = re.sub(r'^\@([\w\.]+):\040?', '', line)
5492                 param_name = m2.group(1)
5493                 param_desc = line
5494                 # Allow variations of 'Returns'
5495                 if re.search(r'^[Rr]eturns?$', param_name):
5496                     param_name = "Returns"
5497
5498                 # Allow varargs variations
5499                 if re.search(r'^.*\.\.\.$', param_name):
5500                     param_name = "..."
5501
5502
5503                 # strip trailing whitespaces and blank lines
5504                 line = re.sub(r'\s+\n$', '\n', line, flags=re.M)
5505                 line = re.sub(r'\n+$', '\n', line, flags=re.M|re.S)
5506                 logging.info("Found param for symbol %s : '%s'= '%s'", current_symbol, param_name, line)
5507
5508                 if param_name == "Deprecated":
5509                     in_deprecated = True
5510                     Deprecated[current_symbol] = line
5511                 elif param_name == "Since":
5512                     in_since = True
5513                     Since[current_symbol] = line.strip()
5514                 elif param_name == "Stability":
5515                     in_stability = True
5516                     StabilityLevel[current_symbol] = line
5517                 else:
5518                     params[param_name] = param_desc
5519
5520             else:
5521                 # strip trailing whitespaces and blank lines
5522                 line = re.sub(r'\s+\n$', '\n', line, flags=re.M)
5523                 line = re.sub(r'\n+$', '\n', line, flags=re.M|re.S)
5524
5525                 if re.search(r'^\s+$', line):
5526                     if in_deprecated:
5527                         Deprecated[current_symbol] += line
5528                     elif in_since:
5529                         common.LogWarning(template, line_number, "multi-line since docs found")
5530                         #$Since{$current_symbol} += $_
5531                     elif in_stability:
5532                         StabilityLevel[current_symbol] += line
5533                     elif param_name:
5534                         params[param_name] += line
5535                     else:
5536                         symbol_doc += line
5537
5538     # Remember to finish the current symbol doccs.
5539     if current_symbol != '':
5540
5541         symbol_doc = re.sub(r'\s+$', '', symbol_doc)
5542         SymbolTypes[current_symbol] = current_type
5543         SymbolDocs[current_symbol] = symbol_doc
5544
5545         # Check that the stability level is valid.
5546         if StabilityLevel[current_symbol]:
5547             StabilityLevel[current_symbol] = ParseStabilityLevel(StabilityLevel[current_symbol], template, line_number, "Stability level for " + current_symbol)
5548
5549         if param_name:
5550             SymbolParams[current_symbol] = params
5551         else:
5552             # Delete any existing params in case we are overriding a
5553             # previously read template.
5554             del SymbolParams[current_symbol]
5555
5556     DOCS.close()
5557     return 1
5558
5559
5560
5561 #############################################################################
5562 # Function    : ReadObjectHierarchy
5563 # Description : This reads in the $MODULE-hierarchy.txt file containing all
5564 #               the GtkObject subclasses described in this module (and their
5565 #               ancestors).
5566 #               It places them in the @Objects array, and places their level
5567 #               in the object hierarchy in the @ObjectLevels array, at the
5568 #               same index. GtkObject, the root object, has a level of 1.
5569 #
5570 #               This also generates tree_index.sgml as it goes along.
5571 #
5572 # Arguments   : none
5573 #############################################################################
5574
5575 def ReadObjectHierarchy():
5576
5577     global Objects, ObjectLevels
5578     Objects = []
5579     ObjectLevels = []
5580
5581     if not os.path.isfile(OBJECT_TREE_FILE):
5582         return
5583
5584     INPUT = open(OBJECT_TREE_FILE)
5585
5586     # Only emit objects if they are supposed to be documented, or if
5587     # they have documented children. To implement this, we maintain a
5588     # stack of pending objects which will be emitted if a documented
5589     # child turns up.
5590     pending_objects = []
5591     pending_levels = []
5592     root = None
5593     tree = []
5594     for line in INPUT:
5595         m1 = re.search(r'\S+', line)
5596         if not m1:
5597             continue
5598
5599         gobject = m1.group(0)
5600         level = len(line[:m1.start()]) / 2 + 1
5601
5602         if level == 1:
5603             root = gobject
5604
5605         while pending_levels and pending_levels[-1] >= level:
5606             pending_objects.pop()
5607             pending_levels.pop()
5608
5609         pending_objects.append(gobject)
5610         pending_levels.append(level)
5611
5612         if gobject in KnownSymbols:
5613             while len(pending_levels) > 0:
5614                 gobject = pending_objects.pop(0)
5615                 level = pending_levels.pop(0)
5616                 xref = MakeXRef(gobject)
5617
5618                 tree.append(' ' * (level * 4) + xref)
5619                 Objects.append(gobject)
5620                 ObjectLevels.append(level)
5621                 ObjectRoots[gobject] = root
5622         #else
5623         #    common.LogWarning($OBJECT_TREE_FILE, line_number, "unknown type $object")
5624         #
5625
5626     INPUT.close()
5627
5628     # FIXME: use xml
5629     # my $old_tree_index = "$DB_OUTPUT_DIR/tree_index.$xml"
5630     old_tree_index = os.path.join(DB_OUTPUT_DIR, "tree_index.sgml")
5631     new_tree_index = os.path.join(DB_OUTPUT_DIR, "tree_index.new")
5632
5633     logging.info('got %d entries for hierarchy', len(tree))
5634
5635     OUTPUT = open(new_tree_index, 'w')
5636     OUTPUT.write(MakeDocHeader("screen") + "\n<screen>\n" + AddTreeLineArt(tree) + "\n</screen>\n")
5637     OUTPUT.close()
5638
5639     common.UpdateFileIfChanged(old_tree_index, new_tree_index, 0)
5640
5641     OutputObjectList()
5642
5643
5644 #############################################################################
5645 # Function    : ReadInterfaces
5646 # Description : This reads in the $MODULE.interfaces file.
5647 #
5648 # Arguments   : none
5649 #############################################################################
5650
5651 def ReadInterfaces():
5652     global Interfaces
5653     Interfaces = {}
5654
5655     if not os.path.isfile(INTERFACES_FILE):
5656         return
5657
5658     INPUT = open(INTERFACES_FILE)
5659
5660     for line in INPUT:
5661         line = line.strip()
5662         ifaces = line.split()
5663         gobject = ifaces.pop(0)
5664         if gobject in KnownSymbols and KnownSymbols[gobject] == 1:
5665             knownIfaces = []
5666
5667             # filter out private interfaces, but leave foreign interfaces
5668             for iface in ifaces:
5669                 if iface not in KnownSymbols or KnownSymbols[iface] == 1:
5670                     knownIfaces.append(iface)
5671
5672             Interfaces[gobject] = ' '.join(knownIfaces)
5673             logging.info("Interfaces for %s: %s", gobject, Interfaces[gobject])
5674         else:
5675             logging.info("skipping interfaces for unknown symbol: %s", gobject)
5676
5677     INPUT.close()
5678
5679
5680 #############################################################################
5681 # Function    : ReadPrerequisites
5682 # Description : This reads in the $MODULE.prerequisites file.
5683 #
5684 # Arguments   : none
5685 #############################################################################
5686
5687 def ReadPrerequisites():
5688     global Prerequisites
5689     Prerequisites = {}
5690
5691     if not os.path.isfile(PREREQUISITES_FILE):
5692         return
5693
5694     INPUT = open(PREREQUISITES_FILE)
5695
5696     for line in INPUT:
5697         line = line.strip()
5698         prereqs = line.split()
5699         iface = prereqs.pop(0)
5700         if iface in KnownSymbols and KnownSymbols[iface] == 1:
5701             knownPrereqs = []
5702
5703             # filter out private prerequisites, but leave foreign prerequisites
5704             for prereq in prereqs:
5705                 if prereq not in KnownSymbols or KnownSymbols[prereq] == 1:
5706                     knownPrereqs.append(prereq)
5707
5708             Prerequisites[iface] = ' '.join(knownPrereqs)
5709
5710     INPUT.close()
5711
5712
5713 #############################################################################
5714 # Function    : ReadArgsFile
5715 # Description : This reads in an existing file which contains information on
5716 #                all GTK args. It creates the arrays @ArgObjects, @ArgNames,
5717 #                @ArgTypes, @ArgFlags, @ArgNicks and @ArgBlurbs containing info
5718 #               on the args.
5719 # Arguments   : $file - the file containing the arg information.
5720 #############################################################################
5721
5722 def ReadArgsFile(ifile):
5723     in_arg = False
5724     arg_object = None
5725     arg_name = None
5726     arg_type = None
5727     arg_flags = None
5728     arg_nick = None
5729     arg_blurb = None
5730     arg_default = None
5731     arg_range = None
5732
5733     # Reset the args info.
5734     global ArgObjects, ArgNames, ArgTypes, ArgFlags, ArgNicks, ArgBlurbs, ArgDefaults, ArgRanges
5735     ArgObjects = []
5736     ArgNames = []
5737     ArgTypes = []
5738     ArgFlags = []
5739     ArgNicks = []
5740     ArgBlurbs = []
5741     ArgDefaults = []
5742     ArgRanges = []
5743
5744     if not os.path.isfile(ifile):
5745         return
5746
5747     INPUT = open(ifile)
5748     line_number = 0
5749     for line in INPUT:
5750         line_number += 1
5751         if not in_arg:
5752             if line.startswith('<ARG>'):
5753                 in_arg = True
5754                 arg_object = ''
5755                 arg_name = ''
5756                 arg_type = ''
5757                 arg_flags = ''
5758                 arg_nick = ''
5759                 arg_blurb = ''
5760                 arg_default = ''
5761                 arg_range = ''
5762
5763         else:
5764             m1 = re.search(r'^<NAME>(.*)</NAME>', line)
5765             m2 = re.search(r'^<TYPE>(.*)</TYPE>', line)
5766             m3 = re.search(r'^<RANGE>(.*)</RANGE>', line)
5767             m4 = re.search(r'^<FLAGS>(.*)</FLAGS>', line)
5768             m5 = re.search(r'^<NICK>(.*)</NICK>', line)
5769             m6 = re.search(r'^<BLURB>(.*)</BLURB>', line)
5770             m7 = re.search(r'^<DEFAULT>(.*)</DEFAULT>', line)
5771             if m1:
5772                 arg_name = m1.group(1)
5773                 m1_1 = re.search(r'^(.*)::(.*)$', arg_name)
5774                 if m1_1:
5775                     arg_object = m1_1.group(1)
5776                     arg_name = m1_1.group(2).replace('_', '-')
5777                     logging.info("Found arg: %s", arg_name)
5778                 else:
5779                     common.LogWarning(ifile, line_number, "Invalid argument name: " + arg_name)
5780
5781             elif m2:
5782                 arg_type = m2.group(1)
5783             elif m3:
5784                 arg_range = m3.group(1)
5785             elif m4:
5786                 arg_flags = m4.group(1)
5787             elif m5:
5788                 arg_nick = m5.group(1)
5789             elif m6:
5790                 arg_blurb = m6.group(1)
5791                 if arg_blurb == "(null)":
5792                     arg_blurb = ''
5793                     common.LogWarning(ifile, line_number, "Property %s:%s has no documentation." % (arg_object, arg_name))
5794
5795             elif m7:
5796                 arg_default = m7.group(1)
5797             elif re.search(r'^</ARG>', line):
5798                 logging.info("Found end of arg: %s::%s\n%s : %s", arg_object, arg_name, arg_type, arg_flags)
5799                 ArgObjects.append(arg_object)
5800                 ArgNames.append(arg_name)
5801                 ArgTypes.append(arg_type)
5802                 ArgRanges.append(arg_range)
5803                 ArgFlags.append(arg_flags)
5804                 ArgNicks.append(arg_nick)
5805                 ArgBlurbs.append(arg_blurb)
5806                 ArgDefaults.append(arg_default)
5807                 in_arg = False
5808
5809     INPUT.close()
5810
5811
5812 #############################################################################
5813 # Function    : AddTreeLineArt
5814 # Description : Add unicode lineart to a pre-indented string array and returns
5815 #               it as as multiline string.
5816 # Arguments   : @tree - array of indented strings.
5817 #############################################################################
5818
5819 def AddTreeLineArt(tree):
5820     # iterate bottom up over the tree
5821     for i in range(len(tree) - 1, -1, -1):
5822         # count leading spaces
5823         m = re.search(r'^([^<A-Za-z]*)', tree[i])
5824         indent = len(m.group(1))
5825         # replace with ╰───, if place of ╰ is not space insert ├
5826         if indent > 4:
5827             if tree[i][indent-4] == " ":
5828                 tree[i] = tree[i][:indent-4] + "--- " + tree[i][indent:]
5829             else:
5830                 tree[i] = tree[i][:indent-4] + "+-- " + tree[i][indent:]
5831
5832             # go lines up while space and insert |
5833             j = i - 1
5834             while j >= 0 and tree[j][indent-4] == ' ':
5835                 tree[j] = tree[j][:indent-4] + '|' + tree[j][indent-3:]
5836                 j -= 1
5837
5838     res = "\n".join(tree)
5839     # unicode chars for: ╰──
5840     res = re.sub(r'---', '<phrase role=\"lineart\">&#9584;&#9472;&#9472;</phrase>', res)
5841     # unicde chars for: ├──
5842     res = re.sub(r'\+--', '<phrase role=\"lineart\">&#9500;&#9472;&#9472;</phrase>', res)
5843     # unicode char for: │
5844     res = re.sub(r'\|', '<phrase role=\"lineart\">&#9474;</phrase>', res)
5845
5846     return res
5847
5848
5849 #############################################################################
5850 # Function    : CheckIsObject
5851 # Description : Returns 1 if the given name is a GObject or a subclass.
5852 #                It uses the global @Objects array.
5853 #                Note that the @Objects array only contains classes in the
5854 #                current module and their ancestors - not all GObject classes.
5855 # Arguments   : $name - the name to check.
5856 #############################################################################
5857
5858 def CheckIsObject(name):
5859     root = ObjectRoots.get(name)
5860     # Let GBoxed pass as an object here to get -struct appended to the id
5861     # and prevent conflicts with sections.
5862     return root and root != 'GEnum' and root != 'GFlags'
5863
5864
5865
5866 #############################################################################
5867 # Function    : MakeReturnField
5868 # Description : Pads a string to $RETURN_TYPE_FIELD_WIDTH.
5869 # Arguments   : $str - the string to pad.
5870 #############################################################################
5871
5872 def MakeReturnField(s):
5873     return s + (' ' * (RETURN_TYPE_FIELD_WIDTH - len(s)))
5874
5875
5876 #############################################################################
5877 # Function    : GetSymbolSourceFile
5878 # Description : Get the filename where the symbol docs where taken from.
5879 # Arguments   : $symbol - the symbol name
5880 #############################################################################
5881
5882 def GetSymbolSourceFile(symbol):
5883     if symbol in SourceSymbolSourceFile:
5884         return SourceSymbolSourceFile[symbol]
5885     if symbol in SymbolSourceFile:
5886         return SymbolSourceFile[symbol]
5887     return ''
5888
5889 #############################################################################
5890 # Function    : GetSymbolSourceLine
5891 # Description : Get the file line where the symbol docs where taken from.
5892 # Arguments   : $symbol - the symbol name
5893 #############################################################################
5894
5895 def GetSymbolSourceLine(symbol):
5896     if symbol in SourceSymbolSourceLine:
5897         return SourceSymbolSourceLine[symbol]
5898     if symbol in SymbolSourceLine:
5899         return SymbolSourceLine[symbol]
5900     return 0
5901
5902
5903 if __name__ == '__main__':
5904     Run()