gtkdoc-mkdb.in

   1 #!@PYTHON@
   2 # -*- python; coding: utf-8 -*-
   3 #
   4 # gtk-doc - GTK DocBook documentation generator.
   5 # Copyright (C) 1998  Damon Chaplin
   6 #               2007-2016  Stefan Sauer
   7 #
   8 # This program is free software; you can redistribute it and/or modify
   9 # it under the terms of the GNU General Public License as published by
  10 # the Free Software Foundation; either version 2 of the License, or
  11 # (at your option) any later version.
  12 #
  13 # This program is distributed in the hope that it will be useful,
  14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 # GNU General Public License for more details.
  17 #
  18 # You should have received a copy of the GNU General Public License
  19 # along with this program; if not, write to the Free Software
  20 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  21 #
  22
  23 #############################################################################
  24 # Script      : gtkdoc-mkdb
  25 # Description : This creates the DocBook files from the edited templates.
  26 #############################################################################
  27
  28 from __future__ import print_function
  29
  30 import argparse
  31 import logging
  32 import os
  33 import re
  34 import sys
  35
  36 from gtkdoc import common
  37
  38 # Options
  39
  40 # name of documentation module
  41 MODULE = None
  42 TMPL_DIR = None
  43 DB_OUTPUT_DIR = None
  44 SOURCE_DIRS = None
  45 SOURCE_SUFFIXES = ''
  46 IGNORE_FILES = ''
  47 MAIN_SGML_FILE = None
  48 EXPAND_CONTENT_FILES = ''
  49 INLINE_MARKUP_MODE = None
  50 DEFAULT_STABILITY = None
  51 DEFAULT_INCLUDES = None
  52 OUTPUT_FORMAT = None
  53 NAME_SPACE = ''
  54 OUTPUT_ALL_SYMBOLS = None
  55 OUTPUT_SYMBOLS_WITHOUT_SINCE = None
  56 ROOT_DIR = "."
  57 OBJECT_TREE_FILE = None
  58 INTERFACES_FILE = None
  59 PREREQUISITES_FILE = None
  60 SIGNALS_FILE = None
  61 ARGS_FILE = None
  62
  63 # These global arrays store information on signals. Each signal has an entry
  64 # in each of these arrays at the same index, like a multi-dimensional array.
  65 SignalObjects = []        # The GtkObject which emits the signal.
  66 SignalNames = []        # The signal name.
  67 SignalReturns = []        # The return type.
  68 SignalFlags = []        # Flags for the signal
  69 SignalPrototypes = []        # The rest of the prototype of the signal handler.
  70
  71 # These global arrays store information on Args. Each Arg has an entry
  72 # in each of these arrays at the same index, like a multi-dimensional array.
  73 ArgObjects = []                # The GtkObject which has the Arg.
  74 ArgNames = []                # The Arg name.
  75 ArgTypes = []                # The Arg type - gint, GtkArrowType etc.
  76 ArgFlags = []                # How the Arg can be used - readable/writable etc.
  77 ArgNicks = []                # The nickname of the Arg.
  78 ArgBlurbs = []          # Docstring of the Arg.
  79 ArgDefaults = []        # Default value of the Arg.
  80 ArgRanges = []                # The range of the Arg type
  81
  82 # These global hashes store declaration info keyed on a symbol name.
  83 Declarations = {}
  84 DeclarationTypes = {}
  85 DeclarationConditional = {}
  86 DeclarationOutput = {}
  87 Deprecated = {}
  88 Since = {}
  89 StabilityLevel = {}
  90 StructHasTypedef = {}
  91
  92 # These global hashes store the existing documentation.
  93 SymbolDocs = {}
  94 SymbolTypes = {}
  95 SymbolParams = {}
  96 SymbolSourceFile = {}
  97 SymbolSourceLine = {}
  98 SymbolAnnotations = {}
  99
 100 # These global hashes store documentation scanned from the source files.
 101 SourceSymbolDocs = {}
 102 SourceSymbolParams = {}
 103 SourceSymbolSourceFile = {}
 104 SourceSymbolSourceLine = {}
 105
 106 # all documentation goes in here, so we can do coverage analysis
 107 AllSymbols = {}
 108 AllIncompleteSymbols = {}
 109 AllUnusedSymbols = {}
 110 AllDocumentedSymbols = {}
 111
 112 # Undeclared yet documented symbols
 113 UndeclaredSymbols = {}
 114
 115 # These global arrays store GObject, subclasses and the hierarchy (also of
 116 # non-object derived types).
 117 Objects = []
 118 ObjectLevels = []
 119 ObjectRoots = {}
 120
 121 Interfaces = {}
 122 Prerequisites = {}
 123
 124 # holds the symbols which are mentioned in $MODULE-sections.txt and in which
 125 # section they are defined
 126 KnownSymbols = {}
 127 SymbolSection = {}
 128 SymbolSectionId = {}
 129
 130 # collects index entries
 131 IndexEntriesFull = {}
 132 IndexEntriesSince = {}
 133 IndexEntriesDeprecated = {}
 134
 135 # Standard C preprocessor directives, which we ignore for '#' abbreviations.
 136 PreProcessorDirectives = {
 137     'assert': 1,
 138     'define': 1,
 139     'elif': 1,
 140     'else': 1,
 141     'endif': 1,
 142     'error': 1,
 143     'if': 1,
 144     'ifdef': 1,
 145     'ifndef': 1,
 146     'include': 1,
 147     'line': 1,
 148     'pragma': 1,
 149     'unassert': 1,
 150     'undef': 1,
 151     'warning': 1
 152 }
 153
 154 # remember used annotation (to write minimal glossary)
 155 AnnotationsUsed = {}
 156
 157 # the regexp that parses the annotation is in ScanSourceFile()
 158 AnnotationDefinition = {
 159     # the GObjectIntrospection annotations are defined at:
 160     # https://live.gnome.org/GObjectIntrospection/Annotations
 161     'allow-none': "NULL is OK, both for passing and for returning.",
 162     'nullable': "NULL may be passed as the value in, out, in-out; or as a return value.",
 163     'not nullable': "NULL must not be passed as the value in, out, in-out; or as a return value.",
 164     'optional': "NULL may be passed instead of a pointer to a location.",
 165     'array': "Parameter points to an array of items.",
 166     'attribute': "Deprecated free-form custom annotation, replaced by (attributes) annotation.",
 167     'attributes': "Free-form key-value pairs.",
 168     'closure': "This parameter is a 'user_data', for callbacks; many bindings can pass NULL here.",
 169     'constructor': "This symbol is a constructor, not a static method.",
 170     'destroy': "This parameter is a 'destroy_data', for callbacks.",
 171     'default': "Default parameter value (for in case the <acronym>shadows</acronym>-to function has less parameters).",
 172     'element-type': "Generics and defining elements of containers and arrays.",
 173     'error-domains': "Typed errors. Similar to throws in Java.",
 174     'foreign': "This is a foreign struct.",
 175     'get-value-func': "The specified function is used to convert a struct from a GValue, must be a GTypeInstance.",
 176     'in': "Parameter for input. Default is <acronym>transfer none</acronym>.",
 177     'inout': "Parameter for input and for returning results. Default is <acronym>transfer full</acronym>.",
 178     'in-out': "Parameter for input and for returning results. Default is <acronym>transfer full</acronym>.",
 179     'method': "This is a method",
 180     'not-error': "A GError parameter is not to be handled like a normal GError.",
 181     'out': "Parameter for returning results. Default is <acronym>transfer full</acronym>.",
 182     'out caller-allocates': "Out parameter, where caller must allocate storage.",
 183     'out callee-allocates': "Out parameter, where caller must allocate storage.",
 184     'ref-func': "The specified function is used to ref a struct, must be a GTypeInstance.",
 185     'rename-to': "Rename the original symbol's name to SYMBOL.",
 186     'scope call': "The callback is valid only during the call to the method.",
 187     'scope async': "The callback is valid until first called.",
 188     'scope notified': "The callback is valid until the GDestroyNotify argument is called.",
 189     'set-value-func': "The specified function is used to convert from a struct to a GValue, must be a GTypeInstance.",
 190     'skip': "Exposed in C code, not necessarily available in other languages.",
 191     'transfer container': "Free data container after the code is done.",
 192     'transfer floating': "Alias for <acronym>transfer none</acronym>, used for objects with floating refs.",
 193     'transfer full': "Free data after the code is done.",
 194     'transfer none': "Don't free data after the code is done.",
 195     'type': "Override the parsed C type with given type.",
 196     'unref-func': "The specified function is used to unref a struct, must be a GTypeInstance.",
 197     'virtual': "This is the invoker for a virtual method.",
 198     'value': "The specified value overrides the evaluated value of the constant.",
 199     # Stability Level definition
 200     # https://bugzilla.gnome.org/show_bug.cgi?id=170860
 201     'Stable': '''
 202 The intention of a Stable interface is to enable arbitrary third parties to
 203 develop applications to these interfaces, release them, and have confidence that
 204 they will run on all minor releases of the product (after the one in which the
 205 interface was introduced, and within the same major release). Even at a major
 206 release, incompatible changes are expected to be rare, and to have strong
 207 justifications.
 208     ''',
 209     'Unstable': '''
 210 Unstable interfaces are experimental or transitional. They are typically used to
 211 give outside developers early access to new or rapidly changing technology, or
 212 to provide an interim solution to a problem where a more general solution is
 213 anticipated. No claims are made about either source or binary compatibility from
 214 one minor release to the next.
 215
 216 The Unstable interface level is a warning that these interfaces are  subject to
 217 change without warning and should not be used in unbundled products.
 218
 219 Given such caveats, customer impact need not be a factor when considering
 220 incompatible changes to an Unstable interface in a major or minor release.
 221 Nonetheless, when such changes are introduced, the changes should still be
 222 mentioned in the release notes for the affected release.
 223     ''',
 224     'Private': '''
 225 An interface that can be used within the GNOME stack itself, but that is not
 226 documented for end-users.  Such functions should only be used in specified and
 227 documented ways.
 228     ''',
 229 }
 230
 231 # Elements to consider non-block items in MarkDown parsing
 232 MD_TEXT_LEVEL_ELEMENTS = {"literal": 1,
 233                           "emphasis": 1,
 234                           "envar": 1,
 235                           "filename": 1,
 236                           "firstterm": 1,
 237                           "footnote": 1,
 238                           "function": 1,
 239                           "manvolnum": 1,
 240                           "option": 1,
 241                           "replaceable": 1,
 242                           "structfield": 1,
 243                           "structname": 1,
 244                           "title": 1,
 245                           "varname": 1,
 246                          }
 247 MD_ESCAPABLE_CHARS = {"\\": 1,
 248                       "`": 1,
 249                       "*": 1,
 250                       "_": 1,
 251                       "{": 1,
 252                       "}": 1,
 253                       "[": 1,
 254                       "]": 1,
 255                       "(": 1,
 256                       ")": 1,
 257                       ">": 1,
 258                       "#": 1,
 259                       "+": 1,
 260                       "-": 1,
 261                       ".": 1,
 262                       "!": 1,
 263                      }
 264 MD_GTK_ESCAPABLE_CHARS = {"@": 1,
 265                           "%": 1,
 266                          }
 267
 268 # Function and other declaration output settings.
 269 RETURN_TYPE_FIELD_WIDTH = 20
 270 SYMBOL_FIELD_WIDTH = 36
 271 MAX_SYMBOL_FIELD_WIDTH = 40
 272 SIGNAL_FIELD_WIDTH = 16
 273 PARAM_FIELD_COUNT = 2
 274
 275 # XML, SGML formatting helper
 276 doctype_header = None
 277
 278
 279 parser = argparse.ArgumentParser()
 280 parser.add_argument('--module', default='')
 281 parser.add_argument('--source-dir', dest='source_dir', default='')
 282 parser.add_argument('--source-suffixes', dest='source_suffixes', default='')
 283 parser.add_argument('--ignore-files', dest='ignore_files', default='')
 284 parser.add_argument('--output-dir', dest='output_dir', default='')
 285 parser.add_argument('--tmpl-dir', dest='tmpl_dir', default='')
 286 parser.add_argument('--main-sgml-file', dest='main_sgml_file', default='')
 287 parser.add_argument('--expand-content-files', dest='expand_content_files', default='')
 288 group = parser.add_mutually_exclusive_group()
 289 group.add_argument('--sgml-mode', action='store_true', default=False, dest='sgml_mode')
 290 group.add_argument('--xml-mode', action='store_true', default=False, dest='xml_mode')
 291 parser.add_argument('--default-stability', dest='default_stability', choices=['Stable', 'Private', 'Unstable'], default='Stable')
 292 parser.add_argument('--default-includes', dest='default_includes', default='')
 293 parser.add_argument('--output-format', dest='default_format', default='')
 294 parser.add_argument('--name-space', dest='name_space', default='')
 295 parser.add_argument('--outputallsymbols', default=False, action='store_true')
 296 parser.add_argument('--outputsymbolswithoutsince', default=False, action='store_true')
 297
 298
 299 def Run():
 300     global MODULE, TMPL_DIR, SOURCE_DIRS, SOURCE_SUFFIXES, IGNORE_FILES, MAIN_SGML_FILE, EXPAND_CONTENT_FILES, \
 301         INLINE_MARKUP_MODE, DEFAULT_STABILITY, DEFAULT_INCLUDES, OUTPUT_FORMAT, NAME_SPACE, OUTPUT_ALL_SYMBOLS, \
 302         OUTPUT_SYMBOLS_WITHOUT_SINCE, DB_OUTPUT_DIR, OBJECT_TREE_FILE, INTERFACES_FILE, PREREQUISITES_FILE, \
 303         SIGNALS_FILE, ARGS_FILE, doctype_header
 304
 305     options = parser.parse_args()
 306
 307     # We should pass the options variable around instead of this global variable horror
 308     # but too much of the code expects these to be around. Fix this once the transition is done.
 309     MODULE = options.module
 310     TMPL_DIR = options.tmpl_dir
 311     SOURCE_DIRS = options.source_dir
 312     SOURCE_SUFFIXES = options.source_suffixes
 313     IGNORE_FILES = options.ignore_files
 314     MAIN_SGML_FILE = options.main_sgml_file
 315     EXPAND_CONTENT_FILES = options.expand_content_files
 316     INLINE_MARKUP_MODE = options.xml_mode or options.sgml_mode
 317     DEFAULT_STABILITY = options.default_stability
 318     DEFAULT_INCLUDES = options.default_includes
 319     OUTPUT_FORMAT = options.default_format
 320     NAME_SPACE = options.name_space
 321     OUTPUT_ALL_SYMBOLS = options.outputallsymbols
 322     OUTPUT_SYMBOLS_WITHOUT_SINCE = options.outputsymbolswithoutsince
 323
 324     logging.info(" ignore files: " + IGNORE_FILES)
 325
 326     # check output format
 327     if OUTPUT_FORMAT is None or OUTPUT_FORMAT == '':
 328         OUTPUT_FORMAT = "xml"
 329     else:
 330         OUTPUT_FORMAT = OUTPUT_FORMAT.lower()
 331     if OUTPUT_FORMAT != "xml":
 332         sys.exit("Invalid format '%s' passed to --output.format" % OUTPUT_FORMAT)
 333
 334     if not MAIN_SGML_FILE:
 335         # backwards compatibility
 336         if os.path.exists(MODULE + "-docs.sgml"):
 337             MAIN_SGML_FILE = MODULE + "-docs.sgml"
 338         else:
 339             MAIN_SGML_FILE = MODULE + "-docs.xml"
 340
 341     # extract docbook header or define default
 342     if os.path.exists(MAIN_SGML_FILE):
 343         INPUT = open(MAIN_SGML_FILE)
 344         doctype_header = ''
 345         for line in INPUT:
 346             if re.search(r'^\s*<(book|chapter|article)', line):
 347                 # check that the top-level tagSYSTEM or the doctype decl contain the xinclude namespace decl
 348                 if not re.search(r'http:\/\/www.w3.org\/200[13]\/XInclude', line) and not re.search(r'http:\/\/www.w3.org\/200[13]\/XInclude', doctype_header, flags=re.MULTILINE):
 349                     doctype_header = ''
 350                 break
 351
 352             # if there are SYSTEM ENTITIES here, we should prepend "../" to the path
 353             # FIXME: not sure if we can do this now, as people already work-around the problem
 354             # s#<!ENTITY % ([a-zA-Z-]+) SYSTEM \"([^/][a-zA-Z./]+)\">#<!ENTITY % $1 SYSTEM \"../$2\">#;
 355             #<!ENTITY % gtkdocentities SYSTEM \"([^"]*)\">#<!ENTITY % gtkdocentities SYSTEM \"../$1\">#;
 356             doctype_header += line
 357         INPUT.close()
 358     else:
 359         doctype_header = '''<?xml version="1.0"?>
 360 <!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN"
 361                "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd"
 362 [
 363   <!ENTITY % local.common.attrib "xmlns:xi  CDATA  #FIXED 'http://www.w3.org/2003/XInclude'">
 364   <!ENTITY % gtkdocentities SYSTEM "../xml/gtkdocentities.ent">
 365   %gtkdocentities;
 366 ]>
 367 '''
 368
 369     doctype_header = doctype_header.strip()
 370
 371     # All the files are written in subdirectories beneath here.
 372     TMPL_DIR = TMPL_DIR if TMPL_DIR else os.path.join(ROOT_DIR, "tmpl")
 373
 374     # This is where we put all the DocBook output.
 375     DB_OUTPUT_DIR = DB_OUTPUT_DIR if DB_OUTPUT_DIR else os.path.join(ROOT_DIR, "xml")
 376
 377     # This file contains the object hierarchy.
 378     OBJECT_TREE_FILE = os.path.join(ROOT_DIR, MODULE + ".hierarchy")
 379
 380     # This file contains the interfaces.
 381     INTERFACES_FILE = os.path.join(ROOT_DIR, MODULE + ".interfaces")
 382
 383     # This file contains the prerequisites.
 384     PREREQUISITES_FILE = os.path.join(ROOT_DIR, MODULE + ".prerequisites")
 385
 386     # This file contains signal arguments and names.
 387     SIGNALS_FILE = os.path.join(ROOT_DIR, MODULE + ".signals")
 388
 389     # The file containing Arg information.
 390     ARGS_FILE = os.path.join(ROOT_DIR, MODULE + ".args")
 391
 392     # Create the root DocBook output directory if it doens't exist.
 393     if not os.path.isdir(DB_OUTPUT_DIR):
 394         os.mkdir(DB_OUTPUT_DIR)
 395
 396     ReadKnownSymbols(os.path.join(ROOT_DIR, MODULE + "-sections.txt"))
 397     ReadSignalsFile(SIGNALS_FILE)
 398     ReadArgsFile(ARGS_FILE)
 399     ReadObjectHierarchy()
 400     ReadInterfaces()
 401     ReadPrerequisites()
 402
 403     ReadDeclarationsFile(os.path.join(ROOT_DIR, MODULE + "-decl.txt"), 0)
 404     if os.path.isfile(os.path.join(ROOT_DIR, MODULE + "-overrides.txt")):
 405         ReadDeclarationsFile(os.path.join(ROOT_DIR, MODULE + "-overrides.txt"), 1)
 406
 407     for sdir in SOURCE_DIRS:
 408         ReadSourceDocumentation(sdir)
 409
 410     changed = OutputDB(os.path.join(ROOT_DIR, MODULE + "-sections.txt"))
 411
 412     # If any of the DocBook files have changed, update the timestamp file (so
 413     # it can be used for Makefile dependencies).
 414     if changed or not os.path.exists(os.path.join(ROOT_DIR, "sgml.stamp")):
 415
 416         # try to detect the common prefix
 417         # GtkWidget, GTK_WIDGET, gtk_widget -> gtk
 418         if NAME_SPACE == '':
 419             NAME_SPACE = ''
 420             pos = 0
 421             ratio = 0.0
 422             while True:
 423                 prefix = {}
 424                 letter = ''
 425                 for symbol in IndexEntriesFull.keys():
 426                     if NAME_SPACE == '' or NAME_SPACE.lower() in symbol.lower():
 427                         if len(symbol) > pos:
 428                             letter = symbol[pos:pos+1]
 429                             # stop prefix scanning
 430                             if letter == "_":
 431                                 # stop on "_"
 432                                 break
 433                             # Should we also stop on a uppercase char, if last was lowercase
 434                             #   GtkWidget, if we have the 'W' and had the 't' before
 435                             # or should we count upper and lowercase, and stop one 2nd uppercase, if we already had a lowercase
 436                             #   GtkWidget, the 'W' would be the 2nd uppercase and with 't','k' we had lowercase chars before
 437                             # need to recound each time as this is per symbol
 438                             ul = letter.upper()
 439                             if ul in prefix:
 440                                 prefix[ul] += 1
 441                             else:
 442                                 prefix[ul] = 1
 443
 444                 if letter != '' and letter != "_":
 445                     maxletter = ''
 446                     maxsymbols = 0
 447                     for letter in prefix.keys():
 448                         #print "$letter: $prefix{$letter}.\n";
 449                         if prefix[letter] > maxsymbols:
 450                             maxletter = letter
 451                             maxsymbols = prefix[letter]
 452
 453                     ratio = float(len(IndexEntriesFull)) / prefix[maxletter]
 454                     #print "most symbols start with $maxletter, that is ". (100 * $ratio) ." %\n";
 455                     if ratio > 0.9:
 456                         # do another round
 457                         NAME_SPACE += maxletter
 458
 459                     pos += 1
 460
 461                 else:
 462                     ratio = 0.0
 463
 464                 if ratio > 0.9:
 465                     break
 466
 467         OutputIndexFull()
 468         OutputDeprecatedIndex()
 469         OutputSinceIndexes()
 470         OutputAnnotationGlossary()
 471
 472         open(os.path.join(ROOT_DIR, 'sgml.stamp'), 'w').write('timestamp')
 473
 474 #############################################################################
 475 # Function    : OutputObjectList
 476 # Description : This outputs the alphabetical list of objects, in a columned
 477 #                table.
 478 #               FIXME: Currently this also outputs ancestor objects
 479 #                which may not actually be in this module.
 480 # Arguments   : none
 481 #############################################################################
 482
 483 def OutputObjectList():
 484     cols = 3
 485
 486     # FIXME: use .xml
 487     # my $old_object_index = "$DB_OUTPUT_DIR/object_index.xml";
 488     old_object_index = os.path.join(DB_OUTPUT_DIR, "object_index.sgml")
 489     new_object_index = os.path.join(DB_OUTPUT_DIR, "object_index.new")
 490
 491     OUTPUT = open(new_object_index, 'w')
 492
 493     OUTPUT.write('''
 494 %s
 495 <informaltable pgwide="1" frame="none">
 496 <tgroup cols="$cols">
 497 <colspec colwidth="1*"/>
 498 <colspec colwidth="1*"/>
 499 <colspec colwidth="1*"/>
 500 <tbody>
 501 ''' % MakeDocHeader("informaltable"))
 502
 503     count = 0
 504     object = None
 505     for object in sorted(Objects):
 506         xref = MakeXRef(object)
 507         if count % cols == 0:
 508             OUTPUT.write("<row>\n")
 509         OUTPUT.write("<entry>%s</entry>\n" % xref)
 510         if count % cols == cols - 1:
 511             OUTPUT.write("</row>\n")
 512         count += 1
 513
 514     if count == 0:
 515         # emit an empty row, since empty tables are invalid
 516         OUTPUT.write("<row><entry> </entry></row>\n")
 517
 518     else:
 519         if count % cols > 0:
 520             OUTPUT.write("</row>\n")
 521
 522     OUTPUT.write('''</tbody></tgroup></informaltable>\n''')
 523     OUTPUT.close()
 524
 525     common.UpdateFileIfChanged(old_object_index, new_object_index, 0)
 526
 527
 528 #############################################################################
 529 # Function    : TrimTextBlock
 530 # Description : Trims extra whitespace. Empty lines inside a block are
 531 #                preserved.
 532 # Arguments   : $desc - the text block to trim. May contain newlines.
 533 #############################################################################
 534
 535 def TrimTextBlock(desc):
 536     # strip leading spaces on the block
 537     desc = re.sub(r'^\s+', '', desc)
 538     # strip trailing spaces on every line
 539     desc = re.sub(r'\s+$', '\n', desc, flags=re.MULTILINE)
 540
 541     return desc
 542
 543
 544 #############################################################################
 545 # Function    : OutputDB
 546 # Description : This collects the output for each section of the docs, and
 547 #                outputs each file when the end of the section is found.
 548 # Arguments   : $file - the $MODULE-sections.txt file which contains all of
 549 #                the functions/macros/structs etc. being documented, organised
 550 #                into sections and subsections.
 551 #############################################################################
 552
 553 def OutputDB(file):
 554
 555     logging.info("Reading: %s", file)
 556     INPUT = open(file)
 557     filename = ''
 558     book_top = ''
 559     book_bottom = ''
 560     includes = DEFAULT_INCLUDES if DEFAULT_INCLUDES else ''
 561     section_includes = ''
 562     in_section = 0
 563     title = ''
 564     section_id = ''
 565     subsection = ''
 566     num_symbols = 0
 567     changed = 0
 568     functions_synop = ''
 569     other_synop = ''
 570     functions_details = ''
 571     other_details = ''
 572     signals_synop = ''
 573     signals_desc = ''
 574     args_synop = ''
 575     child_args_synop = ''
 576     style_args_synop = ''
 577     args_desc = ''
 578     child_args_desc = ''
 579     style_args_desc = ''
 580     hierarchy_str = ''
 581     hierarchy = []
 582     interfaces = ''
 583     implementations = ''
 584     prerequisites = ''
 585     derived = ''
 586     file_objects = []
 587     templates = {}
 588     symbol_def_line = {}
 589
 590     # merge the source docs, in case there are no templates
 591     MergeSourceDocumentation()
 592
 593     line_number = 0
 594     for line in INPUT:
 595         line_number += 1
 596
 597         if line.strip() == '':
 598             continue
 599
 600         m1 = re.search(r'^<SUBSECTION\s*(.*)', line)
 601         m2 = re.search(r'^<TITLE>(.*)<\/TITLE', line)
 602         m3 = re.search(r'^<FILE>(.*)<\/FILE>', line)
 603         m4 = re.search(r'^<INCLUDE>(.*)<\/INCLUDE>', line)
 604         m5 = re.search(r'^(\S+)', line)
 605
 606         if line.startswith('<SECTION>'):
 607             num_symbols = 0
 608             in_section = False
 609             file_objects = []
 610             symbol_def_line = {}
 611
 612         elif m1:
 613             other_synop += "\n"
 614             functions_synop += "\n"
 615             subsection = m1.group(1)
 616
 617         elif re.search(r'^<SUBSECTION>', line):
 618             pass
 619         elif m2:
 620             title = m2.group(1)
 621             logging.info("Section: %s", title)
 622
 623             # We don't want warnings if object & class structs aren't used.
 624             DeclarationOutput[title] = 1
 625             DeclarationOutput["%sClass" % title] = 1
 626             DeclarationOutput["%sIface" % title] = 1
 627             DeclarationOutput["%sInterface" % title] = 1
 628
 629         elif m3:
 630             filename = m3.group(1)
 631             if filename in templates:
 632                 if ReadTemplateFile(os.path.join(TMPL_DIR, filename), 1):
 633                     MergeSourceDocumentation()
 634                     templates[filename] = line_number
 635             else:
 636                 common.LogWarning(file, line_number, ("Double <FILE>%s</FILE> entry. " % file) + "Previous occurrence on line " + templates[filename] + ".")
 637             if title == ''  and ("%s/%s:Title" % (TMPL_DIR, filename)) & SourceSymbolDocs:
 638                 title = SourceSymbolDocs["%s/%s:Title" % (TMPL_DIR, filename)]
 639                  # Remove trailing blanks
 640                 title = title.rstrip()
 641
 642         elif m4:
 643             if in_section:
 644                 section_includes = m4.group(1)
 645             else:
 646                 if DEFAULT_INCLUDES:
 647                     common.LogWarning(file, line_number, "Default <INCLUDE> being overridden by command line option.")
 648                 else:
 649                     includes = m4.group(1)
 650
 651         elif re.search(r'^<\/SECTION>', line):
 652             logging.info("End of section: $title\n")
 653             if num_symbols > 0:
 654                 # collect documents
 655                 book_bottom += "    <xi:include href=\"xml/%s.xml\"/>\n" % filename
 656
 657                 i = os.path.join(TMPL_DIR, filename + ":Include")
 658
 659                 if i in SourceSymbolDocs:
 660                     if section_includes:
 661                         common.LogWarning(file, line_number, "Section <INCLUDE> being overridden by inline comments.")
 662                     section_includes = SourceSymbolDocs[i]
 663
 664                 if section_includes == '':
 665                     section_includes = includes
 666
 667                 signals_synop = re.sub(r'^\n*', '', signals_synop)
 668                 signals_synop = re.sub(r'\n+$', '\n', signals_synop)
 669
 670                 if signals_synop != '':
 671                     signals_synop = '''<refsect1 id="%s.signals" role="signal_proto">
 672 <title role="signal_proto.title">Signals</title>
 673 <informaltable frame="none">
 674 <tgroup cols="3">
 675 <colspec colname="signals_return" colwidth="150px"/>
 676 <colspec colname="signals_name" colwidth="300px"/>
 677 <colspec colname="signals_flags" colwidth="200px"/>
 678 <tbody>
 679 %s
 680 </tbody>
 681 </tgroup>
 682 </informaltable>
 683 </refsect1>
 684 ''' % (section_id, signals_synop)
 685                     signals_desc = TrimTextBlock(signals_desc)
 686                     signals_desc = '''<refsect1 id="%s.signal-details" role="signals">
 687 <title role="signals.title">Signal Details</title>
 688 %s
 689 </refsect1>
 690 ''' % (section_id, signals_desc)
 691
 692                 args_synop = re.sub(r'^\n*', '', args_synop)
 693                 args_synop = re.sub(r'\n+$', '\n', args_synop)
 694                 if args_synop != '':
 695                     args_synop = '''<refsect1 id="%s.properties" role="properties">
 696 <title role="properties.title">Properties</title>
 697 <informaltable frame="none">
 698 <tgroup cols="3">
 699 <colspec colname="properties_type" colwidth="150px"/>
 700 <colspec colname="properties_name" colwidth="300px"/>
 701 <colspec colname="properties_flags" colwidth="200px"/>
 702 <tbody>
 703 %s
 704 </tbody>
 705 </tgroup>
 706 </informaltable>
 707 </refsect1>
 708 ''' %(section_id, args_synop)
 709                     args_desc = TrimTextBlock(args_desc)
 710                     args_desc = '''<refsect1 id="%s.property-details" role="property_details">
 711 <title role="property_details.title">Property Details</title>
 712 %s
 713 </refsect1>
 714 ''' % (section_id, args_desc)
 715
 716                 child_args_synop = re.sub(r'^\n*', '', child_args_synop)
 717                 child_args_synop = re.sub(r'\n+$', '\n', child_args_synop)
 718                 if child_args_synop != '':
 719                     args_synop += '''<refsect1 id="%s.child-properties" role="child_properties">
 720 <title role="child_properties.title">Child Properties</title>
 721 <informaltable frame="none">
 722 <tgroup cols="3">
 723 <colspec colname="child_properties_type" colwidth="150px"/>
 724 <colspec colname="child_properties_name" colwidth="300px"/>
 725 <colspec colname="child_properties_flags" colwidth="200px"/>
 726 <tbody>
 727 %s
 728 </tbody>
 729 </tgroup>
 730 </informaltable>
 731 </refsect1>
 732 ''' % (section_id, child_args_synop)
 733                     child_args_desc = TrimTextBlock(child_args_desc)
 734                     args_desc += '''<refsect1 id="%s.child-property-details" role="child_property_details">
 735 <title role="child_property_details.title">Child Property Details</title>
 736 %s
 737 </refsect1>
 738 ''' % (section_id, child_args_desc)
 739
 740
 741                 style_args_synop = re.sub(r'^\n*', '', style_args_synop)
 742                 style_args_synop = re.sub(r'\n+$', '\n', style_args_synop)
 743                 if style_args_synop != '':
 744                     args_synop += '''<refsect1 id="%s.style-properties" role="style_properties">
 745 <title role="style_properties.title">Style Properties</title>
 746 <informaltable frame="none">
 747 <tgroup cols="3">
 748 <colspec colname="style_properties_type" colwidth="150px"/>
 749 <colspec colname="style_properties_name" colwidth="300px"/>
 750 <colspec colname="style_properties_flags" colwidth="200px"/>
 751 <tbody>
 752 %s
 753 </tbody>
 754 </tgroup>
 755 </informaltable>
 756 </refsect1>
 757 ''' % (section_id, style_args_synop)
 758                     style_args_desc = TrimTextBlock(style_args_desc)
 759                     args_desc += '''<refsect1 id="%s.style-property-details" role="style_properties_details">
 760 <title role="style_properties_details.title">Style Property Details</title>
 761 %s
 762 </refsect1>
 763 ''' % (section_id, style_args_desc)
 764
 765                 hierarchy_str = AddTreeLineArt(hierarchy)
 766                 if hierarchy_str != '':
 767                     hierarchy_str = '''<refsect1 id="%s.object-hierarchy" role="object_hierarchy">
 768 <title role="object_hierarchy.title">Object Hierarchy</title>
 769 <screen>%s
 770 </screen>
 771 </refsect1>
 772 ''' % (section_id, hierarchy_str)
 773
 774                 interfaces = TrimTextBlock(interfaces)
 775                 if interfaces != '':
 776                     interfaces = '''<refsect1 id="%s.implemented-interfaces" role="impl_interfaces">
 777 <title role="impl_interfaces.title">Implemented Interfaces</title>
 778 %s
 779 </refsect1>
 780 ''' % (section_id, interfaces)
 781
 782                 implementations = TrimTextBlock(implementations)
 783                 if implementations != '':
 784                     implementations = '''<refsect1 id="%s.implementations" role="implementations">
 785 <title role="implementations.title">Known Implementations</title>
 786 %s
 787 </refsect1>
 788 ''' % (section_id, implementations)
 789
 790                 prerequisites = TrimTextBlock(prerequisites)
 791                 if prerequisites != '':
 792                     prerequisites = '''<refsect1 id="%s.prerequisites" role="prerequisites">
 793 <title role="prerequisites.title">Prerequisites</title>
 794 %s
 795 </refsect1>
 796 ''' % (section_id, prerequisites)
 797
 798
 799                 derived = TrimTextBlock(derived)
 800                 if derived != '':
 801                     derived = '''<refsect1 id="%s.derived-interfaces" role="derived_interfaces">
 802 <title role="derived_interfaces.title">Known Derived Interfaces</title>
 803 %s
 804 </refsect1>
 805 ''' % (section_id, derived)
 806
 807                 functions_synop = re.sub(r'^\n*', '', functions_synop)
 808                 functions_synop = re.sub(r'\n+$', '\n', functions_synop)
 809                 if functions_synop != '':
 810                     functions_synop = '''<refsect1 id="%s.functions" role="functions_proto">
 811 <title role="functions_proto.title">Functions</title>
 812 <informaltable pgwide="1" frame="none">
 813 <tgroup cols="2">
 814 <colspec colname="functions_return" colwidth="150px"/>
 815 <colspec colname="functions_name"/>
 816 <tbody>
 817 %s
 818 </tbody>
 819 </tgroup>
 820 </informaltable>
 821 </refsect1>
 822 ''' % (section_id, functions_synop)
 823
 824                 other_synop = re.sub(r'^\n*', '', other_synop)
 825                 other_synop = re.sub(r'\n+$', '\n', other_synop)
 826                 if other_synop != '':
 827                     other_synop = '''<refsect1 id="%s.other" role="other_proto">
 828 <title role="other_proto.title">Types and Values</title>
 829 <informaltable role="enum_members_table" pgwide="1" frame="none">
 830 <tgroup cols="2">
 831 <colspec colname="name" colwidth="150px"/>
 832 <colspec colname="description"/>
 833 <tbody>
 834 %s
 835 </tbody>
 836 </tgroup>
 837 </informaltable>
 838 </refsect1>
 839 ''' % (section_id, other_synop)
 840
 841                 file_changed = OutputDBFile(filename, title, section_id,
 842                                             section_includes,
 843                                             functions_synop, other_synop,
 844                                             functions_details, other_details,
 845                                             signals_synop, signals_desc,
 846                                             args_synop, args_desc,
 847                                             hierarchy_str, interfaces,
 848                                             implementations,
 849                                             prerequisites, derived,
 850                                             file_objects)
 851                 if file_changed:
 852                     changed = True
 853
 854             title = ''
 855             section_id = ''
 856             subsection = ''
 857             in_section = 0
 858             section_includes = ''
 859             functions_synop = ''
 860             other_synop = ''
 861             functions_details = ''
 862             other_details = ''
 863             signals_synop = ''
 864             signals_desc = ''
 865             args_synop = ''
 866             child_args_synop = ''
 867             style_args_synop = ''
 868             args_desc = ''
 869             child_args_desc = ''
 870             style_args_desc = ''
 871             hierarchy_str = ''
 872             hierarchy = []
 873             interfaces = ''
 874             implementations = ''
 875             prerequisites = ''
 876             derived = ''
 877
 878         elif m5:
 879             symbol = m5.group(1)
 880             logging.info("  Symbol: %s in subsection: %s", symbol, subsection)
 881
 882             # check for duplicate entries
 883             if symbol in symbol_def_line:
 884                 declaration = Declarations.get(symbol, None)
 885                 if declaration:
 886                     if CheckIsObject(symbol):
 887                         file_objects.append(symbol)
 888
 889                     # We don't want standard macros/functions of GObjects,
 890                     # or private declarations.
 891                     if subsection != "Standard" and subsection != "Private":
 892                         synop, desc = OutputDeclaration(symbol, declaration)
 893                         type = DeclarationTypes[symbol]
 894
 895                         if type == 'FUNCTION' or type == 'USER_FUNCTION':
 896                             functions_synop += synop
 897                             functions_details += desc
 898                         elif type == 'MACRO' and re.search(symbol + r'\(', declaration):
 899                             functions_synop += synop
 900                             functions_details += desc
 901                         else:
 902                             other_synop += synop
 903                             other_details += +desc
 904
 905                     sig_synop, sig_desc = GetSignals(symbol)
 906                     arg_synop, child_arg_synop, style_arg_synop, arg_desc, child_arg_desc, style_arg_desc = GetArgs(symbol)
 907                     ifaces = GetInterfaces(symbol)
 908                     impls = GetImplementations(symbol)
 909                     prereqs = GetPrerequisites(symbol)
 910                     der = GetDerived(symbol)
 911                     hierarchy = GetHierarchy(symbol, hierarchy)
 912
 913                     signals_synop += sig_synop
 914                     signals_desc += sig_desc
 915                     args_synop += arg_synop
 916                     child_args_synop += child_arg_synop
 917                     style_args_synop += style_arg_synop
 918                     args_desc += arg_desc
 919                     child_args_desc += child_arg_desc
 920                     style_args_desc += style_arg_desc
 921                     interfaces += ifaces
 922                     implementations += impls
 923                     prerequisites += prereqs
 924                     derived += der
 925
 926                     # Note that the declaration has been output.
 927                     DeclarationOutput[symbol] = True
 928                 elif subsection != "Standard" and subsection != "Private":
 929                     UndeclaredSymbols[symbol] = True
 930                     common.LogWarning(file, line_number, "No declaration found for %s." % symbol)
 931
 932                 num_symbols += 1
 933                 symbol_def_line[symbol] = line_number
 934
 935                 if section_id == '':
 936                     if title == '' and filename == '':
 937                         common.LogWarning(file, line_number, "Section has no title and no file.")
 938
 939                     # FIXME: one of those would be enough
 940                     # filename should be an internal detail for gtk-doc
 941                     if title == '':
 942                         title = filename
 943                     elif filename == '':
 944                         filename = title
 945
 946                     filename = filename.replace(' ', '_')
 947
 948                     section_id = SourceSymbolDocs[os.path.join(TMPL_DIR, filename + ":Section_Id")]
 949                     if section_id and section_id.strip() != '':
 950                         # Remove trailing blanks and use as is
 951                         section_id = section_id.rstrip()
 952                     elif CheckIsObject(title):
 953                         # GObjects use their class name as the ID.
 954                         section_id = common.CreateValidSGMLID(title)
 955                     else:
 956                         section_id = common.CreateValidSGMLID(MODULE + '-' + title)
 957
 958                 SymbolSection[symbol] = title
 959                 SymbolSectionId[symbol] = section_id
 960
 961             else:
 962                 common.LogWarning(file, line_number, "Double symbol entry for %s. "
 963                                   "Previous occurrence on line %d." % (symbol, symbol_def_line[symbol]))
 964     INPUT.close()
 965
 966     OutputMissingDocumentation()
 967     OutputUndeclaredSymbols()
 968     OutputUnusedSymbols()
 969
 970     if OUTPUT_ALL_SYMBOLS:
 971         OutputAllSymbols()
 972
 973     if OUTPUT_SYMBOLS_WITHOUT_SINCE:
 974         OutputSymbolsWithoutSince()
 975
 976
 977     for filename in EXPAND_CONTENT_FILES.split():
 978         file_changed = OutputExtraFile(filename)
 979         if file_changed:
 980             changed = True
 981
 982     OutputBook(book_top, book_bottom)
 983
 984     return changed
 985
 986 #############################################################################
 987 # Function    : OutputIndex
 988 # Description : This writes an indexlist that can be included into the main-
 989 #               document into an <index> tag.
 990 #############################################################################
 991
 992 def OutputIndex(basename, apiindexref):
 993     apiindex = apiindexref # FIXME should we copy here?
 994     old_index = os.path.join(DB_OUTPUT_DIR, basename + '.xml')
 995     new_index = os.path.join(DB_OUTPUT_DIR, basename + '.new')
 996     lastletter = " "
 997     divopen = 0
 998     symbol = None
 999     short_symbol = None
1000
1001     OUTPUT = open(new_index, 'w')
1002
1003     OUTPUT.write(MakeDocHeader("indexdiv") + "\n<indexdiv id=\"%s\">\n" % basename)
1004
1005     logging.info("generate %s index (%d entries)\n", basename, len(apiindex))
1006
1007     # do a case insensitive sort while chopping off the prefix
1008     def cpmfunc(intxt):
1009         uc = intxt.upper()
1010         criteria = re.sub(r'^' + NAME_SPACE + r'\_?(.*)', r'\1', uc, flags=re.I)
1011         return (criteria, intxt)
1012
1013     sorted_keys = sorted(apiindex.keys(), key=cpmfunc)
1014
1015     for original in sorted_keys:
1016         symbol = apiindex[original]
1017         match = re.search(r'^' + NAME_SPACE + r'\_?(.*)', original, flags=re.I)
1018         short = match.group(1)
1019         if short and short != '':
1020             short_symbol = short
1021         else:
1022             short_symbol = symbol
1023
1024         # generate a short symbol description
1025         symbol_desc = ''
1026         symbol_section = ''
1027         symbol_section_id = ''
1028         symbol_type = ''
1029         if symbol in DeclarationTypes:
1030             symbol_type = DeclarationTypes[symbol].lower()
1031
1032         if symbol_type == '':
1033             logging.info("trying symbol %s\n", symbol)
1034             m = re.search(r'(.*)::(.*)', symbol)
1035             m2 = re.search(r'(.*):(.*)/', symbol)
1036             if m:
1037                 oname = m.group(1)
1038                 osym = m.group(2)
1039                 i = None
1040                 logging.info("  trying object signal %s:%s in %d signals\n", oname, osym, len(SignalNames))
1041                 for i in range(len(SignalNames)):
1042                     if SignalNames[i] == osym:
1043                         symbol_type = "object signal"
1044                         if oname in SymbolSection:
1045                             symbol_section = SymbolSection[oname]
1046                             symbol_section_id = SymbolSectionId[oname]
1047                         break
1048             elif m2:
1049                 oname = m2.group(1)
1050                 osym = m2.group(2)
1051                 i = None
1052                 logging.info("  trying object property %s::%s in %d properties\n", oname, osym, len(ArgNames))
1053                 for i in range(len(ArgNames)):
1054                     logging.info("    " + ArgNames[i] + "\n")
1055                     if ArgNames[i] == osym:
1056                         symbol_type = "object property"
1057                         if oname in SymbolSection:
1058                             symbol_section = SymbolSection[oname]
1059                             symbol_section_id = SymbolSectionId[oname]
1060                         break
1061         else:
1062             if symbol in SymbolSection:
1063                 symbol_section = SymbolSection[symbol]
1064                 symbol_section_id = SymbolSectionId[symbol]
1065
1066         if symbol_type != '':
1067             symbol_desc = ", " + symbol_type
1068             if symbol_section != '':
1069                 symbol_desc += " in <link linkend=\"%s\">%s</link>" % (symbol_section_id, symbol_section)
1070                 #$symbol_desc.=" in ". &ExpandAbbreviations($symbol, "#$symbol_section")
1071
1072         curletter = short_symbol[0].upper()
1073         id = apiindex[symbol]
1074
1075         logging.info("  add symbol %s with %d to index in section '%s' (derived from %s)\n", symbol, id, curletter, short_symbol)
1076
1077         if curletter != lastletter:
1078             lastletter = curletter
1079
1080             if divopen:
1081                 OUTPUT.write("</indexdiv>\n")
1082
1083             OUTPUT.write("<indexdiv><title>$curletter</title>\n")
1084             divopen = True
1085
1086         OUTPUT.write('<indexentry><primaryie linkends="$id"><link linkend="$id">$symbol</link>$symbol_desc</primaryie></indexentry>\n')
1087
1088     if divopen:
1089         OUTPUT.write("</indexdiv>\n")
1090
1091     OUTPUT.write("</indexdiv>\n")
1092     OUTPUT.close()
1093
1094     common.UpdateFileIfChanged(old_index, new_index, 0)
1095
1096
1097
1098 #############################################################################
1099 # Function    : OutputIndexFull
1100 # Description : This writes the full api indexlist that can be included into the
1101 #               main document into an <index> tag.
1102 #############################################################################
1103
1104 def OutputIndexFull():
1105     OutputIndex("api-index-full", IndexEntriesFull)
1106
1107
1108 #############################################################################
1109 # Function    : OutputDeprecatedIndex
1110 # Description : This writes the deprecated api indexlist that can be included
1111 #               into the main document into an <index> tag.
1112 #############################################################################
1113
1114 def OutputDeprecatedIndex():
1115     OutputIndex("api-index-deprecated", IndexEntriesDeprecated)
1116
1117
1118 #############################################################################
1119 # Function    : OutputSinceIndexes
1120 # Description : This writes the 'since' api indexlists that can be included into
1121 #               the main document into an <index> tag.
1122 #############################################################################
1123
1124 def OutputSinceIndexes():
1125     raise RuntimeError('I have no idea what this does.')
1126 #    my @sinces = keys %{{ map { $_ => 1 } values %Since }
1127 #
1128 #    foreach my $version (@sinces)
1129 #        logging.info("Since : [$version]\n")
1130 #        # TODO make filtered hash
1131 #        #my %index = grep { $Since{$_} == $version } %IndexEntriesSince
1132 #        my %index = map { $_ => $IndexEntriesSince{$_} } grep { $Since{$_} == $version } keys %IndexEntriesSince
1133 #
1134 #        &OutputIndex ("api-index-$version", \%index)
1135
1136
1137
1138 #############################################################################
1139 # Function    : OutputAnnotationGlossary
1140 # Description : This writes a glossary of the used annotation terms into a
1141 #               separate glossary file that can be included into the main
1142 #               document.
1143 #############################################################################
1144
1145 def OutputAnnotationGlossary():
1146     old_glossary = os.path.join(DB_OUTPUT_DIR, "annotation-glossary.xml")
1147     new_glossary = os.path.join(DB_OUTPUT_DIR, "annotation-glossary.new")
1148     lastletter = " "
1149     divopen = False
1150
1151     # if there are no annotations used return
1152     if len(AnnotationsUsed) == 0:
1153         return
1154
1155     # add acronyms that are referenced from acronym text
1156 #rerun:
1157     for annotation in AnnotationsUsed:
1158         if annotation in AnnotationDefinition:
1159             m = re.search(r'<acronym>([\w ]+)<\/acronym>', AnnotationDefinition[annotation])
1160             if m:
1161                 if m.group(1) in AnnotationsUsed:
1162                     AnnotationsUsed[m.group(1)] = 1
1163                     return OutputAnnotationGlossary()
1164
1165     OUTPUT = open(new_glossary, 'w')
1166
1167     OUTPUT.write('''%s
1168 <glossary id="annotation-glossary">
1169   <title>Annotation Glossary</title>
1170 ''' % MakeDocHeader("glossary"))
1171
1172     for annotation in sorted(AnnotationsUsed.keys(), keys=str.lower()):
1173         if annotation in AnnotationDefinition:
1174             defi = AnnotationDefinition[annotation]
1175             curletter = annotation[0].upper()
1176
1177             if curletter != lastletter:
1178                 lastletter = curletter
1179
1180                 if divopen:
1181                     OUTPUT.write("</glossdiv>\n")
1182
1183                 OUTPUT.write("<glossdiv><title>%s</title>\n" % curletter)
1184                 divopen = True
1185
1186             OUTPUT.write('''
1187     <glossentry>
1188       <glossterm><anchor id="annotation-glossterm-%s"/>%s</glossterm>
1189       <glossdef>
1190         <para>%s</para>
1191       </glossdef>
1192     </glossentry>
1193 ''' % (annotation, annotation, defi))
1194
1195     if divopen:
1196         OUTPUT.write("</glossdiv>\n")
1197
1198     OUTPUT.write("</glossary>\n")
1199     OUTPUT.close()
1200
1201     common.UpdateFileIfChanged(old_glossary, new_glossary, 0)
1202
1203
1204 #############################################################################
1205 # Function    : ReadKnownSymbols
1206 # Description : This collects the names of non-private symbols from the
1207 #               $MODULE-sections.txt file.
1208 # Arguments   : $file - the $MODULE-sections.txt file which contains all of
1209 #                the functions/macros/structs etc. being documented, organised
1210 #                into sections and subsections.
1211 #############################################################################
1212
1213 def ReadKnownSymbols(file):
1214
1215     subsection = ''
1216
1217     logging.info("Reading: %s", file)
1218     INPUT = open(file)
1219
1220     for line in INPUT:
1221         if line.strip() == '':
1222             continue
1223
1224         m = re.search(r'^<SECTION>', line)
1225         if m:
1226             subsection = ''
1227             continue
1228
1229         m = re.search(r'^<SUBSECTION\s*(.*)', line, flags=re.I)
1230         if m:
1231             subsection = m.group(1)
1232             continue
1233
1234         if re.search(r'^<SUBSECTION>', line):
1235             continue
1236
1237         if re.search(r'^<TITLE>(.*)<\/TITLE>', line):
1238             continue
1239
1240         m = re.search(r'^<FILE>(.*)<\/FILE>', line)
1241         if m:
1242             KnownSymbols[os.path.join(TMPL_DIR, m.group(1) + ":Long_Description")] = 1
1243             KnownSymbols[os.path.join(TMPL_DIR, m.group(1) + ":Short_Description")] = 1
1244             continue
1245
1246         m = re.search(r'^<INCLUDE>(.*)<\/INCLUDE>', line)
1247         if m:
1248             continue
1249
1250         m = re.search(r'^<\/SECTION>', line)
1251         if m:
1252             continue
1253
1254         m = re.search(r'^(\S+)', line)
1255         if m:
1256             symbol = m.group(1)
1257
1258             if subsection != "Standard" and subsection != "Private":
1259                 KnownSymbols[symbol] = 1
1260             else:
1261                 KnownSymbols[symbol] = 0
1262     INPUT.close()
1263
1264 #############################################################################
1265 # Function    : OutputDeclaration
1266 # Description : Returns the synopsis and detailed description DocBook
1267 #                describing one function/macro etc.
1268 # Arguments   : $symbol - the name of the function/macro begin described.
1269 #                $declaration - the declaration of the function/macro.
1270 #############################################################################
1271
1272 def OutputDeclaration(symbol, declaration):
1273     dtype = DeclarationTypes[symbol]
1274     if dtype == 'MACRO':
1275         return OutputMacro(symbol, declaration)
1276     elif dtype == 'TYPEDEF':
1277         return OutputTypedef(symbol, declaration)
1278     elif dtype == 'STRUCT':
1279         return OutputStruct(symbol, declaration)
1280     elif dtype == 'ENUM':
1281         return OutputEnum(symbol, declaration)
1282     elif dtype == 'UNION':
1283         return OutputUnion(symbol, declaration)
1284     elif dtype == 'VARIABLE':
1285         return OutputVariable(symbol, declaration)
1286     elif dtype == 'FUNCTION':
1287         return OutputFunction(symbol, declaration, dtype)
1288     elif dtype == 'USER_FUNCTION':
1289         return OutputFunction(symbol, declaration, dtype)
1290     else:
1291         sys.exit("Unknown symbol type " + dtype)
1292
1293
1294 #############################################################################
1295 # Function    : OutputSymbolTraits
1296 # Description : Returns the Since and StabilityLevel paragraphs for a symbol.
1297 # Arguments   : $symbol - the name of the function/macro begin described.
1298 #############################################################################
1299
1300 def OutputSymbolTraits(symbol):
1301     desc = ''
1302
1303     if symbol in Since:
1304         link_id = "api-index-" + Since[symbol]
1305         desc += "<para role=\"since\">Since: <link linkend=\"%s\">%s</link></para>" % (link_id, Since[symbol])
1306
1307     if symbol in StabilityLevel:
1308         stability = StabilityLevel[symbol]
1309         AnnotationsUsed[stability] = True
1310         desc += "<para role=\"stability\">Stability Level: <acronym>%s</acronym></para>" % stability
1311     return desc
1312
1313
1314 #############################################################################
1315 # Function    : Output{Symbol,Section}ExtraLinks
1316 # Description : Returns extralinks for the symbol (if enabled).
1317 # Arguments   : $symbol - the name of the function/macro begin described.
1318 #############################################################################
1319
1320 def uri_escape(text):
1321     if text is None:
1322         return None
1323
1324     # Build a char to hex map
1325     escapes = {}
1326     for i in range(256):
1327         escapes[chr(i)] = "%%%02X" % i
1328
1329     # Default unsafe characters.  RFC 2732 ^(uric - reserved)
1330     def do_escape(char):
1331         return escapes[char]
1332     text = re.sub(r"([^A-Za-z0-9\-_.!~*'()]", do_escape, text)
1333
1334     return text
1335
1336 def OutputSymbolExtraLinks(symbol):
1337     desc = ''
1338
1339     if False: # NEW FEATURE: needs configurability
1340         sstr = uri_escape(symbol)
1341         mstr = uri_escape(MODULE)
1342         desc += '''<ulink role="extralinks" url="http://www.google.com/codesearch?q=%s">code search</ulink>
1343 <ulink role="extralinks" url="http://library.gnome.org/edit?module=%s&amp;symbol=%s">edit documentation</ulink>
1344 ''' % (sstr, mstr, sstr)
1345
1346     return desc
1347
1348
1349 def OutputSectionExtraLinks(symbol, docsymbol):
1350     desc = ''
1351
1352     if False: # NEW FEATURE: needs configurability
1353         sstr = uri_escape(symbol)
1354         mstr = uri_escape(MODULE)
1355         dsstr = uri_escape(docsymbol)
1356         desc += '''<ulink role="extralinks" url="http://www.google.com/codesearch?q=%s">code search</ulink>
1357 <ulink role="extralinks" url="http://library.gnome.org/edit?module=%s&amp;symbol=%s">edit documentation</ulink>
1358 ''' % (sstr, mstr, dsstr)
1359     return desc
1360
1361
1362 #############################################################################
1363 # Function    : OutputMacro
1364 # Description : Returns the synopsis and detailed description of a macro.
1365 # Arguments   : $symbol - the macro.
1366 #                $declaration - the declaration of the macro.
1367 #############################################################################
1368
1369 def OutputMacro(symbol, declaration):
1370     sid = common.CreateValidSGMLID(symbol)
1371     condition = MakeConditionDescription(symbol)
1372     synop = "<row><entry role=\"define_keyword\">#define</entry><entry role=\"function_name\"><link linkend=\"%s\">%s</link>" % (sid, symbol)
1373     desc = None
1374
1375     fields = common.ParseMacroDeclaration(declaration, CreateValidSGML)
1376     title = symbol  +'()' if len(fields) > 0 else ''
1377
1378     desc = "<refsect2 id=\"%s\" role=\"macro\"condition>\n<title>title</title>\n" % (sid, condition, title)
1379     desc += MakeIndexterms(symbol, sid)
1380     desc += "\n"
1381     desc += OutputSymbolExtraLinks(symbol)
1382
1383     if len(fields) > 0:
1384         synop += "<phrase role=\"c_punctuation\">()</phrase>"
1385
1386     synop += "</entry></row>\n"
1387
1388     # Don't output the macro definition if is is a conditional macro or it
1389     # looks like a function, i.e. starts with "g_" or "_?gnome_", or it is
1390     # longer than 2 lines, otherwise we get lots of complicated macros like
1391     # g_assert.
1392     if symbol in DeclarationConditional and not symbol.startswith('g_') \
1393         and not re.search(r'^_?gnome_', symbol) and declaration.count('\n/') < 2:
1394         decl_out = CreateValidSGML(declaration)
1395         desc += "<programlisting language=\"C\">%s</programlisting>\n" % decl_out
1396     else:
1397         desc += "<programlisting language=\"C\">" + MakeReturnField("#define") + symbol
1398         m = re.search(r'^\s*#\s*define\s+\w+(\([^\)]*\))', declaration)
1399         if m:
1400             args = m.group(1)
1401             pad = ' ' * RETURN_TYPE_FIELD_WIDTH - len("#define ")
1402             # Align each line so that if should all line up OK.
1403             args = args.replace('\n', '\n' + pad)
1404             desc += CreateValidSGML(args)
1405
1406         desc += "</programlisting>\n"
1407
1408
1409     desc += MakeDeprecationNote(symbol)
1410
1411     parameters = OutputParamDescriptions("MACRO", symbol, fields)
1412
1413     if symbol in SymbolDocs:
1414         symbol_docs = ConvertMarkDown(symbol, SymbolDocs[symbol])
1415         desc += symbol_docs
1416
1417
1418     desc += parameters
1419     desc += OutputSymbolTraits(symbol)
1420     desc += "</refsect2>\n"
1421     return (synop, desc)
1422
1423
1424 #############################################################################
1425 # Function    : OutputTypedef
1426 # Description : Returns the synopsis and detailed description of a typedef.
1427 # Arguments   : $symbol - the typedef.
1428 #                $declaration - the declaration of the typedef,
1429 #                  e.g. 'typedef unsigned int guint;'
1430 #############################################################################
1431
1432 def OutputTypedef(symbol, declaration):
1433     sid = common.CreateValidSGMLID(symbol)
1434     condition = MakeConditionDescription(symbol)
1435     desc = "<refsect2 id=\"%s\" role=\"typedef\"%s>\n<title>%s</title>\n" % (sid, condition, symbol)
1436     synop = "<row><entry role=\"typedef_keyword\">typedef</entry><entry role=\"function_name\"><link linkend=\"%s\">%s</link></entry></row>\n" % (sid, symbol)
1437
1438     desc += MakeIndexterms(symbol, sid)
1439     desc += "\n"
1440     desc += OutputSymbolExtraLinks(symbol)
1441
1442     if  symbol in DeclarationConditional:
1443         decl_out = CreateValidSGML(declaration)
1444         desc += "<programlisting language=\"C\">%s</programlisting>\n" % decl_out
1445
1446
1447     desc += MakeDeprecationNote(symbol)
1448
1449     if symbol in SymbolDocs:
1450         desc += ConvertMarkDown(symbol, SymbolDocs[symbol])
1451
1452     desc += OutputSymbolTraits(symbol)
1453     desc += "</refsect2>\n"
1454     return (synop, desc)
1455
1456
1457
1458 #############################################################################
1459 # Function    : OutputStruct
1460 # Description : Returns the synopsis and detailed description of a struct.
1461 #                We check if it is a object struct, and if so we only output
1462 #                parts of it that are noted as public fields.
1463 #                We also use a different IDs for object structs, since the
1464 #                original ID is used for the entire RefEntry.
1465 # Arguments   : $symbol - the struct.
1466 #                $declaration - the declaration of the struct.
1467 #############################################################################
1468
1469 def OutputStruct(symbol, declaration):
1470
1471     is_gtype = False
1472     default_to_public = True
1473     if CheckIsObject(symbol):
1474         logging.info("Found struct gtype: %s", symbol)
1475         is_gtype = True
1476         default_to_public = ObjectRoots[symbol] == 'GBoxed'
1477
1478
1479     sid = None
1480     condition = None
1481     if is_gtype:
1482         sid = common.CreateValidSGMLID(symbol + "_struct")
1483         condition = MakeConditionDescription(symbol + "_struct")
1484     else:
1485         sid = common.CreateValidSGMLID(symbol)
1486         condition = MakeConditionDescription(symbol)
1487
1488
1489     # Determine if it is a simple struct or it also has a typedef.
1490     has_typedef = False
1491     if StructHasTypedef[symbol] or re.search(r'^\s*typedef\s+', declaration):
1492         has_typedef = True
1493
1494     type_output = None
1495     desc = None
1496     if has_typedef:
1497         # For structs with typedefs we just output the struct name.
1498         type_output = ''
1499         desc = "<refsect2 id=\"%s\" role=\"struct\"%s>\n<title>%s</title>\n" % (sid, condition, symbol)
1500     else:
1501         type_output = "struct"
1502         desc = "<refsect2 id=\"%s\" role=\"struct\"%s>\n<title>struct %s</title>\n" % (sid, condition, symbol)
1503
1504     synop = "<row><entry role=\"datatype_keyword\">%s</entry><entry role=\"function_name\"><link linkend=\"%s\">%s</link></entry></row>\n" % (type_output, sid, symbol)
1505
1506     desc += MakeIndexterms(symbol, sid)
1507     desc += "\n"
1508     desc += OutputSymbolExtraLinks(symbol)
1509
1510     # Form a pretty-printed, private-data-removed form of the declaration
1511
1512     decl_out = ''
1513     if re.search(r'^\s*$', declaration):
1514         logging.info("Found opaque struct: %s", symbol)
1515         decl_out = "typedef struct _%s %s;" % (symbol, symbol)
1516     elif re.search(r'^\s*struct\s+\w+\s*;\s*', declaration):
1517         logging.info("Found opaque struct: %s", symbol)
1518         decl_out = "struct %s;" % symbol
1519     else:
1520         public = default_to_public
1521         new_declaration = ''
1522         decl_line = None
1523         decl = declaration
1524
1525         m = re.search(r'^\s*(typedef\s+)?struct\s*\w*\s*(?:\/\*.*\*\/)?\s*{(.*)}\s*\w*\s*;\s*$', decl, flags=re.S)
1526         if m:
1527             struct_contents = m.group(2)
1528
1529             for decl_line  in struct_contents.split('\n'):
1530                 logging.info("Struct line: %s", decl_line)
1531                 m2 = re.search(r'/\*\s*<\s*public\s*>\s*\*/', decl_line)
1532                 m3 = re.search(r'/\*\s*<\s*(private|protected)\s*>\s*\*/', decl_line)
1533                 if m2:
1534                     public = True
1535                 elif m3:
1536                     public = False
1537                 elif public:
1538                     new_declaration += decl_line + "\n"
1539
1540             if new_declaration:
1541                 # Strip any blank lines off the ends.
1542                 new_declaration = '\n'.join([x.strip() for x in new_declaration.split('\n')])
1543
1544                 if has_typedef:
1545                     decl_out = "typedef struct {\n" + new_declaration + "} $symbol;\n"
1546                 else:
1547                     decl_out = "struct %s {\n%s };\n" % (symbol, new_declaration)
1548
1549         else:
1550             common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1551                               "Couldn't parse struct:\n%s" % declaration)
1552
1553         # If we couldn't parse the struct or it was all private, output an
1554         # empty struct declaration.
1555         if decl_out == '':
1556             if has_typedef:
1557                 decl_out = "typedef struct _%s %s;" % (symbol, symbol)
1558             else:
1559                 decl_out = "struct %s;" % symbol
1560
1561     decl_out = CreateValidSGML(decl_out)
1562     desc += "<programlisting language=\"C\">%s</programlisting>\n" % decl_out
1563
1564     desc += MakeDeprecationNote(symbol)
1565
1566     if symbol in SymbolDocs:
1567         desc += ConvertMarkDown(symbol, SymbolDocs[symbol])
1568
1569
1570     # Create a table of fields and descriptions
1571
1572     # FIXME: Inserting &#160's into the produced type declarations here would
1573     #        improve the output in most situations ... except for function
1574     #        members of structs!
1575     def depfunc(*args):
1576         return '<structfield id="%s">%s</structfield>' % (common.CreateValidSGMLID(sid + '.' + args[0]), args[0])
1577
1578     fields = common.ParseStructDeclaration(declaration, not default_to_public,
1579                                            0, MakeXRef, depfunc)
1580     params = SymbolParams[symbol]
1581
1582     # If no parameters are filled in, we don't generate the description
1583     # table, for backwards compatibility.
1584
1585     found = False
1586     if params:
1587         for i in range(1, len(params)+1, PARAM_FIELD_COUNT):
1588             if re.search(r'\S', params[i]):
1589                 found = True
1590                 break
1591
1592     if found:
1593         field_descrs = {} # FIXME, don't know what this does, original was: @$params
1594         missing_parameters = ''
1595         unused_parameters = ''
1596         sid = common.CreateValidSGMLID(symbol + ".members")
1597
1598         desc += '''<refsect3 id="%s" role="struct_members">\n<title>Members</title>
1599 <informaltable role="struct_members_table" pgwide="1" frame="none">
1600 <tgroup cols="3">
1601 <colspec colname="struct_members_name" colwidth="300px"/>
1602 <colspec colname="struct_members_description"/>
1603 <colspec colname="struct_members_annotations" colwidth="200px"/>
1604 <tbody>
1605 ''' % sid
1606
1607         while len(fields) > 0:
1608             field_name = fields.pop(0)
1609             text = fields.pop(0)
1610             field_descr = field_descrs[field_name]
1611             param_annotations = ''
1612
1613             desc += "<row role=\"member\"><entry role=\"struct_member_name\"><para>%s</para></entry>\n" % text
1614             if field_descr:
1615                 (field_descr, param_annotations) = ExpandAnnotation(symbol, field_descr)
1616                 field_descr = ConvertMarkDown(symbol, field_descr)
1617                 # trim
1618                 field_descr = re.sub(r'^(\s|\n)+', '', field_descr, flags=re.M|re.S)
1619                 field_descr = re.sub(r'(\s|\n)+$', '', field_descr, flags=re.M|re.S)
1620                 desc += "<entry role=\"struct_member_description\">%s</entry>\n<entry role=\"struct_member_annotations\">%s</entry>\n" % (field_descr, param_annotations)
1621                 del field_descrs[field_name]
1622             else:
1623                 common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1624                                   "Field description for %s::%s is missing in source code comment block." % (symbol, field_name))
1625                 if missing_parameters != '':
1626                     missing_parameters += ", " + field_name
1627                 else:
1628                     missing_parameters = field_name
1629
1630                 desc += "<entry /><entry />\n"
1631
1632             desc += "</row>\n"
1633
1634         desc += "</tbody></tgroup></informaltable>\n</refsect3>\n"
1635         for field_name in field_descrs:
1636             # Documenting those standard fields is not required anymore, but
1637             # we don't want to warn if they are documented anyway.
1638             m = re.search(r'(g_iface|parent_instance|parent_class)', field_name)
1639             if m:
1640                 continue
1641
1642             common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1643                               "Field description for %s::%s is not used from source code comment block." % (symbol, field_name))
1644             if unused_parameters != '':
1645                 unused_parameters += ", " + field_name
1646             else:
1647                 unused_parameters = field_name
1648
1649         # remember missing/unused parameters (needed in tmpl-free build)
1650         if missing_parameters != '' and (symbol not in AllIncompleteSymbols):
1651             AllIncompleteSymbols[symbol] = missing_parameters
1652
1653         if unused_parameters != '' and (symbol not in AllUnusedSymbols):
1654             AllUnusedSymbols[symbol] = unused_parameters
1655     else:
1656         if len(fields) > 0:
1657             if symbol not in AllIncompleteSymbols:
1658                 AllIncompleteSymbols[symbol] = "<items>"
1659                 common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1660                                   "Field descriptions for struct %s are missing in source code comment block." % symbol)
1661                 logging.info("Remaining structs fields: " + ':'.join(fields) + "\n")
1662
1663     desc += OutputSymbolTraits(symbol)
1664     desc += "</refsect2>\n"
1665     return (synop, desc)
1666
1667
1668
1669 #############################################################################
1670 # Function    : OutputUnion
1671 # Description : Returns the synopsis and detailed description of a union.
1672 # Arguments   : $symbol - the union.
1673 #                $declaration - the declaration of the union.
1674 #############################################################################
1675
1676 def OutputUnion(symbol, declaration):
1677
1678     is_gtype = False
1679     if CheckIsObject(symbol):
1680         logging.info("Found union gtype: %s\n", symbol)
1681         is_gtype = True
1682
1683
1684     sid = None
1685     condition = None
1686     if is_gtype:
1687         sid = common.CreateValidSGMLID(symbol + "_union")
1688         condition = MakeConditionDescription(symbol + "_union")
1689     else:
1690         sid = common.CreateValidSGMLID(symbol)
1691         condition = MakeConditionDescription(symbol)
1692
1693
1694     # Determine if it is a simple struct or it also has a typedef.
1695     has_typedef = False
1696     if StructHasTypedef[symbol] or re.search(r'^\s*typedef\s+', declaration):
1697         has_typedef = True
1698
1699
1700     type_output = None
1701     desc = None
1702     if has_typedef:
1703         # For unions with typedefs we just output the union name.
1704         type_output = ''
1705         desc = "<refsect2 id=\"%s\" role=\"union\"%s>\n<title>%s</title>\n" % (sid, condition, symbol)
1706     else:
1707         type_output = "union"
1708         desc = "<refsect2 id=\"%s\" role=\"union\"%s>\n<title>union %s</title>\n" % (sid, condition, symbol)
1709
1710     synop = "<row><entry role=\"datatype_keyword\">%s</entry><entry role=\"function_name\"><link linkend=\"%s$id\">%s</link></entry></row>\n" % (type_output, sid, symbol)
1711
1712     desc += MakeIndexterms(symbol, sid)
1713     desc += "\n"
1714     desc += OutputSymbolExtraLinks(symbol)
1715     desc += MakeDeprecationNote(symbol)
1716
1717     if symbol in SymbolDocs:
1718         desc += ConvertMarkDown(symbol, SymbolDocs[symbol])
1719
1720
1721     # Create a table of fields and descriptions
1722
1723     # FIXME: Inserting &#160's into the produced type declarations here would
1724     #        improve the output in most situations ... except for function
1725     #        members of structs!
1726     def pfunc(*args):
1727         return '<structfield id="%s">%s</structfield>' % (common.CreateValidSGMLID(sid + '.' + args[0]), args[0])
1728     fields = common.ParseStructDeclaration(declaration, 0, 0, MakeXRef, pfunc)
1729     params = SymbolParams[symbol]
1730
1731     # If no parameters are filled in, we don't generate the description
1732     # table, for backwards compatibility
1733
1734     found = False
1735     if params:
1736         for i in range(1, len(params)+1, PARAM_FIELD_COUNT):
1737             if re.search(r'\S', params[i]):
1738                 found = True
1739                 break
1740
1741     if found:
1742         field_descrs = {} # FIXME same as above: @$params
1743         missing_parameters = ''
1744         unused_parameters = ''
1745         sid = common.CreateValidSGMLID('%s.members' % symbol)
1746
1747         desc += '''<refsect3 id="%s" role="union_members">\n<title>Members</title>
1748 <informaltable role="union_members_table" pgwide="1" frame="none">
1749 <tgroup cols="3">
1750 <colspec colname="union_members_name" colwidth="300px"/>
1751 <colspec colname="union_members_description"/>
1752 <colspec colname="union_members_annotations" colwidth="200px"/>
1753 <tbody>
1754 ''' % sid
1755
1756         while len(fields) > 0:
1757             field_name = fields.pop(0)
1758             text = fields.pop(0)
1759             field_descr = field_descrs[field_name]
1760             param_annotations = ''
1761
1762             desc += "<row><entry role=\"union_member_name\"><para>%s</para></entry>\n" % text
1763             if field_descr:
1764                 (field_descr, param_annotations) = ExpandAnnotation(symbol, field_descr)
1765                 field_descr = ConvertMarkDown(symbol, field_descr)
1766
1767                 # trim
1768                 field_descr = re.sub(r'^(\s|\n)+', '', field_descr, flags=re.M|re.S)
1769                 field_descr = re.sub(r'(\s|\n)+$', '', field_descr, flags=re.M|re.S)
1770                 desc += "<entry role=\"union_member_description\">%s</entry>\n<entry role=\"union_member_annotations\">%s</entry>\n" % (field_descr, param_annotations)
1771                 del field_descrs[field_name]
1772             else:
1773                 common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1774                                   "Field description for %s::%s is missing in source code comment block." % (symbol, field_name))
1775                 if missing_parameters != '':
1776                     missing_parameters += ", " + field_name
1777                 else:
1778                     missing_parameters = field_name
1779
1780                 desc += "<entry /><entry />\n"
1781
1782             desc += "</row>\n"
1783
1784         desc += "</tbody></tgroup></informaltable>\n</refsect3>"
1785         for field_name in field_descrs:
1786             common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1787                               "Field description for %s::%s is not used from source code comment block." % (symbol, field_name))
1788             if unused_parameters != '':
1789                 unused_parameters += ", " + field_name
1790             else:
1791                 unused_parameters = field_name
1792
1793         # remember missing/unused parameters (needed in tmpl-free build)
1794         if missing_parameters != '' and (symbol not in AllIncompleteSymbols):
1795             AllIncompleteSymbols[symbol] = missing_parameters
1796
1797         if unused_parameters != '' and (symbol not in AllUnusedSymbols):
1798             AllUnusedSymbols[symbol] = unused_parameters
1799     else:
1800         if len(fields) > 0:
1801             if symbol not in AllIncompleteSymbols:
1802                 AllIncompleteSymbols[symbol] = "<items>"
1803                 common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1804                                   "Field descriptions for union %s are missing in source code comment block." % symbol)
1805                 logging.info("Remaining union fields: " + ':'.join(fields) + "\n")
1806
1807     desc += OutputSymbolTraits(symbol)
1808     desc += "</refsect2>\n"
1809     return (synop, desc)
1810
1811 #############################################################################
1812 # Function    : OutputEnum
1813 # Description : Returns the synopsis and detailed description of a enum.
1814 # Arguments   : $symbol - the enum.
1815 #                $declaration - the declaration of the enum.
1816 #############################################################################
1817
1818 def OutputEnum(symbol, declaration):
1819     is_gtype = False
1820     if CheckIsObject(symbol):
1821         logging.info("Found enum gtype: %s", symbol)
1822         is_gtype = True
1823
1824     sid = None
1825     condition = None
1826     if is_gtype:
1827         sid = common.CreateValidSGMLID(symbol + "_enum")
1828         condition = MakeConditionDescription(symbol + "_enum")
1829     else:
1830         sid = common.CreateValidSGMLID(symbol)
1831         condition = MakeConditionDescription(symbol)
1832
1833
1834     synop = "<row><entry role=\"datatype_keyword\">enum</entry><entry role=\"function_name\"><link linkend=\"\">$symbol</link></entry></row>\n" % sid
1835     desc = "<refsect2 id=\"%s\" role=\"enum\"%s>\n<title>enum %s</title>\n" % (sid, condition, symbol)
1836
1837     desc += MakeIndexterms(symbol, sid)
1838     desc += "\n"
1839     desc += OutputSymbolExtraLinks(symbol)
1840     desc += MakeDeprecationNote(symbol)
1841
1842     if symbol in SymbolDocs:
1843         desc += ConvertMarkDown(symbol, SymbolDocs[symbol])
1844
1845     # Create a table of fields and descriptions
1846
1847     fields = common.ParseEnumDeclaration(declaration)
1848     params = SymbolParams[symbol]
1849
1850     # If nothing at all is documented log a single summary warning at the end.
1851     # Otherwise, warn about each undocumented item.
1852
1853     found = False
1854     if params:
1855         for i in range(1, len(params), PARAM_FIELD_COUNT):
1856             if re.search(r'\S', params[i]):
1857                 found = True
1858                 break
1859
1860     field_descrs = params if params else {}
1861     missing_parameters = ''
1862     unused_parameters = ''
1863
1864     sid = common.CreateValidSGMLID("%s.members" % symbol)
1865     desc += '''<refsect3 id="%s" role="enum_members">\n<title>Members</title>
1866 <informaltable role="enum_members_table" pgwide="1" frame="none">
1867 <tgroup cols="3">
1868 <colspec colname="enum_members_name" colwidth="300px"/>
1869 <colspec colname="enum_members_description"/>
1870 <colspec colname="enum_members_annotations" colwidth="200px"/>
1871 <tbody>
1872 ''' % sid
1873
1874     for field_name in fields:
1875         field_descr = field_descrs[field_name]
1876         param_annotations = ''
1877
1878         sid = common.CreateValidSGMLID(field_name)
1879         condition = MakeConditionDescription(field_name)
1880         desc += "<row role=\"constant\"><entry role=\"enum_member_name\"><para id=\"%s\">$field_name</para></entry>\n" % sid
1881         if field_descr:
1882             field_descr, param_annotations = ExpandAnnotation(symbol, field_descr)
1883             field_descr = ConvertMarkDown(symbol, field_descr)
1884             desc += "<entry role=\"enum_member_description\">%s</entry>\n<entry role=\"enum_member_annotations\">%s</entry>\n" % (field_descr, param_annotations)
1885             del field_descrs[field_name]
1886         else:
1887             if found:
1888                 common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1889                                   "Value description for %s::%s is missing in source code comment block." % (symbol, field_name))
1890                 if missing_parameters != '':
1891                     missing_parameters += ", " + field_name
1892                 else:
1893                     missing_parameters = field_name
1894             desc += "<entry /><entry />\n"
1895         desc += "</row>\n"
1896
1897     desc += "</tbody></tgroup></informaltable>\n</refsect3>"
1898     for field_name in field_descrs:
1899         common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1900                           "Value description for %s::%s is not used from source code comment block." % (symbol, field_name))
1901         if unused_parameters != '':
1902             unused_parameters += ", " + field_name
1903         else:
1904             unused_parameters = field_name
1905
1906     # remember missing/unused parameters (needed in tmpl-free build)
1907     if missing_parameters != '' and (symbol not in AllIncompleteSymbols):
1908         AllIncompleteSymbols[symbol] = missing_parameters
1909
1910     if unused_parameters != '' and (symbol not in AllUnusedSymbols):
1911         AllUnusedSymbols[symbol] = unused_parameters
1912
1913     if not found:
1914         if len(fields) > 0:
1915             if symbol not in AllIncompleteSymbols:
1916                 AllIncompleteSymbols[symbol] = "<items>"
1917                 common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
1918                                   "Value descriptions for %s are missing in source code comment block." % symbol)
1919
1920     desc += OutputSymbolTraits(symbol)
1921     desc += "</refsect2>\n"
1922     return (synop, desc)
1923
1924
1925
1926 #############################################################################
1927 # Function    : OutputVariable
1928 # Description : Returns the synopsis and detailed description of a variable.
1929 # Arguments   : $symbol - the extern'ed variable.
1930 #                $declaration - the declaration of the variable.
1931 #############################################################################
1932
1933 def OutputVariable(symbol, declaration):
1934     sid = common.CreateValidSGMLID(symbol)
1935     condition = MakeConditionDescription(symbol)
1936
1937     logging.info("ouputing variable: '%s' '%s'", symbol, declaration)
1938
1939     type_output = None
1940     m1 = re.search(r'^\s*extern\s+((const\s+|signed\s+|unsigned\s+|long\s+|short\s+)*\w+)(\s+\*+|\*+|\s)(\s*)(const\s+)*([A-Za-z]\w*)\s*;', declaration)
1941     m2 = re.search(r'\s*((const\s+|signed\s+|unsigned\s+|long\s+|short\s+)*\w+)(\s+\*+|\*+|\s)(\s*)(const\s+)*([A-Za-z]\w*)\s*=', declaration)
1942     if m1:
1943         g1 = m1.group(1)
1944         g2 = m1.group(2)
1945         g3 = m1.group(3)
1946         g4 = m1.group(4)
1947         g5 = m1.group(5)
1948         mod1 = g1 if g1 else ''
1949         ptr = g3 if g2 else ''
1950         space = g4 if g4 else ''
1951         mod2 = g5 if g5 else ''
1952         type_output = "extern %s%s%s%s" % (mod1, ptr, space, mod2)
1953     elif m2:
1954         g1 = m1.group(1)
1955         g2 = m1.group(2)
1956         g3 = m1.group(3)
1957         g4 = m1.group(4)
1958         g5 = m1.group(5)
1959         mod1 = g1 if g1 else ''
1960         ptr = g3 if g3 else ''
1961         space = g4 if g4 else ''
1962         mod2 = g5 if g5 else ''
1963         type_output = '%s%s%s%s' % (mod1, ptr, space, mod2)
1964     else:
1965         type_output = "extern"
1966
1967     synop = "<row><entry role=\"variable_type\">%s</entry><entry role=\"function_name\"><link linkend=\"%s\">%s</link></entry></row>\n" % (type_output, sid, symbol)
1968
1969     desc = "<refsect2 id=\"%s\" role=\"variable\"%s>\n<title>%s</title>\n" % (sid, condition, symbol)
1970
1971     desc += MakeIndexterms(symbol, sid)
1972     desc += "\n"
1973     desc += OutputSymbolExtraLinks(symbol)
1974
1975     decl_out = CreateValidSGML(declaration)
1976     desc += "<programlisting language=\"C\">%s</programlisting>\n" % decl_out
1977
1978     desc += MakeDeprecationNote(symbol)
1979
1980     if symbol in SymbolDocs:
1981         desc += ConvertMarkDown(symbol, SymbolDocs[symbol])
1982
1983     if symbol in SymbolAnnotations:
1984         param_desc = SymbolAnnotations[symbol]
1985         param_annotations = ''
1986         (param_desc, param_annotations) = ExpandAnnotation(symbol, param_desc)
1987         if param_annotations != '':
1988             desc += "\n<para>$param_annotations</para>"
1989
1990
1991
1992     desc += OutputSymbolTraits(symbol)
1993     desc += "</refsect2>\n"
1994     return (synop, desc)
1995
1996
1997
1998 #############################################################################
1999 # Function    : OutputFunction
2000 # Description : Returns the synopsis and detailed description of a function.
2001 # Arguments   : $symbol - the function.
2002 #                $declaration - the declaration of the function.
2003 #############################################################################
2004
2005 def OutputFunction(symbol, declaration, symbol_type):
2006     sid = common.CreateValidSGMLID(symbol)
2007     condition = MakeConditionDescription(symbol)
2008
2009     # Take out the return type          $1                                                                                       $2   $3
2010     m = re.search(r'<RETURNS>\s*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|long\s+|short\s+|struct\s+|enum\s+)*)(\w+)(\s*\**\s*(?:const|G_CONST_RETURN)?\s*\**\s*(?:restrict)?\s*)<\/RETURNS>\n', declaration)
2011     type_modifier = m.group(1) if m.group(1) else ''
2012     type = m.group(2)
2013     pointer = m.group(3)
2014     # Trim trailing spaces as we are going to pad to $RETURN_TYPE_FIELD_WIDTH below anyway
2015     pointer = pointer.rstrip()
2016     xref = MakeXRef(type, tagify(type, "returnvalue"))
2017     start = ''
2018     #if ($symbol_type == 'USER_FUNCTION')
2019     #    $start = "typedef "
2020     #
2021
2022     # We output const rather than G_CONST_RETURN.
2023     type_modifier = re.sub(r'G_CONST_RETURN', 'const', type_modifier)
2024     pointer = re.sub(r'G_CONST_RETURN', 'const', pointer)
2025     pointer = re.sub(r'^\s+', '&#160', pointer)
2026
2027     ret_type_output = None
2028     ret_type_output = "%s%s%s%s\n" % (start, type_modifier, xref, pointer)
2029
2030     indent_len = len(symbol) + 2
2031     char1 = char2 = char3 = ''
2032     if symbol_type == 'USER_FUNCTION':
2033         indent_len += 3
2034         char1 = "<phrase role=\"c_punctuation\">(</phrase>"
2035         char2 = "*"
2036         char3 = "<phrase role=\"c_punctuation\">)</phrase>"
2037
2038
2039     symbol_output = "%s<link linkend=\"%s\">%s%s</link>%s" % (char1, sid, char2, symbol, char3)
2040     if indent_len < MAX_SYMBOL_FIELD_WIDTH:
2041         symbol_desc_output = "%s%s%s%s" % (char1, char2, symbol, char3)
2042     else:
2043         indent_len = MAX_SYMBOL_FIELD_WIDTH - 8
2044         symbol_desc_output = ('%s%s%s%s\n' % (char1, char2, symbol, char3)) + (' ' * (indent_len - 1))
2045
2046     synop = "<row><entry role=\"function_type\">%s</entry><entry role=\"function_name\">%s&#160;<phrase role=\"c_punctuation\">()</phrase></entry></row>\n" % (ret_type_output, symbol_output)
2047
2048     desc = "<refsect2 id=\"%s\" role=\"function\"%s>\n<title>%s&#160;()</title>\n" % (sid, condition, symbol)
2049
2050     desc += MakeIndexterms(symbol, sid)
2051     desc += "\n"
2052     desc += OutputSymbolExtraLinks(symbol)
2053
2054     desc += "<programlisting language=\"C\">%s%s(" % (ret_type_output, symbol_desc_output)
2055
2056     def tagfun(*args):
2057         return tagify(args[0], "parameter")
2058
2059     fields = common.ParseFunctionDeclaration(declaration, MakeXRef, tagfun)
2060
2061     for i in range(1, len(fields) +1, 2):
2062         field_name = fields[i]
2063
2064         if i == 1:
2065             desc += field_name
2066         else:
2067             desc += ",\n" + (' ' * indent_len) + field_name
2068
2069     desc += ");</programlisting>\n"
2070
2071     desc += MakeDeprecationNote(symbol)
2072
2073     if symbol in SymbolDocs:
2074         desc += ConvertMarkDown(symbol, SymbolDocs[symbol])
2075
2076     if symbol in SymbolAnnotations:
2077         param_desc = SymbolAnnotations[symbol]
2078         param_annotations = ''
2079         (param_desc, param_annotations) = ExpandAnnotation(symbol, param_desc)
2080         if param_annotations != '':
2081             desc += "\n<para>%s</para>" % param_annotations
2082
2083     desc += OutputParamDescriptions("FUNCTION", symbol, fields)
2084     desc += OutputSymbolTraits(symbol)
2085     desc += "</refsect2>\n"
2086     return (synop, desc)
2087
2088 #############################################################################
2089 # Function    : OutputParamDescriptions
2090 # Description : Returns the DocBook output describing the parameters of a
2091 #                function, macro or signal handler.
2092 # Arguments   : $symbol_type - 'FUNCTION', 'MACRO' or 'SIGNAL'. Signal
2093 #                  handlers have an implicit user_data parameter last.
2094 #                $symbol - the name of the function/macro being described.
2095 #               @fields - parsed fields from the declaration, used to determine
2096 #                  undocumented/unused entries
2097 #############################################################################
2098
2099 def OutputParamDescriptions(symbol_type, symbol, fields):
2100     output = ''
2101     params = SymbolParams[symbol]
2102     num_params = 0
2103     field_descrs = {}
2104
2105     if len(fields) > 0:
2106         field_descrs = {} # FIXME convert @fields
2107         del field_descrs["void"]
2108         del field_descrs["Returns"]
2109
2110
2111     if params:
2112         returns = ''
2113         params_desc = ''
2114         missing_parameters = ''
2115         unused_parameters = ''
2116
2117         for j in range(0, len(params), PARAM_FIELD_COUNT):
2118             param_name = params[j]
2119             param_desc = params[j + 1]
2120             param_annotations = ''
2121
2122             (param_desc, param_annotations) = ExpandAnnotation(symbol, param_desc)
2123             param_desc = ConvertMarkDown(symbol, param_desc)
2124             # trim
2125             param_desc = re.sub(r'^(\s|\n)+', '', param_desc, flags=re.M|re.S)
2126             param_desc = re.sub(r'(\s|\n)+', '', param_desc, flags=re.M|re.S)
2127             if param_name == "Returns":
2128                 returns = param_desc
2129                 if param_annotations != '':
2130                     returns += "\n<para>%s</para>" % param_annotations
2131
2132                 elif param_name == "void":
2133                     # FIXME: &common.LogWarning()?
2134                     logging.info("!!!! void in params for %s?\n", symbol)
2135             else:
2136                 if len(fields) > 0:
2137                     if param_name in field_descrs:
2138                         common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
2139                                           "Parameter description for %s::%s is not used from source code comment block." % (symbol, param_name))
2140                         if unused_parameters != '':
2141                             unused_parameters += ", " + param_name
2142                         else:
2143                             unused_parameters = param_name
2144                     else:
2145                         del field_descrs[param_name]
2146
2147
2148                 if param_desc != '':
2149                     params_desc += "<row><entry role=\"parameter_name\"><para>%s</para></entry>\n<entry role=\"parameter_description\">%s</entry>\n<entry role=\"parameter_annotations\">%s</entry></row>\n" % (param_name, param_desc, param_annotations)
2150                     num_params += 1
2151
2152         for param_name in field_descrs:
2153             common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
2154                               "Parameter description for %s::%s is missing in source code comment block." % (symbol, param_name))
2155             if missing_parameters != '':
2156                 missing_parameters += ", " + param_name
2157             else:
2158                 missing_parameters = param_name
2159
2160         # Signals have an implicit user_data parameter which we describe.
2161         if symbol_type == "SIGNAL":
2162             params_desc += "<row><entry role=\"parameter_name\"><simpara>user_data</simpara></entry>\n<entry role=\"parameter_description\"><simpara>user data set when the signal handler was connected.</simpara></entry>\n<entry role=\"parameter_annotations\"></entry></row>\n"
2163
2164
2165         # Start a table if we need one.
2166         if params_desc != '':
2167             sid = common.CreateValidSGMLID("%s.parameters" % symbol)
2168
2169             output += '''<refsect3 id="%s" role="parameters">\n<title>Parameters</title>
2170 <informaltable role="parameters_table" pgwide="1" frame="none">
2171 <tgroup cols="3">
2172 <colspec colname="parameters_name" colwidth="150px"/>
2173 <colspec colname="parameters_description"/>
2174 <colspec colname="parameters_annotations" colwidth="200px"/>
2175 <tbody>
2176 ''' % sid
2177             output += params_desc
2178             output += "</tbody></tgroup></informaltable>\n</refsect3>"
2179
2180         # Output the returns info last
2181         if returns != '':
2182             sid = common.CreateValidSGMLID("%s.returns" % symbol)
2183
2184             output += '''
2185 <refsect3 id="%s" role=\"returns\">\n<title>Returns</title>
2186 ''' % sid
2187             output += returns
2188             output += "\n</refsect3>"
2189
2190
2191         # remember missing/unused parameters (needed in tmpl-free build)
2192         if missing_parameters != '' and (symbol not in AllIncompleteSymbols):
2193             AllIncompleteSymbols[symbol] = missing_parameters
2194
2195         if unused_parameters != '' and (symbol not in AllUnusedSymbols):
2196             AllUnusedSymbols[symbol] = unused_parameters
2197
2198     if num_params == 0 and len(fields) > 0 and len(field_descrs) > 0:
2199         if symbol not in AllIncompleteSymbols:
2200             AllIncompleteSymbols[symbol] = "<parameters>"
2201     return output
2202
2203
2204
2205 #############################################################################
2206 # Function    : ParseStabilityLevel
2207 # Description : Parses a stability level and outputs a warning if it isn't
2208 #               valid.
2209 # Arguments   : $stability - the stability text.
2210 #                $file, $line - context for error message
2211 #                $message - description of where the level is from, to use in
2212 #               any error message.
2213 # Returns     : The parsed stability level string.
2214 #############################################################################
2215
2216 def ParseStabilityLevel(stability, file, line, message):
2217
2218     stability = stability.strip()
2219     sl = stability.lower()
2220
2221     if sl == 'stable':
2222         stability = "Stable"
2223     elif stability == 'unstable':
2224         stability = "Unstable"
2225     elif stability == 'private':
2226         stability = "Private"
2227     else:
2228         common.LogWarning(file, line, "%s is %s." % (message, stability) +\
2229             "It should be one of these: Stable, Unstable, or Private.")
2230     return stability
2231
2232
2233
2234 #############################################################################
2235 # Function    : OutputDBFile
2236 # Description : Outputs the final DocBook file for one section.
2237 # Arguments   : $file - the name of the file.
2238 #               $title - the title from the $MODULE-sections.txt file, which
2239 #                 will be overridden by the title in the template file.
2240 #               $section_id - the id to use for the toplevel tag.
2241 #               $includes - comma-separates list of include files added at top of
2242 #                 synopsis, with '<' '>' around them (if not already enclosed in '').
2243 #               $functions_synop - reference to the DocBook for the Functions Synopsis part.
2244 #               $other_synop - reference to the DocBook for the Types and Values Synopsis part.
2245 #               $functions_details - reference to the DocBook for the Functions Details part.
2246 #               $other_details - reference to the DocBook for the Types and Values Details part.
2247 #               $signal_synop - reference to the DocBook for the Signal Synopsis part
2248 #               $signal_desc - reference to the DocBook for the Signal Description part
2249 #               $args_synop - reference to the DocBook for the Arg Synopsis part
2250 #               $args_desc - reference to the DocBook for the Arg Description part
2251 #               $hierarchy - reference to the DocBook for the Object Hierarchy part
2252 #               $interfaces - reference to the DocBook for the Interfaces part
2253 #               $implementations - reference to the DocBook for the Known Implementations part
2254 #               $prerequisites - reference to the DocBook for the Prerequisites part
2255 #               $derived - reference to the DocBook for the Derived Interfaces part
2256 #               $file_objects - reference to an array of objects in this file
2257 #############################################################################
2258
2259 def OutputDBFile(file, title, section_id, includes, functions_synop, other_synop, functions_details, other_details, signals_synop, signals_desc, args_synop, args_desc, hierarchy, interfaces, implementations, prerequisites, derived, file_objects):
2260
2261     logging.info("Output docbook for file %s with title '%s'\n", file, title)
2262
2263     # The edited title overrides the one from the sections file.
2264     new_title = SymbolDocs[os.path.join(TMPL_DIR, file + ":Title")]
2265     if new_title and not re.search(r'^\s*$', new_title):
2266         title = new_title
2267         logging.info("Found title: %s", title)
2268
2269     short_desc = SymbolDocs[os.path.join(TMPL_DIR, file + ":Short_Description")]
2270     if not short_desc or re.search(r'^\s*$', short_desc):
2271         short_desc = ''
2272     else:
2273         # Don't use ConvertMarkDown here for now since we don't want blocks
2274         short_desc = ExpandAbbreviations(":Short_description" % title,
2275                                          short_desc)
2276         logging.info("Found short_desc: %s", short_desc)
2277
2278     long_desc = SymbolDocs[os.path.join(TMPL_DIR, file + ":Long_Description")]
2279     if not long_desc or re.search(r'^\s*$', long_desc):
2280         long_desc = ''
2281     else:
2282         long_desc = ConvertMarkDown(":Long_description" % title,
2283                                     long_desc)
2284         logging.info("Found long_desc: ", long_desc)
2285
2286     see_also = SymbolDocs[os.path.join(TMPL_DIR, file + ":See_Also")]
2287     if not see_also or re.search(r'^\s*(<para>)?\s*(</para>)?\s*$', see_also):
2288         see_also = ''
2289     else:
2290         see_also = ConvertMarkDown("%s:See_Also" % title, see_also)
2291         logging.info("Found see_also: %s", see_also)
2292
2293     if see_also:
2294         see_also = "<refsect1 id=\"%s.see-also\">\n<title>See Also</title>\n%s\n</refsect1>\n" % (section_id, see_also)
2295
2296     stability = SymbolDocs[os.path.join(TMPL_DIR, file + ":Stability_Level")]
2297     if not stability or re.search(r'^\s*$', stability):
2298         stability = ''
2299     else:
2300         stability = ParseStabilityLevel(stability, file, line_number, "Section stability level")
2301         logging.info("Found stability: %s", stability)
2302
2303     if stability:
2304         AnnotationsUsed[stability] = 1
2305         stability = "<refsect1 id=\"%s.stability-level\">\n<title>Stability Level</title>\n<acronym>%s</acronym>, unless otherwise indicated\n</refsect1>\n" % (section_id, stability)
2306     elif DEFAULT_STABILITY:
2307         AnnotationsUsed[DEFAULT_STABILITY] = 1
2308         stability = "<refsect1 id=\"%s.stability-level\">\n<title>Stability Level</title>\n<acronym>%s</acronym>, unless otherwise indicated\n</refsect1>\n" % (section_id, DEFAULT_STABILITY)
2309
2310
2311     image = SymbolDocs[os.path.join(TMPL_DIR, file + ":Image")]
2312     if image or re.search(r'^\s*$', image):
2313         image = ''
2314     else:
2315         image = image.strip()
2316
2317         format = None
2318
2319         il = image.lower()
2320         if re.search(r'jpe?g$', il):
2321             format = "format='JPEG'"
2322         elif il.endswith('png'):
2323             format = "format='PNG'"
2324         elif il.endswith('svg'):
2325             format = "format='SVG'"
2326         else:
2327             format = ''
2328
2329
2330         image = "  <inlinegraphic fileref='%s' %s/>\n" % (image, format)
2331
2332     include_output = ''
2333     if includes:
2334         include_output += "<refsect1 id=\"%s.includes\"><title>Includes</title><synopsis>" % section_id
2335         for include in includes.split(','):
2336             if re.search(r'^\".+\"$', include):
2337                 include_output += "#include %s\n" % include
2338             else:
2339                 include = re.sub(r'^\s+|\s+$', '', include, flags=re.S)
2340                 include_output += "#include &lt;%s&gt;\n" % include
2341
2342
2343         include_output += "</synopsis></refsect1>\n"
2344
2345
2346     extralinks = OutputSectionExtraLinks(title, "Section:%s" % file)
2347
2348     old_db_file = os.path.join(DB_OUTPUT_DIR, file + '.xml')
2349     new_db_file = os.path.join(DB_OUTPUT_DIR, file, '.xml.new')
2350
2351     OUTPUT = open(new_db_file, 'w')
2352
2353     object_anchors = ''
2354     for fobject in file_objects:
2355         if fobject == section_id:
2356             continue
2357         sid = common.CreateValidSGMLID(fobject)
2358         logging.info("Adding anchor for %s\n", fobject)
2359         object_anchors += "<anchor id=\"%s\"/>" % sid
2360
2361     # Make sure we produce valid docbook
2362     if not functions_details:
2363         functions_details = "<para />"
2364
2365     # We used to output this, but is messes up our common.UpdateFileIfChanged code
2366     # since it changes every day (and it is only used in the man pages):
2367     # "<refentry id="$section_id" revision="$mday $month $year">"
2368
2369     OUTPUT.write(r'''${\( MakeDocHeader ("refentry") )
2370 <refentry id="$section_id">
2371 <refmeta>
2372 <refentrytitle role="top_of_page" id="%s.top_of_page">%s</refentrytitle>
2373 <manvolnum>3</manvolnum>
2374 <refmiscinfo>\U%s\E Library%s</refmiscinfo>
2375 </refmeta>
2376 <refnamediv>
2377 <refname>%s</refname>
2378 <refpurpose>%s</refpurpose>
2379 </refnamediv>
2380 %s
2381 %s%s%s%s%s%s%s%s%s%s
2382 $include_output
2383 <refsect1 id="%s.description" role="desc">
2384 <title role="desc.title">Description</title>
2385 %s%s
2386 </refsect1>
2387 <refsect1 id="%s.functions_details" role="details">
2388 <title role="details.title">Functions</title>
2389 %s
2390 </refsect1>
2391 <refsect1 id="%s.other_details" role="details">
2392 <title role="details.title">Types and Values</title>
2393 %s
2394 </refsect1>
2395 %s%s%s
2396 </refentry>
2397 ''' % (section_id, title, MODULE, image, title, short_desc, stability, functions_synop, args_synop, signals_synop, object_anchors, other_synop, hierarchy, prerequisites, derived, interfaces, implementations, section_id, extralinks, long_desc, section_id, functions_details, section_id, other_details, args_desc, signals_desc, see_also))
2398     OUTPUT.close()
2399
2400     return common.UpdateFileIfChanged(old_db_file, new_db_file, 0)
2401
2402
2403 #############################################################################
2404 # Function    : OutputProgramDBFile
2405 # Description : Outputs the final DocBook file for one program.
2406 # Arguments   : $file - the name of the file.
2407 #               $section_id - the id to use for the toplevel tag.
2408 #############################################################################
2409
2410 def OutputProgramDBFile(program, section_id):
2411     logging.info("Output program docbook for %s", program)
2412
2413     short_desc = SourceSymbolDocs[os.path.join(TMPL_DIR, program + ":Short_Description")]
2414     if not short_desc or re.search(r'^\s*$', short_desc):
2415         short_desc = ''
2416     else:
2417         # Don't use ConvertMarkDown here for now since we don't want blocks
2418         short_desc = ExpandAbbreviations(program, short_desc)
2419         logging.info("Found short_desc: %s", short_desc)
2420
2421     synopsis = SourceSymbolDocs[os.path.join(TMPL_DIR, program + ":Synopsis")]
2422     if synopsis and re.search(r'~\s*$', synopsis):
2423         items = synopsis.split(' ')
2424         for i in range(0, len(items)):
2425             parameter = items[i]
2426             choice = "plain"
2427             rep = ''
2428
2429             # first parameter is the command name
2430             if i == 0:
2431                 synopsis = "<command>%s</command>\n" % parameter
2432                 continue
2433
2434             # square brackets indicate optional parameters, curly brackets
2435             # indicate required parameters ("plain" parameters are also
2436             # mandatory, but do not get extra decoration)
2437             m1 = re.search(r'^\[(.+?)\]$', parameter)
2438             m2 = re.search(r'^\{(.+?)\}$', parameter)
2439             m3 = re.search(r'\.\.\.$', parameter)
2440             m4 = re.search(r'\*(.+?)\*', parameter)
2441             if m1:
2442                 choice = "opt"
2443             elif m2:
2444                 choice = "req"
2445
2446             # parameters ending in "..." are repeatable
2447             if m3:
2448                 rep = ' rep=\"repeat\"'
2449
2450             # italic parameters are replaceable parameters
2451             if m4:
2452                 parameter = "<replaceable>%s</replaceable>" % parameter
2453
2454
2455             synopsis += "<arg choice=\"%s\"%s>" % (choice, rep)
2456             synopsis += parameter
2457             synopsis += "</arg>\n"
2458
2459         logging.info("Found synopsis: %s", synopsis)
2460     else:
2461         synopsis = "<command>%s</command>" % program
2462
2463
2464     long_desc = SourceSymbolDocs[os.path.join(TMPL_DIR, program + ":Long_Description")]
2465     if not long_desc or re.search(r'^\s*$', long_desc):
2466         long_desc = ''
2467     else:
2468         long_desc = ConvertMarkDown("%s:Long_description" % program, long_desc)
2469         logging.info("Found long_desc: %s", long_desc)
2470
2471     options = ''
2472     o = os.path.join(TMPL_DIR, program + ":Options")
2473     if o in SourceSymbolDocs:
2474         opts = SourceSymbolDocs[o]
2475
2476         options = "<refsect1>\n<title>Options</title>\n<variablelist>\n"
2477         for k in range(0, len(opts), 2):
2478             opt_desc = opts[k+1]
2479
2480             opt_desc = re.sub(r'\*(.+?)\*', r'<replaceable>\1<\/replaceable>', opt_desc)
2481
2482             options += "<varlistentry>\n<term>"
2483             opt_names = opts[k].split(' ')
2484             for i in range(len(opt_names)):
2485                 prefix = ', ' if i > 0 else ''
2486                 opt_names[i] = re.sub(r'\*(.+?)\*', r'<replaceable>\1<\/replaceable>', opt_names[i])
2487
2488                 options += "%s<option>%s</option>\n" % (prefix, opt_names[i])
2489
2490             options += "</term>\n"
2491             options += "<listitem><para>%s</para></listitem>\n" % opt_desc
2492             options += "</varlistentry>\n"
2493
2494         options += "</variablelist></refsect1>\n"
2495
2496
2497     exit_status = SourceSymbolDocs[os.path.join(TMPL_DIR, program + ":Returns")]
2498     if exit_status and exit_status != '':
2499         exit_status = ConvertMarkDown("%s:Returns" % program, exit_status)
2500         exit_status = "<refsect1 id=\"%s.exit-status\">\n<title>Exit Status</title>\n%s\n</refsect1>\n" % (section_id, exit_status)
2501     else:
2502         exit_status = ''
2503
2504
2505     see_also = SourceSymbolDocs[os.path.join(TMPL_DIR, program + ":See_Also")]
2506     if see_also or re.search(r'^\s*(<para>)?\s*(</para>)?\s*$', see_also):
2507         see_also = ''
2508     else:
2509         see_also = ConvertMarkDown("%s:See_Also" % program, see_also)
2510         logging.info("Found see_also: %s", see_also)
2511
2512     if see_also:
2513         see_also = "<refsect1 id=\"%s.see-also\">\n<title>See Also</title>\n%s\n</refsect1>\n" % (section_id, see_also)
2514
2515
2516     old_db_file = os.path.join(DB_OUTPUT_DIR, program + ".xml")
2517     new_db_file = os.path.join(DB_OUTPUT_DIR, program + ".xml.new")
2518
2519     OUTPUT = open(new_db_file, 'w')
2520
2521     OUTPUT.write('''%s
2522 <refentry id="%s">
2523 <refmeta>
2524 <refentrytitle role="top_of_page" id="%s.top_of_page">$program</refentrytitle>
2525 <manvolnum>1</manvolnum>
2526 <refmiscinfo>User Commands</refmiscinfo>
2527 </refmeta>
2528 <refnamediv>
2529 <refname>%s</refname>
2530 <refpurpose>%s</refpurpose>
2531 </refnamediv>
2532 <refsynopsisdiv>
2533 <cmdsynopsis>%s</cmdsynopsis>
2534 </refsynopsisdiv>
2535 <refsect1 id="%s.description" role="desc">
2536 <title role="desc.title">Description</title>
2537 %s
2538 </refsect1>
2539 %s%s%s
2540 </refentry>
2541 ''' % (MakeDocHeader("refentry"), section_id, section_id, program, short_desc, synopsis, section_id, long_desc, options, exit_status, see_also))
2542     OUTPUT.close()
2543
2544     return common.UpdateFileIfChanged(old_db_file, new_db_file, 0)
2545
2546
2547 #############################################################################
2548 # Function    : OutputExtraFile
2549 # Description : Copies an "extra" DocBook file into the output directory,
2550 #               expanding abbreviations
2551 # Arguments   : $file - the source file.
2552 #############################################################################
2553 def OutputExtraFile(file):
2554
2555     basename = re.sub(r'^.*/', '', file)
2556
2557     old_db_file = os.path.join(DB_OUTPUT_DIR, basename)
2558     new_db_file = os.path.join(DB_OUTPUT_DIR, basename + ".new")
2559
2560     contents = open(file).read()
2561
2562     OUTPUT = open(new_db_file, 'w')
2563
2564     OUTPUT.write(ExpandAbbreviations(basename + " file", contents))
2565     OUTPUT.close()
2566
2567     return common.UpdateFileIfChanged(old_db_file, new_db_file, 0)
2568
2569 #############################################################################
2570 # Function    : OutputBook
2571 # Description : Outputs the entities that need to be included into the
2572 #                main docbook file for the module.
2573 # Arguments   : $book_top - the declarations of the entities, which are added
2574 #                  at the top of the main docbook file.
2575 #                $book_bottom - the references to the entities, which are
2576 #                  added in the main docbook file at the desired position.
2577 #############################################################################
2578
2579 def OutputBook(book_top, book_bottom):
2580
2581     old_file = os.path.join(DB_OUTPUT_DIR, MODULE + "-doc.top")
2582     new_file = os.path.join(DB_OUTPUT_DIR, MODULE + "-doc.top.new")
2583
2584     OUTPUT = open(new_file, 'w')
2585     OUTPUT.write(book_top)
2586     OUTPUT.close()
2587
2588     common.UpdateFileIfChanged(old_file, new_file, 0)
2589
2590
2591     old_file = os.path.join(DB_OUTPUT_DIR, MODULE + "-doc.bottom")
2592     new_file = os.path.join(DB_OUTPUT_DIR, MODULE + "-doc.bottom.new")
2593
2594     OUTPUT = open(new_file, 'w')
2595     OUTPUT.write(book_bottom)
2596     OUTPUT.close()
2597
2598     common.UpdateFileIfChanged(old_file, new_file, 0)
2599
2600
2601     # If the main docbook file hasn't been created yet, we create it here.
2602     # The user can tweak it later.
2603     if MAIN_SGML_FILE and not os.path.exists(MAIN_SGML_FILE):
2604         OUTPUT = open(MAIN_SGML_FILE, 'w')
2605
2606         OUTPUT.write('''
2607 <book id="index">
2608   <bookinfo>
2609     <title>&package_name; Reference Manual</title>
2610     <releaseinfo>
2611       for &package_string;.
2612       The latest version of this documentation can be found on-line at
2613       <ulink role="online-location" url="http://[SERVER]/&package_name;/index.html">http://[SERVER]/&package_name;/</ulink>.
2614     </releaseinfo>
2615   </bookinfo>
2616
2617   <chapter>
2618     <title>[Insert title here]</title>
2619     $book_bottom
2620   </chapter>
2621 ''' % MakeDocHeader("book"))
2622         if os.path.exists(OBJECT_TREE_FILE):
2623             OUTPUT.write('''  <chapter id="object-tree">
2624     <title>Object Hierarchy</title>
2625     <xi:include href="xml/tree_index.sgml"/>
2626   </chapter>
2627 ''')
2628         else:
2629             OUTPUT.write('''  <!-- enable this when you use gobject types
2630   <chapter id="object-tree">
2631     <title>Object Hierarchy</title>
2632     <xi:include href="xml/tree_index.sgml"/>
2633   </chapter>
2634   -->
2635 ''')
2636
2637         OUTPUT.write('''  <index id="api-index-full">
2638     <title>API Index</title>
2639     <xi:include href="xml/api-index-full.xml"><xi:fallback /></xi:include>
2640   </index>
2641   <index id="deprecated-api-index" role="deprecated">
2642     <title>Index of deprecated API</title>
2643     <xi:include href="xml/api-index-deprecated.xml"><xi:fallback /></xi:include>
2644   </index>
2645 ''')
2646         if len(AnnotationsUsed) > 0:
2647             OUTPUT.write('''  <xi:include href="xml/annotation-glossary.xml"><xi:fallback /></xi:include>
2648 ''')
2649         else:
2650             OUTPUT.write('''  <!-- enable this when you use gobject introspection annotations
2651   <xi:include href="xml/annotation-glossary.xml"><xi:fallback /></xi:include>
2652   -->
2653 ''')
2654
2655         OUTPUT.write('''
2656 </book>
2657 ''')
2658
2659         OUTPUT.close()
2660
2661 #############################################################################
2662 # Function    : CreateValidSGML
2663 # Description : This turns any chars which are used in SGML into entities,
2664 #                e.g. '<' into '&lt;'
2665 # Arguments   : $text - the text to turn into proper SGML.
2666 #############################################################################
2667
2668 def CreateValidSGML(text):
2669     text = re.sub(r'&', r'&amp;', text)        # Do this first, or the others get messed up.
2670     text = re.sub(r'<', r'&lt;', text)
2671     text = re.sub(r'>', r'&gt;', text)
2672     # browsers render single tabs inconsistently
2673     text = re.sub(r'([^\s])\t([^\s])', r'\1&#160;\2', text)
2674     return text
2675
2676
2677 #############################################################################
2678 # Function    : ConvertSGMLChars
2679 # Description : This is used for text in source code comment blocks, to turn
2680 #               chars which are used in SGML into entities, e.g. '<' into
2681 #               '&lt;'. Depending on $INLINE_MARKUP_MODE, this is done
2682 #               unconditionally or only if the character doesn't seem to be
2683 #               part of an SGML construct (tag or entity reference).
2684 # Arguments   : $text - the text to turn into proper SGML.
2685 #############################################################################
2686
2687 def ConvertSGMLChars(symbol, text):
2688
2689     if INLINE_MARKUP_MODE:
2690         # For the XML/SGML mode only convert to entities outside CDATA sections.
2691         return ModifyXMLElements(text, symbol,
2692                                  "<!\\[CDATA\\[|<programlisting[^>]*>",
2693                                  ConvertSGMLCharsEndTag,
2694                                  ConvertSGMLCharsCallback)
2695     # For the simple non-sgml mode, convert to entities everywhere.
2696
2697     # First, convert freestanding & to &amp
2698     text = re.sub(r'&(?![a-zA-Z#]+;)', r'&amp;', text)
2699     text = re.sub(r'<', r'/&lt', text)
2700     # Allow ">" at beginning of string for blockquote markdown
2701     text = re.sub(r'''(?<=[^\w\n"'\/-])>''', r'&gt;', text)
2702
2703     return text
2704
2705 def ConvertSGMLCharsEndTag(t):
2706     if t == r'<!\[CDATA\[':
2707         return "]]>"
2708     return "</programlisting>"
2709
2710 def ConvertSGMLCharsCallback(text, symbol, tag):
2711     if re.search(r'^<programlisting', tag):
2712         # We can handle <programlisting> specially here.
2713         return ModifyXMLElements(text, symbol,
2714                                  "<!\\[CDATA\\[",
2715                                  ConvertSGMLCharsEndTag,
2716                                  ConvertSGMLCharsCallback2)
2717     elif tag == '':
2718         # If we're not in CDATA convert to entities.
2719         text = re.sub(r'&(?![a-zA-Z#]+;)', r'&amp;', text)        # Do this first, or the others get messed up.
2720         text = re.sub(r'<(?![a-zA-Z\/!])', r'&lt;', text)
2721         # Allow ">" at beginning of string for blockquote markdown
2722         text = re.sub(r'''(?<=[^\w\n"'\/-])>''', r'&gt;', text)
2723
2724         # Handle "#include <xxxxx>"
2725         text = re.sub(r'#include(\s+)<([^>]+)>', r'#include\1&lt;\2&gt;', text)
2726
2727     return text
2728
2729
2730 def ConvertSGMLCharsCallback2(text, symbol, tag):
2731
2732     # If we're not in CDATA convert to entities.
2733     # We could handle <programlisting> differently, though I'm not sure it helps.
2734     if tag == '':
2735         # replace only if its not a tag
2736         text = re.sub(r'&(?![a-zA-Z#]+;)', r'&amp;', text)        # Do this first, or the others get messed up.
2737         text = re.sub(r'<(?![a-zA-Z\/!])', r'&lt;', text)
2738         text = re.sub(r'''(?<![a-zA-Z0-9"'\/-])>''', r'&gt;', text)
2739         # Handle "#include <xxxxx>"
2740         text = re.sub(r'/#include(\s+)<([^>]+)>', r'#include\1&lt;\2&gt;', text)
2741
2742     return text
2743
2744
2745 #############################################################################
2746 # Function    : ExpandAnnotation
2747 # Description : This turns annotations into acronym tags.
2748 # Arguments   : $symbol - the symbol being documented, for error messages.
2749 #                $text - the text to expand.
2750 #############################################################################
2751 def ExpandAnnotation(symbol, param_desc):
2752     param_annotations = ''
2753
2754     # look for annotations at the start of the comment part
2755     # function level annotations don't end with a colon ':'
2756     m = re.search(r'^\s*\((.*?)\)(:|$)', param_desc)
2757     if m:
2758         param_desc = param_desc[m.end():]
2759
2760         annotations = re.split(r'\)\s*\(', m.group(1))
2761         logging.info("annotations for %s: '%s'\n", symbol, m.group(1))
2762         for annotation in annotations:
2763             # need to search for the longest key-match in %AnnotationDefinition
2764             match_length = 0
2765             match_annotation = ''
2766
2767             for annotationdef in AnnotationDefinition:
2768                 if annotation.startswith(annotationdef):
2769                     if len(annotationdef) > match_length:
2770                         match_length = len(annotationdef)
2771                         match_annotation = annotationdef
2772
2773             annotation_extra = ''
2774             if match_annotation != '':
2775                 m = re.search(match_annotation + r'\s+(.*)', annotation)
2776                 if m:
2777                     annotation_extra = " " + m.group(1)
2778
2779                 AnnotationsUsed[match_annotation] = 1
2780                 param_annotations += "[<acronym>%s</acronym>%s]" % (match_annotation, annotation_extra)
2781             else:
2782                 common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
2783                                   "unknown annotation \"%s\" in documentation for %s." % (annotation, symbol))
2784                 param_annotations += "[%s]" % annotation
2785
2786
2787         param_desc = param_desc.strip()
2788         m = re.search(r'^(.*?)\.*\s*$', param_desc)
2789         param_desc = "%s. " % m.group(1)
2790
2791     if param_annotations != '':
2792         param_annotations = "<emphasis role=\"annotation\">%s</emphasis>" % param_annotations
2793
2794     return (param_desc, param_annotations)
2795
2796
2797 #############################################################################
2798 # Function    : ExpandAbbreviations
2799 # Description : This turns the abbreviations function(), macro(), @param,
2800 #                %constant, and #symbol into appropriate DocBook markup.
2801 #               CDATA sections and <programlisting> parts are skipped.
2802 # Arguments   : $symbol - the symbol being documented, for error messages.
2803 #                $text - the text to expand.
2804 #############################################################################
2805
2806 def ExpandAbbreviations(symbol, text):
2807     # Note: This is a fallback and normally done in the markdown parser
2808
2809     # Convert "|[" and "]|" into the start and end of program listing examples.
2810     # Support \[<!-- language="C" --> modifiers
2811     text = re.sub(r'\|\[<!-- language="([^"]+)" -->', r'<informalexample><programlisting language="\1"><![CDATA[', text)
2812     text = re.sub(r'\|\[', r'<informalexample><programlisting><![CDATA[', text)
2813     text = re.sub(r'\]\|', r']]></programlisting></informalexample>', text)
2814
2815     # keep CDATA unmodified, preserve ulink tags (ideally we preseve all tags
2816     # as such)
2817     return ModifyXMLElements(text, symbol,
2818                              "<!\\[CDATA\\[|<ulink[^>]*>|<programlisting[^>]*>|<!DOCTYPE",
2819                              ExpandAbbreviationsEndTag,
2820                              ExpandAbbreviationsCallback)
2821
2822
2823 # Returns the end tag (as a regexp) corresponding to the given start tag.
2824 def ExpandAbbreviationsEndTag(start_tag):
2825     if start_tag == r'<!\[CDATA\[':
2826         return "]]>"
2827     if start_tag == "<!DOCTYPE":
2828         return ">"
2829     m = re.search(r'<(\w+)', start_tag)
2830     return "</%s>" % m.group(1)
2831
2832 # Called inside or outside each CDATA or <programlisting> section.
2833 def ExpandAbbreviationsCallback(text, symbol, tag):
2834
2835     if tag.startswith(r'^<programlisting'):
2836         # Handle any embedded CDATA sections.
2837         return ModifyXMLElements(text, symbol,
2838                                  "<!\\[CDATA\\[",
2839                                  ExpandAbbreviationsEndTag,
2840                                  ExpandAbbreviationsCallback2)
2841     elif tag == '':
2842         # NOTE: this is a fallback. It is normally done by the Markdown parser.
2843
2844         # We are outside any CDATA or <programlisting> sections, so we expand
2845         # any gtk-doc abbreviations.
2846
2847         # Convert '@param()'
2848         # FIXME: we could make those also links ($symbol.$2), but that would be less
2849         # useful as the link target is a few lines up or down
2850         text = re.sub(r'(\A|[^\\])\@(\w+((\.|->)\w+)*)\s*\(\)', r'\1<parameter>\2()', text)
2851
2852         # Convert 'function()' or 'macro()'.
2853         # if there is abc_*_def() we don't want to make a link to _def()
2854         # FIXME: also handle abc(def(....)) : but that would need to be done recursively :/
2855         def f1(m):
2856             return '%s.%s;' % (m.group(1), MakeXRef(m.group(2), tagify(m.group(2) + "()", "function")))
2857         text = re.sub(r'([^\*.\w])(\w+)\s*\(\)', f1, text)
2858         # handle #Object.func()
2859         text = re.sub(r'(\A|[^\\])#([\w\-:\.]+[\w]+)\s*\(\)', f1, text)
2860
2861         # Convert '@param', but not '\@param'.
2862         text = re.sub(r'(\A|[^\\])\@(\w+((\.|->)\w+)*)', r'\1<parameter>\2<\/parameter>', text)
2863         text = re.sub(r'/\\\@', r'\@', text)
2864
2865         # Convert '%constant', but not '\%constant'.
2866         # Also allow negative numbers, e.g. %-1.
2867         def f2(m):
2868             return '%s.%s;' % (m.group(1), MakeXRef(m.group(2), tagify(m.group(2), "literal")))
2869         text = re.sub(r'(\A|[^\\])\%(-?\w+)', f2, text)
2870         text = re.sub(r'\\\%', r'\%', text)
2871
2872         # Convert '#symbol', but not '\#symbol'.
2873         def f3(m):
2874             return '%s.%s;' % (m.group(1), MakeHashXRef(m.group(2), "type"))
2875         text = re.sub(r'(\A|[^\\])#([\w\-:\.]+[\w]+)', f3, text)
2876         text = re.sub(r'\\#', '#', text)
2877
2878
2879     return text
2880
2881
2882 # This is called inside a <programlisting>
2883 def ExpandAbbreviationsCallback2(text, symbol, tag):
2884     if tag == '':
2885         # We are inside a <programlisting> but outside any CDATA sections,
2886         # so we expand any gtk-doc abbreviations.
2887         # FIXME: why is this different from &ExpandAbbreviationsCallback(),
2888         #        why not just call it
2889         text = re.sub(r'#(\w+)', lambda m: '%s;' % MakeHashXRef(m.group(1), ''), text)
2890     elif tag == "<![CDATA[":
2891         # NOTE: this is a fallback. It is normally done by the Markdown parser.
2892         text = ReplaceEntities(text, symbol)
2893
2894
2895     return text
2896
2897
2898 def MakeHashXRef(symbol, tag):
2899     text = symbol
2900
2901     # Check for things like '#include', '#define', and skip them.
2902     if PreProcessorDirectives[symbol]:
2903         return "#%s" % symbol
2904
2905     # Get rid of special suffixes ('-struct','-enum').
2906     text = re.sub(r'-struct$', '', text)
2907     text = re.sub(r'-enum$', '', text)
2908
2909     # If the symbol is in the form "Object::signal", then change the symbol to
2910     # "Object-signal" and use "signal" as the text.
2911     if '::' in symbol:
2912         o, s = symbol.split('::', 1)
2913         symbol = '%s-%s' % (o, s)
2914         text = s
2915
2916     # If the symbol is in the form "Object:property", then change the symbol to
2917     # "Object--property" and use "property" as the text.
2918     if '-' in symbol:
2919         o, p = symbol.split(':', 1)
2920         symbol = '%s--%s' % (o, p)
2921         text = p
2922
2923
2924     if tag != '':
2925         text = tagify(text, tag)
2926
2927
2928     return MakeXRef(symbol, text)
2929
2930
2931
2932 #############################################################################
2933 # Function    : ModifyXMLElements
2934 # Description : Looks for given XML element tags within the text, and calls
2935 #               the callback on pieces of text inside & outside those elements.
2936 #               Used for special handling of text inside things like CDATA
2937 #               and <programlisting>.
2938 # Arguments   : $text - the text.
2939 #               $symbol - the symbol currently being documented (only used for
2940 #                      error messages).
2941 #               $start_tag_regexp - the regular expression to match start tags.
2942 #                      e.g. "<!\\[CDATA\\[|<programlisting[^>]*>" to match
2943 #                      CDATA sections or programlisting elements.
2944 #               $end_tag_func - function which is passed the matched start tag
2945 #                      and should return the appropriate end tag string regexp.
2946 #               $callback - callback called with each part of the text. It is
2947 #                      called with a piece of text, the symbol being
2948 #                      documented, and the matched start tag or '' if the text
2949 #                      is outside the XML elements being matched.
2950 #############################################################################
2951 def ModifyXMLElements(text, symbol, start_tag_regexp, end_tag_func, callback):
2952     before_tag = start_tag = end_tag_regexp = end_tag = None
2953     result = ''
2954
2955     m = re.search(start_tag_regexp, text, flags=re.S)
2956     while m:
2957         before_tag = text[0:m.begin()] # Prematch for last successful match string
2958         start_tag = m.group(0)      # Last successful match
2959         text = text[m.end():]       # Postmatch for last successful match string
2960
2961         result += callback(before_tag, symbol, '')
2962         result += start_tag
2963
2964         # get the matching end-tag for current tag
2965         end_tag_regexp = end_tag_func(start_tag)
2966
2967         m2 = re.search(end_tag_regexp, text, flags=re.S)
2968         if m2:
2969             before_tag = text[0:m2.begin()]
2970             end_tag = m2.group(0)
2971             text = text[m2.end():]
2972
2973             result += callback(before_tag, symbol, start_tag)
2974             result += end_tag
2975         else:
2976             common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
2977                               "Can't find tag end: %s in docs for: %s." % (end_tag_regexp, symbol))
2978             # Just assume it is all inside the tag.
2979             result += callback(text, symbol, start_tag)
2980             text = ''
2981         m = re.search(start_tag_regexp, text, flags=re.S)
2982
2983     # Handle any remaining text outside the tags.
2984     result += callback(text, symbol, '')
2985
2986     return result
2987
2988
2989 def noop(*args):
2990     return args[0]
2991
2992
2993 # Adds a tag around some text.
2994 # e.g tagify("Text", "literal") => "<literal>Text</literal>".
2995 def tagify(text, elem):
2996     return "<" + elem + ">" + +text + "</" + +elem + ">"
2997
2998 #############################################################################
2999 # Function    : MakeDocHeader
3000 # Description : Builds a docbook header for the given tag
3001 # Arguments   : $tag - doctype tag
3002 #############################################################################
3003
3004 def MakeDocHeader(tag):
3005     header = doctype_header
3006     header = re.sub(r'<!DOCTYPE \w+', r'<!DOCTYPE ' + tag, header)
3007
3008     # fix the path for book since this is one level up
3009     if tag == "book":
3010         header = re.sub(r'<!ENTITY % gtkdocentities SYSTEM \"../([a-zA-Z./]+)\">', r'<!ENTITY % gtkdocentities SYSTEM \"\1\">', header)
3011     return header
3012
3013
3014
3015 #############################################################################
3016 # Function    : MakeXRef
3017 # Description : This returns a cross-reference link to the given symbol.
3018 #                Though it doesn't try to do this for a few standard C types
3019 #                that it        knows won't be in the documentation.
3020 # Arguments   : $symbol - the symbol to try to create a XRef to.
3021 #               $text - text text to put inside the XRef, defaults to $symbol
3022 #############################################################################
3023
3024 def MakeXRef(symbol, text=None):
3025     symbol = symbol.strip()
3026
3027     if not text:
3028         text = symbol
3029
3030         # Get rid of special suffixes ('-struct','-enum').
3031         text = re.sub(r'-struct$', '', text)
3032         text = re.sub(r'-enum$', '', text)
3033
3034     if ' ' in symbol:
3035         return text
3036
3037     logging.info("Getting type link for %s -> %s\n", symbol, text)
3038
3039     symbol_id = common.CreateValidSGMLID(symbol)
3040     return "<link linkend=\"%s\">%s</link>" % (symbol_id, text)
3041
3042 #############################################################################
3043 # Function    : MakeIndexterms
3044 # Description : This returns a indexterm elements for the given symbol
3045 # Arguments   : $symbol - the symbol to create indexterms for
3046 #############################################################################
3047
3048 def MakeIndexterms(symbol, sid):
3049     terms = ''
3050     sortas = ''
3051
3052     # make the index useful, by ommiting the namespace when sorting
3053     if NAME_SPACE != '':
3054         m = re.search(r'^$NAME_SPACE\_?(.*)', symbol, flags=re.I)
3055         if m:
3056             sortas = ' sortas=\"%s\"' % m.group(1)
3057     if symbol in Deprecated:
3058         terms += "<indexterm zone=\"%s\" role=\"deprecated\"><primary%s>%s</primary></indexterm>" % (sid, sortas, symbol)
3059         IndexEntriesDeprecated[symbol] = sid
3060         IndexEntriesFull[symbol] = sid
3061     if symbol in Since:
3062         since = Since[symbol].strip()
3063         if since != '':
3064             terms += "<indexterm zone=\"%s\" role=\"%s\"><primary%s>%s</primary></indexterm>" % (sid, since, sortas, symbol)
3065         IndexEntriesSince[symbol] = sid
3066         IndexEntriesFull[symbol] = sid
3067     if terms == '':
3068         terms += "<indexterm zone=\"%s\"><primary%s>%s</primary></indexterm>" % (sid, sortas, symbol)
3069         IndexEntriesFull[symbol] = sid
3070     return terms
3071
3072
3073 #############################################################################
3074 # Function    : MakeDeprecationNote
3075 # Description : This returns a deprecation warning for the given symbol.
3076 # Arguments   : $symbol - the symbol to try to create a warning for.
3077 #############################################################################
3078
3079 def MakeDeprecationNote(symbol):
3080     desc = ''
3081     if symbol in Deprecated:
3082         desc += "<warning><para><literal>%s</literal> " % symbol
3083
3084         note = Deprecated[symbol]
3085
3086         m = re.search(r'^\s*([0-9\.]+)\s*:?', note)
3087         if m:
3088             desc += "has been deprecated since version %s and should not be used in newly-written code.</para>" % m.group(1)
3089         else:
3090             desc += "is deprecated and should not be used in newly-written code.</para>"
3091
3092         note = re.sub(r'^\s*([0-9\.]+)\s*:?\s*', '', note)
3093         note = note.strip()
3094
3095         if note != '':
3096             note = ConvertMarkDown(symbol, note)
3097             desc += " " + note
3098
3099         desc += "</warning>\n"
3100
3101     return desc
3102
3103
3104 #############################################################################
3105 # Function    : MakeConditionDescription
3106 # Description : This returns a sumary of conditions for the given symbol.
3107 # Arguments   : $symbol - the symbol to try to create the sumary.
3108 #############################################################################
3109
3110 def MakeConditionDescription(symbol):
3111     desc = ''
3112     if symbol in Deprecated:
3113         if desc != '':
3114             desc += "|"
3115         m = re.search(r'^\s*(.*?)\s*$', Deprecated[symbol])
3116         if m:
3117             desc += "deprecated:%s" % m.group(1)
3118         else:
3119             desc += "deprecated"
3120
3121     if symbol in Since:
3122         if desc != '':
3123             desc += "|"
3124         m = re.search(r'^\s*(.*?)\s*$', Since[symbol])
3125         if m:
3126             desc += "since:%s" % m.group(1)
3127         else:
3128             desc += "since"
3129
3130
3131
3132     if symbol in StabilityLevel:
3133         if desc != '':
3134             desc += "|"
3135
3136         desc += "stability:" + StabilityLevel[symbol]
3137
3138
3139     if desc != '':
3140         cond = desc
3141         cond = re.sub(r'"', r'&quot', cond)
3142         desc = ' condition=\"%s\"' % cond
3143         logging.info("condition for '%s' = '%s'\n", symbol, desc)
3144
3145     return desc
3146
3147
3148 #############################################################################
3149 # Function    : GetHierarchy
3150 # Description : Returns the DocBook output describing the ancestors and
3151 #               immediate children of a GObject subclass. It uses the
3152 #               global @Objects and @ObjectLevels arrays to walk the tree.
3153 #
3154 # Arguments   : $object - the GtkObject subclass.
3155 #               @hierarchy - previous hierarchy
3156 #############################################################################
3157
3158 def GetHierarchy(gobject, hierarchy_ref):
3159     hierarchy = hierarchy_ref
3160
3161     # Find object in the objects array.
3162     found = False
3163     children = []
3164     level = 0
3165     for i in range(len(Objects)):
3166         if found:
3167             if ObjectLevels[i] <= level:
3168                 continue
3169
3170             elif ObjectLevels[i] == level + 1:
3171                 children.append(Objects[i])
3172
3173         elif Objects[i] == gobject:
3174             found = True
3175             j = i
3176             level = ObjectLevels[i]
3177
3178     if found:
3179         return hierarchy
3180
3181     # Walk up the hierarchy, pushing ancestors onto the ancestors array.
3182     ancestors = [gobject]
3183     logging.info("Level: %s\n", level)
3184     while level > 1:
3185         j -= 1
3186         if ObjectLevels[j] < level:
3187             ancestors.append(Objects[j])
3188             level = ObjectLevels[j]
3189             logging.info("Level: %s", level)
3190
3191     # Output the ancestors, indented and with links.
3192     last_index = 0
3193     level = 1
3194     for i in range(len(ancestors), -1, -1):
3195         ancestor = ancestors[i]
3196         ancestor_id = common.CreateValidSGMLID(ancestor)
3197         indent = ' ' * (level * 4)
3198         # Don't add a link to the current object, i.e. when i == 0.
3199         if i > 0:
3200             entry_text = indent + "<link linkend=\"%s\">%s</link>" % (ancestor_id, ancestor)
3201             alt_text = indent + ancestor
3202         else:
3203             entry_text = indent + ancestor
3204             alt_text = indent + "<link linkend=\"%s\">%s</link>" % (ancestor_id, ancestor)
3205
3206         logging.info("Checking for '%s' or '%s'", entry_text, alt_text)
3207         # Check if we already have this object
3208         index = -1
3209         for j in range(len(hierarchy)):
3210             if hierarchy[j] == entry_text or (hierarchy[j] == alt_text):
3211                 index = j
3212                 break
3213         if index == -1:
3214             # We have a new entry, find insert position in alphabetical order
3215             found = 0
3216             for j in range(last_index, len(hierarchy)):
3217                 if re.search(r'^$%s' % indent, hierarchy[j]):
3218                     last_index = j
3219                     found = True
3220                     break
3221                 elif re.search(r'^%s[^ ]' % indent, hierarchy[j]):
3222                     stripped_text = hierarchy[j]
3223                     if r'<link linkend' not in entry_text:
3224                         stripped_text = re.sub(r'<link linkend="[A-Za-z]*">', '', stripped_text)
3225                         stripped_text = re.sub(r'</link>', '', stripped_text)
3226
3227                     if entry_text < stripped_text:
3228                         last_index = j
3229                         found = True
3230                         break
3231
3232             # Append to bottom
3233             if not found:
3234                 last_index = len(hierarchy)
3235
3236             hierarchy.insert(last_index, entry_text)
3237             last_index += 1
3238         else:
3239             # Already have this one, make sure we use the not linked version
3240             if re.search(r'<link linkend=', entry_text):
3241                 hierarchy[j] = entry_text
3242
3243             # Remember index as base insert point
3244             last_index = index + 1
3245
3246         level += 1
3247
3248     # Output the children, indented and with links.
3249     for i in range(len(children)):
3250         sid = common.CreateValidSGMLID(children[i])
3251         indented_text = ' ' * (level * 4) + "<link linkend=\"%s\">%s</link>" % (sid, children[i])
3252         hierarchy.insert(last_index, indented_text)
3253         last_index += 1
3254     return hierarchy
3255
3256
3257 #############################################################################
3258 # Function    : GetInterfaces
3259 # Description : Returns the DocBook output describing the interfaces
3260 #               implemented by a class. It uses the global %Interfaces hash.
3261 # Arguments   : $object - the GtkObject subclass.
3262 #############################################################################
3263
3264 def GetInterfaces(gobject):
3265     text = ''
3266
3267     # Find object in the objects array.
3268     if gobject in Interfaces:
3269         ifaces = Interfaces[gobject].split()
3270         text = '''<para>
3271 % implements
3272 ''' % gobject
3273         for i in range(len(ifaces)):
3274             sid = common.CreateValidSGMLID(ifaces[i])
3275             text += " <link linkend=\"%s\">%s</link>" % (sid, ifaces[i])
3276             if i < len(ifaces) - 1:
3277                 text += ', '
3278             elif i < len(ifaces):
3279                 text += ' and '
3280             else:
3281                 text += '.'
3282         text += '''</para>
3283 '''
3284     return text
3285
3286
3287 #############################################################################
3288 # Function    : GetImplementations
3289 # Description : Returns the DocBook output describing the implementations
3290 #               of an interface. It uses the global %Interfaces hash.
3291 # Arguments   : $object - the GtkObject subclass.
3292 #############################################################################
3293
3294 def GetImplementations(gobject):
3295     impls = []
3296     text = ''
3297
3298     for key in Interfaces:
3299         if re.search(r'\b%s\b' % gobject, Interfaces[key]):
3300             impls.append(key)
3301
3302     if len(impls) > 0:
3303         impls.sort()
3304         text = ''''<para>
3305 %s is implemented by
3306 ''' % gobject
3307         for i in range(len(impls)):
3308             sid = common.CreateValidSGMLID(impls[i])
3309             text += " <link linkend=\"%s\">%s</link>" % (sid, impls[i])
3310             if i < len(impls) - 1:
3311                 text += ', '
3312
3313             elif i < len(impls):
3314                 text += ' and '
3315
3316             else:
3317                 text += '.'
3318         text += '''</para>
3319 '''
3320     return text
3321
3322
3323
3324 #############################################################################
3325 # Function    : GetPrerequisites
3326 # Description : Returns the DocBook output describing the prerequisites
3327 #               of an interface. It uses the global %Prerequisites hash.
3328 # Arguments   : $iface - the interface.
3329 #############################################################################
3330
3331 def GetPrerequisites(iface):
3332     text = ''
3333
3334     if iface in Prerequisites:
3335         text = '''
3336 <para>
3337 %s requires
3338 ''' % iface
3339         prereqs = Prerequisites[iface].split()
3340         for i in range(len(prereqs)):
3341             sid = common.CreateValidSGMLID(prereqs[i])
3342             text += " <link linkend=\"%s\">%s</link>" % (sid, prereqs[i])
3343             if i < len(prereqs) - 1:
3344                 text += ', '
3345             elif i < len(prereqs):
3346                 text += ' and '
3347             else:
3348                 text += '.'
3349
3350
3351         text += '''</para>
3352 '''
3353     return text
3354
3355
3356 #############################################################################
3357 # Function    : GetDerived
3358 # Description : Returns the DocBook output describing the derived interfaces
3359 #               of an interface. It uses the global %Prerequisites hash.
3360 # Arguments   : $iface - the interface.
3361 #############################################################################
3362
3363 def GetDerived(iface):
3364     text = ''
3365
3366     derived = []
3367     for key in Prerequisites:
3368         if re.search(r'\b%s\b' % iface, Prerequisites[key]):
3369             derived.append(key)
3370     if len(derived) > 0:
3371         derived.sort()
3372         text = '''<para>
3373 %s is required by
3374 ''' % iface
3375         for i in range(len(derived)):
3376             sid = common.CreateValidSGMLID(derived[i])
3377             text += " <link linkend=\"%s\">%s</link>" % (sid, derived[i])
3378             if i < len(derived) - 1:
3379                 text += ', '
3380             elif i < len(derived):
3381                 text += ' and '
3382             else:
3383                 text += '.'
3384         text += '''</para>
3385 '''
3386     return text
3387
3388
3389
3390 #############################################################################
3391 # Function    : GetSignals
3392 # Description : Returns the synopsis and detailed description DocBook output
3393 #                for the signal handlers of a given GtkObject subclass.
3394 # Arguments   : $object - the GtkObject subclass, e.g. 'GtkButton'.
3395 #############################################################################
3396
3397 def GetSignals(gobject):
3398     synop = ''
3399     desc = ''
3400
3401     for i in range(len(SignalObjects)):
3402         if SignalObjects[i] == gobject:
3403             logging.info("Found signal: %s\n", SignalNames[i])
3404             name = SignalNames[i]
3405             symbol = '%s::%s' % (gobject, name)
3406             sid = common.CreateValidSGMLID('%s-%s' % (gobject, name))
3407
3408             desc += "<refsect2 id=\"%s\" role=\"signal\"><title>The <literal>“%s”</literal> signal</title>\n" % (sid, name)
3409             desc += MakeIndexterms(symbol, sid)
3410             desc += "\n"
3411             desc += OutputSymbolExtraLinks(symbol)
3412
3413             desc += "<programlisting language=\"C\">"
3414
3415             m = re.search(r'\s*(const\s+)?(\w+)\s*(\**)', SignalReturns[i])
3416             type_modifier = m.group(1) if m.group(1) else ''
3417             gtype = m.group(2)
3418             pointer = m.group(3)
3419             xref = MakeXRef(gtype, tagify(gtype, "returnvalue"))
3420
3421             ret_type_output = '%s%s%s' % (type_modifier, xref, pointer)
3422             callback_name = "user_function"
3423             desc += '%s\n%s (' % (ret_type_output, callback_name)
3424
3425             indentation = ' ' * len(callback_name) + 2
3426             pad = indentation
3427
3428             sourceparams = SourceSymbolParams[symbol]
3429             params = SignalPrototypes[i].split('\n')
3430             type_len = len("gpointer")
3431             name_len = len("user_data")
3432             # do two passes, the first one is to calculate padding
3433             for l in range(2):
3434                 for j in range(len(params)):
3435                     param_name = None
3436                     # allow alphanumerics, '_', '[' & ']' in param names
3437                     m = re.search(r'^\s*(\w+)\s*(\**)\s*([\w\[\]]+)\s*$', params[j])
3438                     if m:
3439                         gtype = m.group(1)
3440                         pointer = m.group(2)
3441                         if sourceparams:
3442                             param_name = sourceparams[PARAM_FIELD_COUNT * j]
3443
3444                         else:
3445                             param_name = m.group(3)
3446
3447                         if not param_name:
3448                             param_name = "arg%d" % j
3449
3450                         if l == 0:
3451                             if len(gtype) + len(pointer) > type_len:
3452                                 type_len = len(gtype) + len(pointer)
3453                             if len(param_name) > name_len:
3454                                 name_len = len(param_name)
3455                         else:
3456                             xref = MakeXRef(gtype, tagify(gtype, "type"))
3457                             pad = ' ' * (type_len - len(type) - len(pointer))
3458                             desc += '%s%s %s%s,\n' % (xref, pad, pointer, param_name)
3459                             desc += indentation
3460
3461                     else:
3462                         common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
3463                                           "Can't parse arg: %s\nArgs:%s" % (params[j], SignalPrototypes[i]))
3464
3465             xref = MakeXRef("gpointer", tagify("gpointer", "type"))
3466             pad = ' ' * (type_len - len("gpointer"))
3467             desc += '%s%s user data)' % (xref, pad)
3468             desc += "</programlisting>\n"
3469
3470             flags = SignalFlags[i]
3471             flags_string = ''
3472
3473             if flags:
3474                 if 'f' in flags:
3475                     flags_string = "<link linkend=\"G-SIGNAL-RUN-FIRST:CAPS\">Run First</link>"
3476
3477                 elif 'l' in flags:
3478                     flags_string = "<link linkend=\"G-SIGNAL-RUN-LAST:CAPS\">Run Last</link>"
3479
3480                 elif 'c' in flags:
3481                     flags_string = "<link linkend=\"G-SIGNAL-RUN-CLEANUP:CAPS\">Cleanup</link>"
3482                     flags_string = "Cleanup"
3483
3484                 if 'r' in flags:
3485                     if flags_string:
3486                         flags_string += " / "
3487                     flags_string = "<link linkend=\"G-SIGNAL-NO-RECURSE:CAPS\">No Recursion</link>"
3488
3489                 if 'd' in flags:
3490                     if flags_string:
3491                         flags_string += " / "
3492                     flags_string = "<link linkend=\"G-SIGNAL-DETAILED:CAPS\">Has Details</link>"
3493
3494                 if 'a' in flags:
3495                     if flags_string:
3496                         flags_string += " / "
3497                     flags_string = "<link linkend=\"G-SIGNAL-ACTION:CAPS\">Action</link>"
3498
3499                 if 'h' in flags:
3500                     if flags_string:
3501                         flags_string += " / "
3502                     flags_string = "<link linkend=\"G-SIGNAL-NO-HOOKS:CAPS\">No Hooks</link>"
3503
3504             synop += "<row><entry role=\"signal_type\">%s</entry><entry role=\"signal_name\"><link linkend=\"%s\">%s</link></entry><entry role=\"signal_flags\">%s</entry></row>\n" % (ret_type_output, sid, name, flags_string)
3505
3506             parameters = OutputParamDescriptions("SIGNAL", symbol)
3507
3508             AllSymbols[symbol] = 1
3509             if SymbolDocs[symbol]:
3510                 symbol_docs = ConvertMarkDown(symbol, SymbolDocs[symbol])
3511
3512                 desc += symbol_docs
3513
3514                 if not IsEmptyDoc(SymbolDocs[symbol]):
3515                     AllDocumentedSymbols[symbol] = 1
3516
3517             if SymbolAnnotations[symbol]:
3518                 param_desc = SymbolAnnotations[symbol]
3519                 param_annotations = ''
3520                 (param_desc, param_annotations) = ExpandAnnotation(symbol, param_desc)
3521                 if param_annotations != '':
3522                     desc += "\n<para>%s</para>" % param_annotations
3523
3524             desc += MakeDeprecationNote(symbol)
3525
3526             desc += parameters
3527             if flags_string:
3528                 desc += "<para>Flags: %s</para>\n" % flags_string
3529
3530             desc += OutputSymbolTraits(symbol)
3531             desc += "</refsect2>"
3532
3533     return (synop, desc)
3534
3535 #############################################################################
3536 # Function    : GetArgs
3537 # Description : Returns the synopsis and detailed description DocBook output
3538 #                for the Args of a given GtkObject subclass.
3539 # Arguments   : $object - the GtkObject subclass, e.g. 'GtkButton'.
3540 #############################################################################
3541
3542 def GetArgs(gobject):
3543     synop = ''
3544     desc = ''
3545     child_synop = ''
3546     child_desc = ''
3547     style_synop = ''
3548     style_desc = ''
3549
3550     for i in range(len(ArgObjects)):
3551         if ArgObjects[i] == gobject:
3552             logging.info("Found arg: %s", ArgNames[i])
3553             name = ArgNames[i]
3554             flags = ArgFlags[i]
3555             flags_string = ''
3556             kind = ''
3557             id_sep = ''
3558
3559             if 'c' in flags:
3560                 kind = "child property"
3561                 id_sep = "c-"
3562             elif 's' in flags:
3563                 kind = "style property"
3564                 id_sep = "s-"
3565             else:
3566                 kind = "property"
3567
3568
3569             # Remember only one colon so we don't clash with signals.
3570             symbol = '%s:%s' % (gobject, name)
3571             # use two dashes and ev. an extra separator here for the same reason.
3572             sid = common.CreateValidSGMLID('%s--%s%s' % (gobject, id_sep, name))
3573
3574             atype = ArgTypes[i]
3575             type_output = None
3576             arange = ArgRanges[i]
3577             range_output = CreateValidSGML(arange)
3578             default = ArgDefaults[i]
3579             default_output = CreateValidSGML(default)
3580
3581             if type == "GtkString":
3582                 type = "char&#160;*"
3583
3584             if type == "GtkSignal":
3585                 type = "GtkSignalFunc, gpointer"
3586                 type_output = MakeXRef("GtkSignalFunc") + ", " + MakeXRef("gpointer")
3587             elif re.search(r'^(\w+)\*$', type):
3588                 m = re.search(r'^(\w+)\*$', type)
3589                 type_output = MakeXRef(m.group(1), tagify(m.group(1), "type")) + "&#160;*"
3590             else:
3591                 type_output = MakeXRef(atype, tagify(atype, "type"))
3592
3593             if 'r' in flags:
3594                 flags_string = "Read"
3595
3596             if 'w' in flags:
3597                 if flags_string:
3598                     flags_string += " / "
3599                 flags_string += "Write"
3600
3601             if 'x' in flags:
3602                 if flags_string:
3603                     flags_string += " / "
3604                 flags_string += "Construct"
3605
3606             if 'X' in flags:
3607                 if flags_string:
3608                     flags_string += " / "
3609                 flags_string += "Construct Only"
3610
3611
3612             AllSymbols[symbol] = 1
3613             blurb = ''
3614             if symbol in SymbolDocs and not IsEmptyDoc(SymbolDocs[symbol]):
3615                 blurb = ConvertMarkDown(symbol, SymbolDocs[symbol])
3616                 logging.info(".. [%s][%s]\n", SymbolDocs[symbol], blurb)
3617                 AllDocumentedSymbols[symbol] = 1
3618
3619             else:
3620                 if ArgBlurbs[i] != '':
3621                     blurb = "<para>" + CreateValidSGML(ArgBlurbs[i]) + "</para>"
3622                     AllDocumentedSymbols[symbol] = 1
3623                 else:
3624                     # FIXME: print a warning?
3625                     logging.info(".. no description\n")
3626
3627             pad1 = ''
3628             if len(name) < 24:
3629                 pad1 = " " * (24 - len(name))
3630
3631
3632             arg_synop = "<row><entry role=\"property_type\">%s</entry><entry role=\"property_name\"><link linkend=\"%s\">%s</link></entry><entry role=\"property_flags\">%s</entry></row>\n" % (type_output, sid, name, flags_string)
3633             arg_desc = "<refsect2 id=\"%s\" role=\"property\"><title>The <literal>“%s”</literal> %s</title>\n" % (sid, name, kind)
3634             arg_desc += MakeIndexterms(symbol, sid)
3635             arg_desc += "\n"
3636             arg_desc += OutputSymbolExtraLinks(symbol)
3637
3638             arg_desc += "<programlisting>  “%s”%s %s</programlisting>\n" % (name, pad1, type_output)
3639             arg_desc += blurb
3640             if symbol in SymbolAnnotations:
3641                 param_desc = SymbolAnnotations[symbol]
3642                 param_annotations = ''
3643                 (param_desc, param_annotations) = ExpandAnnotation(symbol, param_desc)
3644                 if param_annotations != '':
3645                     arg_desc += "\n<para>%s</para>" % param_annotations
3646
3647             arg_desc += MakeDeprecationNote(symbol)
3648
3649             if flags_string:
3650                 arg_desc += "<para>Flags: %s</para>\n" % flags_string
3651
3652             if range != '':
3653                 arg_desc += "<para>Allowed values: %s</para>\n" % range_output
3654
3655             if default != '':
3656                 arg_desc += "<para>Default value: %s</para>\n" % default_output
3657
3658             arg_desc += OutputSymbolTraits(symbol)
3659             arg_desc += "</refsect2>\n"
3660
3661             if 'c' in flags:
3662                 child_synop += arg_synop
3663                 child_desc += arg_desc
3664
3665             elif 's' in flags:
3666                 style_synop += arg_synop
3667                 style_desc += arg_desc
3668
3669             else:
3670                 synop += arg_synop
3671                 desc += arg_desc
3672
3673     return (synop, child_synop, style_synop, desc, child_desc, style_desc)
3674
3675
3676
3677 #############################################################################
3678 # Function    : ReadSourceDocumentation
3679 # Description : This reads in the documentation embedded in comment blocks
3680 #                in the source code (for Gnome).
3681 #
3682 #                Parameter descriptions override any in the template files.
3683 #                Function descriptions are placed before any description from
3684 #                the template files.
3685 #
3686 #                It recursively descends the source directory looking for .c
3687 #                files and scans them looking for specially-formatted comment
3688 #                blocks.
3689 #
3690 # Arguments   : $source_dir - the directory to scan.
3691 #############m###############################################################
3692
3693 def ReadSourceDocumentation(source_dir):
3694
3695     # prepend entries from @SOURCE_DIR
3696     for sdir in SOURCE_DIRS:
3697         # Check if the filename is in the ignore list.
3698         m1 = re.search(r'^%s/(.*)$' % re.escape(sdir), source_dir)
3699         if m1:
3700             m2 = re.search(r'(\s|^)%s(\s|$)' % re.escape(m1.group(1)), IGNORE_FILES)
3701             if m2:
3702                 logging.info("Skipping source directory: %s", source_dir)
3703                 return
3704         else:
3705             logging.info("No match for: %s", (m2.group(1) or source_dir))
3706
3707     logging.info("Scanning source directory: %s", source_dir)
3708
3709     # This array holds any subdirectories found.
3710     subdirs = []
3711
3712     suffix_list = SOURCE_SUFFIXES.split(',')
3713
3714     for ifile in os.listdir(source_dir):
3715         fname = os.path.join(source_dir, ifile)
3716         if ifile.startswith('.'):
3717             continue
3718         elif os.path.isdir(fname):
3719             subdirs.append(ifile)
3720         elif suffix_list:
3721             for suffix in suffix_list:
3722                 if re.search(r'\.%s$' % re.escape(suffix), ifile):
3723                     ScanSourceFile(fname)
3724         elif re.search(r'\.[ch]$', ifile):
3725             ScanSourceFile(fname)
3726
3727     # Now recursively scan the subdirectories.
3728     for sdir in subdirs:
3729         ReadSourceDocumentation(os.path.join(source_dir, sdir))
3730
3731 #############################################################################
3732 # Function    : ScanSourceFile
3733 # Description : Scans one source file looking for specially-formatted comment
3734 #                blocks. Later &MergeSourceDocumentation is used to merge any
3735 #                documentation found with the documentation already read in
3736 #                from the template files.
3737 #
3738 # Arguments   : $file - the file to scan.
3739 #############################################################################
3740
3741 def ScanSourceFile(ifile):
3742
3743     # prepend entries from @SOURCE_DIR
3744     for idir in SOURCE_DIRS:
3745         # Check if the filename is in the ignore list.
3746         m1 = re.search(r'^%s/(.*)$' % re.escape(idir), ifile)
3747         if m1:
3748             m2 = re.search(r'(\s|^)%s(\s|$)' % re.escape(m1.group(1)), IGNORE_FILES)
3749             if m2:
3750                 logging.info("Skipping source file: ", ifile)
3751                 return
3752
3753     m = re.search(r'^.*[\/\\]([^\/\\]*)$', ifile)
3754     if m:
3755         basename = m.group(1)
3756     else:
3757         common.LogWarning(ifile, 1, "Can't find basename for this filename.")
3758         basename = ifile
3759
3760
3761     # Check if the basename is in the list of files to ignore.
3762     if re.search(r'(\s|^)%s(\s|$)' % re.escape(basename), IGNORE_FILES):
3763         logging.info("Skipping source file: %s", ifile)
3764         return
3765
3766
3767     logging.info("Scanning source file: %s", ifile)
3768
3769     SRCFILE = open(ifile)
3770     in_comment_block = False
3771     symbol = None
3772     in_part = ''
3773     description = ''
3774     return_desc = ''
3775     since_desc = stability_desc = deprecated_desc = ''
3776     current_param = None
3777     params = []
3778     line_number = 0
3779     for line in SRCFILE:
3780         line_number += 1
3781         # Look for the start of a comment block.
3782         if not in_comment_block:
3783             if re.search(r'^\s*/\*.*\*/', line):
3784                 #one-line comment - not gtkdoc
3785                 pass
3786             elif re.search(r'^\s*/\*\*\s', line):
3787                 logging.info("Found comment block start\n")
3788
3789                 in_comment_block = True
3790
3791                 # Reset all the symbol data.
3792                 symbol = ''
3793                 in_part = ''
3794                 description = ''
3795                 return_desc = ''
3796                 since_desc = ''
3797                 deprecated_desc = ''
3798                 stability_desc = ''
3799                 current_param = -1
3800                 params = []
3801
3802             continue
3803
3804         # We're in a comment block. Check if we've found the end of it.
3805         if re.search(r'^\s*\*+/', line):
3806             if symbol:
3807                 # maybe its not even meant to be a gtk-doc comment?
3808                 common.LogWarning(ifile, line_number, "Symbol name not found at the start of the comment block.")
3809             else:
3810                 # Add the return value description onto the end of the params.
3811                 if return_desc:
3812                     # TODO(ensonic): check for duplicated Return docs
3813                     # &common.LogWarning($file, line_number, "Multiple Returns for $symbol.")
3814                     params.append("Returns")
3815                     params.append(return_desc)
3816
3817                 # Convert special characters
3818                 description = ConvertSGMLChars(symbol, description)
3819                 for k in range(1, len(params), PARAM_FIELD_COUNT):
3820                     params[k] = ConvertSGMLChars(symbol, params[k])
3821
3822
3823                 # Handle Section docs
3824                 m = re.search(r'SECTION:\s*(.*)', symbol)
3825                 m2 = re.search(r'PROGRAM:\s*(.*)', symbol)
3826                 if m:
3827                     real_symbol = m.group(1)
3828                     long_descr = os.path.join(TMPL_DIR, real_symbol + ":Long_Description")
3829
3830                     if len(KnownSymbols) > 0:
3831                         if long_descr in KnownSymbols or KnownSymbols[long_descr] != 1:
3832                             common.LogWarning(ifile, line_number, "Section %s is not defined in the %s-sections.txt file." % (real_symbol, MODULE))
3833
3834                     logging.info("SECTION DOCS found in source for : '%s'", real_symbol)
3835                     for k in range(0, len(params), PARAM_FIELD_COUNT):
3836                         logging.info("   '" + params[k] + "'\n")
3837                         params[k] = params[k].lower()
3838                         key = None
3839                         if params[k] == "short_description":
3840                             key = os.path.join(TMPL_DIR, real_symbol + ":Short_Description")
3841                         elif params[k] == "see_also":
3842                             key = os.path.join(TMPL_DIR, real_symbol + ":See_Also")
3843                         elif params[k] == "title":
3844                             key = os.path.join(TMPL_DIR, real_symbol + ":Title")
3845                         elif params[k] == "stability":
3846                             key = os.path.join(TMPL_DIR, real_symbol + ":Stability_Level")
3847                         elif params[k] == "section_id":
3848                             key = os.path.join(TMPL_DIR, real_symbol + ":Section_Id")
3849                         elif params[k] == "include":
3850                             key = os.path.join(TMPL_DIR, real_symbol + ":Include")
3851                         elif params[k] == "image":
3852                             key = os.path.join(TMPL_DIR, real_symbol + ":Image")
3853
3854                         if key:
3855                             SourceSymbolDocs[key] = params[k+1]
3856                             SourceSymbolSourceFile[key] = ifile
3857                             SourceSymbolSourceLine[key] = line_number
3858
3859
3860                     SourceSymbolDocs[long_descr] = description
3861                     SourceSymbolSourceFile[long_descr] = ifile
3862                     SourceSymbolSourceLine[long_descr] = line_number
3863                     #$SourceSymbolTypes{$symbol} = "SECTION"
3864                 elif m2:
3865                     real_symbol = m2.group(1)
3866                     key = None
3867                     section_id = None
3868
3869                     logging.info("PROGRAM DOCS found in source for '%s'", real_symbol)
3870                     for k in range(0, len(params), PARAM_FIELD_COUNT):
3871                         logging.info("   '" + params[k] + "'\n")
3872                         params[k] = params[k].lower()
3873                         key = None
3874
3875                         if params[k] == "short_description":
3876                             key = os.path.join(TMPL_DIR, real_symbol, ":Short_Description")
3877                         elif params[k] == "see_also":
3878                             key = os.path.join(TMPL_DIR, real_symbol + ":See_Also")
3879                         elif params[k] == "section_id":
3880                             key = os.path.join(TMPL_DIR, real_symbol + ":Section_Id")
3881                         elif params[k] == "synopsis":
3882                             key = os.path.join(TMPL_DIR, real_symbol + ":Synopsis")
3883                         elif params[k] == "returns":
3884                             key = os.path.join(TMPL_DIR, real_symbol + ":Returns")
3885                         elif re.search(r'^(-.*)', params[k]):
3886                             m4 = re.search(r'^(-.*)', params[k])
3887                             key = os.path.join(TMPL_DIR, real_symbol + ":Options")
3888                             if key in SourceSymbolDocs:
3889                                 opts = SourceSymbolDocs[key]
3890                             else:
3891                                 opts = []
3892
3893                             opts.append(m4.group(1))
3894                             opts.append(params[k+1])
3895
3896                             SourceSymbolDocs[key] = opts
3897                             continue
3898
3899                         if key:
3900                             SourceSymbolDocs[key] = params[k+1]
3901                             SourceSymbolSourceFile[key] = ifile
3902                             SourceSymbolSourceLine[key] = line_number
3903
3904
3905                     long_descr = os.path.join(TMPL_DIR, real_symbol + ":Long_Description")
3906                     SourceSymbolDocs[long_descr] = description
3907                     SourceSymbolSourceFile[long_descr] = ifile
3908                     SourceSymbolSourceLine[long_descr] = line_number
3909
3910                     section_id = SourceSymbolDocs[os.path.join(TMPL_DIR, real_symbol + ":Section_Id")]
3911                     if section_id and not re.search(r'^\s*$', section_id):
3912                         # Remove trailing blanks and use as is
3913                         section_id = section_id.rstrip()
3914                     else:
3915                         section_id = common.CreateValidSGMLID('%s-%s' % (MODULE, real_symbol))
3916                     OutputProgramDBFile(real_symbol, section_id)
3917
3918                 else:
3919                     logging.info("SYMBOL DOCS found in source for : '%s' %d ", symbol, len(description))
3920                     SourceSymbolDocs[symbol] = description
3921                     SourceSymbolParams[symbol] = [params]
3922                     # FIXME $SourceSymbolTypes{$symbol} = "STRUCT,SIGNAL,ARG,FUNCTION,MACRO"
3923                     #if (defined $DeclarationTypes{$symbol})
3924                     #    $SourceSymbolTypes{$symbol} = $DeclarationTypes{$symbol
3925                     #
3926                     SourceSymbolSourceFile[symbol] = ifile
3927                     SourceSymbolSourceLine[symbol] = line_number
3928
3929
3930                 if since_desc:
3931                     arr = since_desc.split('\n')
3932                     since_desc = arr[0].strip()
3933                     extra_lines = arr[1:]
3934                     logging.info("Since(%s) : [%s]\n", symbol, since_desc)
3935                     Since[symbol] = ConvertSGMLChars(symbol, since_desc)
3936                     if len(extra_lines) > 0:
3937                         common.LogWarning(ifile, line_number, "multi-line since docs found")
3938
3939                 if stability_desc:
3940                     stability_desc = ParseStabilityLevel(stability_desc, ifile, line_number, "Stability level for %s" % symbol)
3941                     StabilityLevel[symbol] = ConvertSGMLChars(symbol, stability_desc)
3942
3943
3944                 if deprecated_desc:
3945                     if symbol in Deprecated:
3946                         # don't warn for signals and properties
3947                         #if ($symbol !~ m/::?(.*)/)
3948                         if symbol in DeclarationTypes:
3949                             common.LogWarning(ifile, line_number,
3950                                               "%s is deprecated in the inline comments, but no deprecation guards were found around the declaration. (See the --deprecated-guards option for gtkdoc-scan.)" % symbol)
3951
3952
3953                     Deprecated[symbol] = ConvertSGMLChars(symbol, deprecated_desc)
3954
3955
3956
3957             in_comment_block = False
3958             continue
3959
3960         # Get rid of ' * ' at start of every line in the comment block.
3961         line = re.sub(r'^\s*\*\s?', '', line)
3962         # But make sure we don't get rid of the newline at the end.
3963         if not line.endswith('\n'):
3964             line = line + "\n"
3965
3966         logging.info("scanning : %s", line)
3967
3968         # If we haven't found the symbol name yet, look for it.
3969         if symbol:
3970             m1 = re.search(r'^\s*(SECTION:\s*\S+)', line)
3971             m2 = re.search(r'^\s*(PROGRAM:\s*\S+)', line)
3972             m3 = re.search(r'^\s*([\w:-]*\w)\s*:?\s*(\(.+?\)\s*)*$', line)
3973             if m1:
3974                 symbol = m1.group(1)
3975                 logging.info("SECTION DOCS found in source for : '%s'", symbol)
3976             elif m2:
3977                 symbol = m2.group(1)
3978                 logging.info("PROGRAM DOCS found in source for : '%s'", symbol)
3979             elif m3:
3980                 symbol = m3.group(1)
3981                 annotation = m3.group(2)
3982                 logging.info("SYMBOL DOCS found in source for : '%s'\n", symbol)
3983                 if annotation:
3984                     annotation = annotation.strip()
3985                     if annotation != '':
3986                         SymbolAnnotations[symbol] = annotation
3987                         logging.info("remaining text for $symbol: '%s'\n", annotation)
3988
3989             continue
3990
3991
3992         if in_part == "description":
3993             # Get rid of 'Description:'
3994             line = re.sub(r'^\s*Description:', '', line)
3995
3996
3997         m = re.search(r'^\s*(returns|return\s+value):', line, flags=re.I)
3998         m2 = re.search(r'^\s*since:', line, flags=re.I)
3999         m3 = re.search(r'^\s*deprecated:', line, flags=re.I)
4000         m4 = re.search(r'^\s*stability:', line, flags=re.I)
4001
4002         if m:
4003             # we're in param section and have not seen the blank line
4004             if in_part != '':
4005                 return_desc = line[m.end():]
4006                 in_part = "return"
4007                 continue
4008
4009         if m2:
4010             # we're in param section and have not seen the blank line
4011             if in_part != "param":
4012                 since_desc = line[m2.end():]
4013                 in_part = "since"
4014                 continue
4015
4016         elif m3:
4017             # we're in param section and have not seen the blank line
4018             if in_part != "param":
4019                 deprecated_desc = line[m.end():]
4020                 in_part = "deprecated"
4021                 continue
4022
4023         elif m4:
4024             stability_desc = line[m.end():]
4025             in_part = "stability"
4026             continue
4027
4028
4029         if in_part == "description":
4030             description += line
4031             continue
4032         elif in_part == "return":
4033             return_desc += line
4034             continue
4035         elif in_part == "since":
4036             since_desc += line
4037             continue
4038         elif in_part == "stability":
4039             stability_desc += line
4040             continue
4041         elif in_part == "deprecated":
4042             deprecated_desc += line
4043             continue
4044
4045
4046         # We must be in the parameters. Check for the empty line below them.
4047         if re.search(r'^\s*$', line):
4048             in_part = "description"
4049             continue
4050
4051
4052         # Look for a parameter name.
4053         m = re.search(r'^\s*@(.+?)\s*:\s*', line)
4054         if m:
4055             param_name = m.group(1)
4056             param_desc = line[m.end():]
4057
4058             logging.info("Found parameter: %s", param_name)
4059             # Allow varargs variations
4060             if re.search(r'^\.\.\.$', param_name):
4061                 param_name = "..."
4062
4063             logging.info("Found param for symbol $symbol : '%s'= '%s'", param_name, line)
4064
4065             params.append(param_name)
4066             params.append(param_desc)
4067             current_param += PARAM_FIELD_COUNT
4068             in_part = "param"
4069             continue
4070         elif in_part == '':
4071             logging.info("continuation for %s annotation '%s'", symbol, line)
4072             annotation = line
4073             annotation = re.sub(r'^\s+|\s+$', '', annotation)
4074             SymbolAnnotations[symbol] += annotation
4075             continue
4076
4077
4078         # We must be in the middle of a parameter description, so add it on
4079         # to the last element in @params.
4080         if current_param == -1:
4081             common.LogWarning(file, line_number, "Parsing comment block file : parameter expected, but got '%s'" % line)
4082         else:
4083             params[len(params)] += line
4084
4085
4086     SRCFILE.close()
4087
4088
4089 #############################################################################
4090 # Function    : OutputMissingDocumentation
4091 # Description : Outputs report of documentation coverage to a file
4092 #
4093 # Arguments   : none
4094 #############################################################################
4095
4096 def OutputMissingDocumentation():
4097     old_undocumented_file = os.path.join(ROOT_DIR, MODULE + "undocumented.txt")
4098     new_undocumented_file = os.path.join(ROOT_DIR, MODULE + "-undocumented.new")
4099
4100     n_documented = 0
4101     n_incomplete = 0
4102     total = 0
4103     symbol = None
4104     percent = None
4105     buffer = ''
4106     buffer_deprecated = ''
4107     buffer_descriptions = ''
4108
4109     UNDOCUMENTED = open(new_undocumented_file, 'w')
4110
4111     for symbol in sorted(AllSymbols.keys()):
4112         # FIXME: should we print common.LogWarnings for undocumented stuff?
4113         # DEBUG
4114         #my $ssfile = &GetSymbolSourceFile($symbol)
4115         #my $ssline = &GetSymbolSourceLine($symbol)
4116         #my $location = "defined at " . (defined($ssfile)?$ssfile:"?") . ":" . (defined($ssline)?$ssline:"0") . "\n"
4117         # DEBUG
4118         m = re.search(r':(Title|Long_Description|Short_Description|See_Also|Stability_Level|Include|Section_Id|Image)', symbol)
4119         m2 = re.search(r':(Long_Description|Short_Description)', symbol)
4120         if not m:
4121             total += 1
4122             if symbol in AllDocumentedSymbols:
4123                 n_documented += 1
4124                 if symbol in AllIncompleteSymbols:
4125                     n_incomplete += 1
4126                     buffer += symbol + " (" + AllIncompleteSymbols[symbol] + ")\n"
4127                     #$buffer += "\t0: ".$location
4128
4129             elif symbol in Deprecated:
4130                 if symbol in AllIncompleteSymbols:
4131                     n_incomplete += 1
4132                     buffer_deprecated += symbol + " (" + AllIncompleteSymbols[symbol] + ")\n"
4133                     #$buffer += "\t1a: ".$location
4134                 else:
4135                     buffer_deprecated += symbol + "\n"
4136                     #$buffer += "\t1b: ".$location
4137
4138             else:
4139                 if symbol in AllIncompleteSymbols:
4140                     n_incomplete += 1
4141                     buffer += symbol + " (" + AllIncompleteSymbols[symbol] + ")\n"
4142                     #$buffer += "\t2a: ".$location
4143                 else:
4144                     buffer += symbol + "\n"
4145                     #$buffer += "\t2b: ".$location
4146
4147
4148         elif m2:
4149             total += 1
4150             if symbol in SymbolDocs and len(SymbolDocs[symbol]) > 0\
4151                or symbol in AllDocumentedSymbols and len(AllDocumentedSymbols[symbol]) > 0:
4152                 n_documented += 1
4153             else:
4154                 # cut off the leading namespace ($TMPL_DIR)
4155                 m = re.search(r'^.*\/(.*)$', symbol)
4156                 buffer_descriptions += m.group(1) + "\n"
4157
4158     if total == 0:
4159         percent = 100
4160     else:
4161         percent = (n_documented / total) * 100.0
4162
4163
4164     UNDOCUMENTED.write("%.0f%% symbol docs coverage.\n"%percent)
4165     UNDOCUMENTED.write("%s symbols documented.\n" % n_documented)
4166     UNDOCUMENTED.write("%s symbols incomplete.\n" % n_incomplete)
4167     UNDOCUMENTED.write("%d not documented.\n" % (total - n_documented))
4168
4169     if buffer_deprecated != '':
4170         buffer += "\n" + buffer_deprecated
4171
4172     if buffer_descriptions != '':
4173         buffer += "\n" + buffer_descriptions
4174
4175     if buffer != '':
4176         UNDOCUMENTED.write("\n\n" + buffer)
4177
4178     UNDOCUMENTED.close()
4179
4180     return common.UpdateFileIfChanged(old_undocumented_file, new_undocumented_file, 0)
4181
4182 #    printf "%.0f%% symbol docs coverage", $percent
4183 #    print "($n_documented symbols documented, $n_incomplete symbols incomplete, " . ($total - $n_documented) . " not documented)\n"
4184 #    print "See $MODULE-undocumented.txt for a list of missing docs.\nThe doc coverage percentage doesn't include intro sections.\n"
4185
4186
4187
4188 #############################################################################
4189 # Function    : OutputUndeclaredSymbols
4190 # Description : Outputs symbols that are listed in the section file, but not
4191 #               declaration is found in the sources
4192 #
4193 # Arguments   : none
4194 #############################################################################
4195
4196 def OutputUndeclaredSymbols():
4197     old_undeclared_file = os.path.join(ROOT_DIR, MODULE + "-undeclared.txt")
4198     new_undeclared_file = os.path.join(ROOT_DIR, MODULE + "-undeclared.new")
4199
4200     UNDECLARED = open(new_undeclared_file, 'w')
4201
4202     if UndeclaredSymbols:
4203         UNDECLARED.write("\n".join(sorted(UndeclaredSymbols.keys())))
4204         UNDECLARED.write("\n")
4205         print("See %s-undeclared.txt for the list of undeclared symbols." % MODULE)
4206
4207     UNDECLARED.close()
4208
4209     return common.UpdateFileIfChanged(old_undeclared_file, new_undeclared_file, 0)
4210
4211
4212 #############################################################################
4213 # Function    : OutputUnusedSymbols
4214 # Description : Outputs symbols that are documented in comments, but not
4215 #               declared in the sources
4216 #
4217 # Arguments   : none
4218 #############################################################################
4219
4220 def OutputUnusedSymbols():
4221     num_unused = 0
4222     old_unused_file = os.path.join(ROOT_DIR, MODULE + "-unused.txt")
4223     new_unused_file = os.path.join(ROOT_DIR, MODULE + "-unused.new")
4224
4225     UNUSED = open(new_unused_file, 'w')
4226
4227     for symbol in sorted(Declarations.keys()):
4228         if not symbol in DeclarationOutput:
4229             UNUSED.write("%s\n" % symbol)
4230             num_unused += 1
4231
4232
4233     for symbol in sorted(AllUnusedSymbols.keys()):
4234         UNUSED.write(symbol + "(" + AllUnusedSymbols[symbol] + ")\n")
4235         num_unused += 1
4236
4237     UNUSED.close()
4238     if num_unused != 0:
4239         common.LogWarning(old_unused_file, 1, "%d unused declarations. They should be added to %s-sections.txt in the appropriate place." % (num_unused, MODULE))
4240
4241
4242     return common.UpdateFileIfChanged(old_unused_file, new_unused_file, 0)
4243
4244
4245
4246 #############################################################################
4247 # Function    : OutputAllSymbols
4248 # Description : Outputs list of all symbols to a file
4249 #
4250 # Arguments   : none
4251 #############################################################################
4252
4253 def OutputAllSymbols():
4254     SYMBOLS = open(os.path.join(ROOT_DIR, MODULE + "-symbols.txt"), 'w')
4255
4256     for symbol in sorted(AllSymbols.keys()):
4257         SYMBOLS.write(symbol + "\n")
4258     SYMBOLS.close()
4259
4260
4261 #############################################################################
4262 # Function    : OutputSymbolsWithoutSince
4263 # Description : Outputs list of all symbols without a since tag to a file
4264 #
4265 # Arguments   : none
4266 #############################################################################
4267
4268 def OutputSymbolsWithoutSince():
4269     SYMBOLS = open(os.path.join(ROOT_DIR, MODULE + "-nosince.txt"), 'w')
4270
4271     for symbol in sorted(SourceSymbolDocs.keys()):
4272         if symbol in Since:
4273             SYMBOLS.write(symbol + "\n")
4274     SYMBOLS.close()
4275
4276 #############################################################################
4277 # Function    : MergeSourceDocumentation
4278 # Description : This merges documentation read from a source file into the
4279 #                documentation read in from a template file.
4280 #
4281 #                Parameter descriptions override any in the template files.
4282 #                Function descriptions are placed before any description from
4283 #                the template files.
4284 #
4285 # Arguments   : none
4286 #############################################################################
4287
4288 def MergeSourceDocumentation():
4289     if len(SymbolDocs) > 0:
4290         Symbols = SymbolDocs.keys()
4291         logging.info("num existing entries: %d\n", len(Symbols))
4292
4293     else:
4294         # filter scanned declarations, with what we suppress from -sections.txt
4295         tmp = {}
4296         for symbol in Declarations.keys():
4297             if symbol in KnownSymbols and KnownSymbols[symbol] == 1:
4298                 tmp[symbol] = 1
4299
4300
4301         # , add the rest from -sections.txt
4302         for symbol in KnownSymbols.keys():
4303             if KnownSymbols[symbol] == 1:
4304                 tmp[symbol] = 1
4305
4306
4307         # and add whats found in the source
4308         for symbol in SourceSymbolDocs.keys():
4309             tmp[symbol] = 1
4310
4311         Symbols = tmp.keys()
4312         logging.info("num source entries: %d\n", len(Symbols))
4313
4314     for symbol in Symbols:
4315         AllSymbols[symbol] = 1
4316
4317         have_tmpl_docs = 0
4318
4319         ## see if the symbol is documented in template
4320         tmpl_doc = SymbolDocs.get(symbol, '')
4321         check_tmpl_doc = tmpl_doc
4322         # remove all xml-tags and whitespaces
4323         check_tmpl_doc = re.sub(r'<.*?>', '', check_tmpl_doc)
4324         check_tmpl_doc = re.sub(r'\s', '', check_tmpl_doc)
4325         # anything left ?
4326         if check_tmpl_doc != '':
4327             have_tmpl_docs = 1
4328         else:
4329             # if the docs have just an empty para, don't merge that.
4330             check_tmpl_doc = re.sub(r'(\s|\n)', '', tmpl_doc, flags=re.M|re.S)
4331             if check_tmpl_doc == "<para></para>":
4332                 tmpl_doc = ''
4333
4334         if symbol in SourceSymbolDocs:
4335             stype = DeclarationTypes[symbol]
4336
4337             logging.info("merging [%s] from source\n", symbol)
4338
4339             item = "Parameter"
4340             if stype:
4341                 if stype == 'STRUCT':
4342                     item = "Field"
4343                 elif stype == 'ENUM':
4344                     item = "Value"
4345                 elif stype == 'UNION':
4346                     item = "Field"
4347             else:
4348                 stype = "SIGNAL"
4349
4350             src_doc = SourceSymbolDocs[symbol]
4351             # remove leading and training whitespaces
4352             src_doc = src_doc.strip()
4353
4354             # Don't output warnings for overridden titles as titles are
4355             # automatically generated in the -sections.txt file, and thus they
4356             # are often overridden.
4357             m = re.search(r':Title$', symbol)
4358             if have_tmpl_docs and not m:
4359                 # check if content is different
4360                 if tmpl_doc != src_doc:
4361                     #print "[$tmpl_doc] [$src_doc]\n"
4362                     common.LogWarning(SourceSymbolSourceFile[symbol], SourceSymbolSourceLine[symbol],
4363                                       "Documentation in template " + SymbolSourceFile[symbol] + ":" + SymbolSourceLine[symbol] + " for %s being overridden by inline comments." % symbol)
4364
4365             if src_doc != '':
4366                 AllDocumentedSymbols[symbol] = 1
4367
4368
4369             # Do not add <para> to nothing, it breaks missing docs checks.
4370             src_doc_para = ''
4371             if src_doc != '':
4372                 src_doc_para = src_doc
4373
4374
4375             m = re.search(r'$TMPL_DIR\/.+:Long_Description', symbol)
4376             m2 = re.search(TMPL_DIR + r'\/.+:.+', symbol)
4377             if m:
4378                 SymbolDocs[symbol] = '%s%s' % (src_doc_para, tmpl_doc)
4379             elif m2:
4380                 # For the title/summary/see also section docs we don't want to
4381                 # add any <para> tags.
4382                 SymbolDocs[symbol] = src_doc
4383             else:
4384                 SymbolDocs[symbol] = '%s%s' % (src_doc_para, tmpl_doc)
4385
4386             # merge parameters
4387             if re.search(r'.*::.*', symbol):
4388                 # For signals we prefer the param names from the source docs,
4389                 # since the ones from the templates are likely to contain the
4390                 # artificial argn names which are generated by gtkdoc-scangobj.
4391                 SymbolParams[symbol] = SourceSymbolParams[symbol]
4392                 # FIXME: we need to check for empty docs here as well!
4393             else:
4394                 # The templates contain the definitive parameter names and order,
4395                 # so we will not change that. We only override the actual text.
4396                 tmpl_params = SymbolParams[symbol]
4397                 if tmpl_params:
4398                     logging.info("No merge needed for %s\n", symbol)
4399                     SymbolParams[symbol] = SourceSymbolParams[symbol]
4400                     #  FIXME: we still like to get the number of params and merge
4401                     #  1) we would noticed that params have been removed/renamed
4402                     #  2) we would catch undocumented params
4403                     #  params are not (yet) exported in -decl.txt so that we
4404                     #  could easily grab them :/
4405                 else:
4406                     params = SourceSymbolParams[symbol]
4407                     logging.info("Merge needed for %s, tmpl_params: %d, source_params: %d \n", symbol, len(tmpl_params), len(params))
4408                     for j in range(0, len(tmpl_params), PARAM_FIELD_COUNT):
4409                         tmpl_param_name = tmpl_params[j]
4410
4411                         # Try to find the param in the source comment documentation.
4412                         found = False
4413                         logging.info("  try merge param %s\n", tmpl_param_name)
4414                         for k in range(0, len(params), PARAM_FIELD_COUNT):
4415                             param_name = params[k]
4416                             param_desc = params[k + 1]
4417
4418                             logging.info("    test param  %s\n", param_name)
4419                             # We accept changes in case, since the Gnome source
4420                             # docs contain a lot of these.
4421                             if param_name.lower() == tmpl_param_name.lower():
4422                                 found = True
4423
4424                                 # Override the description.
4425                                 tmpl_params[j + 1] = param_desc
4426
4427                                 # Set the name to '' to mark it as used.
4428                                 params[k] = ''
4429                                 break
4430
4431                         # If it looks like the parameters are there, but not
4432                         # in the right place, try to explain a bit better.
4433                         if found and re.search(r'\@%s:' % tmpl_param_name, src_doc):
4434                             common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
4435                                               "Parameters for %s must start on the line immediately after the function or macro name." % symbol)
4436
4437                     # Now we output a warning if parameters have been described which
4438                     # do not exist.
4439                     for j in range(0, len(params), PARAM_FIELD_COUNT):
4440                         param_name = params[j]
4441                         if param_name:
4442                             # the template builder cannot detect if a macro returns
4443                             # a result or not
4444                             if stype == "MACRO" and param_name == "Returns":
4445                                 # FIXME: do we need to add it then to tmpl_params[] ?
4446                                 num = len(tmpl_params)
4447                                 logging.info("  adding Returns: to macro docs for %s.", symbol)
4448                                 tmpl_params[num+1] = "Returns"
4449                                 tmpl_params[num+2] = params[j+1]
4450                                 continue
4451
4452                             common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
4453                                               "%s described in source code comment block but does not exist. %s: %s %s: %s." % (item, stype, symbol, item, param_name))
4454
4455         else:
4456             if have_tmpl_docs:
4457                 AllDocumentedSymbols[symbol] = 1
4458                 logging.info("merging [%s] from template", symbol)
4459
4460             else:
4461                 logging.info("[%s] undocumented\n", symbol)
4462
4463         # if this symbol is documented, check if docs are complete
4464         check_tmpl_doc = SymbolDocs.get(symbol, '')
4465         # remove all xml-tags and whitespaces
4466         check_tmpl_doc = re.sub(r'<.*?>', '', check_tmpl_doc)
4467         check_tmpl_doc = re.sub(r'\s', '', check_tmpl_doc)
4468         if check_tmpl_doc != '':
4469             tmpl_params = SymbolParams[symbol]
4470             if tmpl_params:
4471                 stype = DeclarationTypes[symbol]
4472
4473                 item = "Parameter"
4474                 if stype:
4475                     if stype == 'STRUCT':
4476                         item = "Field"
4477                     elif stype == 'ENUM':
4478                         item = "Value"
4479                     elif stype == 'UNION':
4480                         item = "Field"
4481
4482                 else:
4483                     stype = "SIGNAL"
4484                 logging.info("Check param docs for %s, tmpl_params: %s entries, type=%s\n", symbol, len(tmpl_params), stype)
4485
4486                 if len(tmpl_params > 0):
4487                     for j in range(0, len(tmpl_params), PARAM_FIELD_COUNT):
4488                         # Output a warning if the parameter is empty and
4489                         # remember for stats.
4490                         tmpl_param_name = tmpl_params[j]
4491                         tmpl_param_desc = tmpl_params[j + 1]
4492                         if tmpl_param_name != "void" and not re.search(r'\S', tmpl_param_desc):
4493                             if symbol in AllIncompleteSymbols[symbol]:
4494                                 AllIncompleteSymbols[symbol] += ", " + tmpl_param_name
4495                             else:
4496                                 AllIncompleteSymbols[symbol] = tmpl_param_name
4497
4498                             common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
4499                                               "%s description for %s::%s is missing in source code comment block." % (item, symbol, tmpl_param_name))
4500
4501                 else:
4502                     if len(tmpl_params) == 0:
4503                         AllIncompleteSymbols[symbol] = "<items>"
4504                         common.LogWarning(GetSymbolSourceFile(symbol), GetSymbolSourceLine(symbol),
4505                                           "%s descriptions for %s are missing in source code comment block." % (item, symbol))
4506
4507                     # $#$tmpl_params==-1 means we don't know about parameters
4508                     # this unfortunately does not tell if there should be some
4509     logging.info("num doc entries: %d\n", len(SymbolDocs))
4510
4511
4512 #############################################################################
4513 # Function    : IsEmptyDoc
4514 # Description : Check if a doc-string is empty. Its also regarded as empty if
4515 #               it only consist of whitespace or e.g. FIXME.
4516 # Arguments   : the doc-string
4517 #############################################################################
4518 def IsEmptyDoc(doc):
4519
4520     if re.search(r'^\s*$', doc):
4521         return True
4522
4523
4524     if re.search(r'^\s*<para>\s*(FIXME)?\s*<\/para>\s*$', doc):
4525         return True
4526
4527
4528     return False
4529
4530
4531 #############################################################################
4532 # Function    : ConvertMarkDown
4533 # Description : Converts mark down syntax to the respective docbook.
4534 #               http://de.wikipedia.org/wiki/Markdown
4535 #               Inspired by the design of ParseDown
4536 #               http://parsedown.org/
4537 #               Copyright (c) 2013 Emanuil Rusev, erusev.com
4538 # Arguments   : the symbol name, the doc-string
4539 #############################################################################
4540
4541 def ConvertMarkDown(symbol, text):
4542     text = MarkDownParse(text, symbol)
4543     return text
4544
4545
4546 # SUPPORTED MARKDOWN
4547 # ==================
4548 #
4549 # Atx-style Headers
4550 # -----------------
4551 #
4552 # # Header 1
4553 #
4554 # ## Header 2 ##
4555 #
4556 # Setext-style Headers
4557 # --------------------
4558 #
4559 # Header 1
4560 # ========
4561 #
4562 # Header 2
4563 # --------
4564 #
4565 # Ordered (unnested) Lists
4566 # ------------------------
4567 #
4568 # 1. item 1
4569 #
4570 # 1. item 2 with loooong
4571 #    description
4572 #
4573 # 3. item 3
4574 #
4575 # Note: we require a blank line above the list items
4576 #
4577
4578 # TODO(ensonic): it would be nice to add id parameters to the refsect2 elements
4579
4580 def MarkDownParseBlocks(linesref, symbol, context):
4581   md_blocks = []
4582   md_block = {"type": ''}
4583
4584   # FIXME the original code had OUTER tag here but there does not seem to be inner loops.
4585   for line in linesref:
4586     first_char = line[0]
4587
4588     logging.info("in '" + md_block["type"] + "' state, parsing '%s'" % line)
4589
4590     if md_block["type"] == "markup":
4591       if not md_block["closed"]:
4592         if md_block["start"] in line:
4593           md_block["depth"] += 1
4594
4595         if md_block["end"] in line:
4596           if md_block["depth"] > 0:
4597             md_block["depth"] -= 1
4598           else:
4599             logging.info("closing tag '%s'", line)
4600             md_block["closed"] = 1
4601             # TODO(ensonic): reparse inner text with MarkDownParseLines?
4602
4603         md_block["text"] += "\n" + line
4604         logging.info("add to markup")
4605         continue
4606
4607     deindented_line = line.lstrip()
4608
4609     m = re.search(r'^[#][ \t]+(.+?)[ \t]*[#]*[ \t]*(?:{#([^}]+)})?[ \t]*$', line)
4610     if md_block["type"] == "heading":
4611       # a heading is ended by any level less than or equal
4612       if md_block["level"] == 1:
4613         if re.search(r'^={4,}[ \t]*$', line):
4614           text = md_block["lines"].pop()
4615           md_block["interrupted"] = 0
4616           md_blocks.append(md_block)
4617
4618           md_block = {'type': "heading",
4619                       'text': text,
4620                       'lines': [],
4621                       'level': 1,
4622                      }
4623           continue
4624         elif m:
4625           md_block["interrupted"] = 0
4626           md_blocks.append(md_block)
4627
4628           md_block = {'type': "heading",
4629                       'text': m.group(1),
4630                       'id': m.group(2),
4631                       'lines': [],
4632                       'level': 1,
4633                      }
4634           continue
4635         else:
4636           # push lines into the block until the end is reached
4637           md_block["lines"].append(line)
4638           continue
4639
4640       else:
4641         m2 = re.search(r'^([#]{1,2})[ \t]+(.+?)[ \t]*[#]*[ \t]*(?:{#([^}]+)})?[ \t]*$', line)
4642         if re.search(r'^[=]{4,}[ \t]*$', line):
4643           text = md_block["lines"].pop()
4644           md_block["interrupted"] = 0
4645           md_blocks.append(md_block)
4646
4647           md_block = {'type': "heading",
4648                       'text': text,
4649                       'lines': [],
4650                       'level': 1,
4651           }
4652           continue
4653         elif re.search(r'^[-]{4,}[ \t]*$', line):
4654           text = md_block["lines"].pop()
4655           md_block["interrupted"] = 0
4656           md_blocks.append(md_block)
4657
4658           md_block = {'type': "heading",
4659                       'text': text,
4660                       'lines': [],
4661                       'level': 2,
4662                       }
4663           continue
4664         elif m2:
4665           md_block["interrupted"] = 0
4666           md_blocks.append(md_block)
4667
4668           md_block = {'type': "heading",
4669                       'text': m2.group(2),
4670                       'id': m2.group(3),
4671                       'lines': [],
4672                       'level': len(m2.group(1))
4673                       }
4674           continue
4675         else:
4676           # push lines into the block until the end is reached
4677           md_block["lines"].append(line)
4678           continue
4679     elif md_block["type"] == "code":
4680       m3 = re.search(r'^[ \t]*\]\|(.*)', line)
4681       if m3:
4682         md_blocks.append(md_block)
4683         md_block = {'type': "paragraph",
4684                     'text': "$1",
4685                     'lines': [],
4686                     }
4687       else:
4688         md_block["lines"].append(line)
4689
4690       continue
4691
4692     if deindented_line == '':
4693       md_block["interrupted"] = 1
4694       continue
4695
4696
4697     if md_block["type"] == "quote":
4698       if not md_block["interrupted"]:
4699         line = re.sub(r'^[ ]*>[ ]?', '', line)
4700         md_block["lines"].append(line)
4701         continue
4702
4703     elif md_block["type"] == "li":
4704       marker = md_block["marker"]
4705       m4 = re.search(r'^([ ]{0,3})($marker)[ ](.*)', line)
4706       if m4:
4707         indentation = m.group(1)
4708         if md_block["indentation"] != indentation:
4709           md_block["lines"].append(line)
4710         else:
4711           lines = m4.group(3)
4712           ordered = md_block["ordered"]
4713           lines = re.sub(r'^[ ]{0,4}', '', lines)
4714           md_block["last"] = 0
4715           md_blocks.append(md_block)
4716           md_block = {'type': "li",
4717                       'ordered': ordered,
4718                       'indentation': indentation,
4719                       'marker': marker,
4720                       'first': 0,
4721                       'last': 1,
4722                       'lines': [lines],
4723                       }
4724
4725         continue
4726
4727       if md_block["interrupted"]:
4728         if first_char == " ":
4729           md_block["lines"].append('')
4730           line =  re.sub(r'^[ ]{0,4}', '', line)
4731           md_block["lines"].append(line)
4732           md_block["interrupted"] = 0
4733           continue
4734
4735       else:
4736         line = re.sub(r'^[ ]{0,4}', '', line)
4737         md_block["lines"].append(line)
4738         continue
4739
4740     # indentation sensitive types
4741     logging.info("parsing '%s'", line)
4742
4743     m5 = re.search(r'^([#]{1,2})[ \t]+(.+?)[ \t]*[#]*[ \t]*(?:{#([^}]+)})?[ \t]*$', line)
4744     m6 = re.search(r'^[ \t]*\|\[[ ]*(?:<!-- language="([^"]+?)" -->)?', line)
4745     m7 = re.search(r'^[ ]*<\??(\w+)[^>]*([\/\?])?[ \t]*>', line)
4746     m8 = re.search(r'^([ ]*)[*+-][ ](.*)', line)
4747     m9 = re.search(r'^[ ]*>[ ]?(.*)', line)
4748     if m5:
4749       # atx heading (#)
4750       md_blocks.append(md_block)
4751
4752       md_block = {'type': "heading",
4753                   'text': m5.group(2),
4754                   'id:': m5.group(3),
4755                   'lines': [],
4756                   'level': len(m5.group(1)),
4757                   }
4758
4759       continue
4760     elif re.search(r'^={4,}[ \t]*$', line):
4761       # setext heading (====)
4762
4763       if md_block["type"] == "paragraph" and md_block["interrupted"]:
4764         md_blocks.append(md_block)
4765         md_block["type"] = "heading"
4766         md_block["lines"] = []
4767         md_block["level"] = 1
4768       continue
4769     elif re.search(r'^-{4,}[ \t]*$/', line):
4770       # setext heading (-----)
4771
4772       if md_block["type"] == "paragraph" and md_block["interrupted"]:
4773         md_blocks.append(md_block)
4774         md_block["type"] = "heading"
4775         md_block["lines"] = []
4776         md_block["level"] = 2
4777
4778       continue
4779     elif m6:
4780       # code
4781       md_block["interrupted"] = 1
4782       md_blocks.append(md_block)
4783       md_block = {'type': "code",
4784                   'language': m6.group(1),
4785                   'lines': [],
4786                   }
4787       continue
4788
4789     # indentation insensitive types
4790     if re.search(r'^[ ]*<!DOCTYPE/', line):
4791       md_blocks.append(md_block)
4792
4793       md_block = {'type'   : "markup",
4794                   'text'   : deindented_line,
4795                   'start'  : "<",
4796                   'end'    : ">",
4797                   'closed' : 0,
4798                   'depth'  : 0,
4799                   }
4800
4801     elif m7:
4802       # markup, including <?xml version="1.0"?>
4803       tag = m7.group(1)
4804       is_self_closing = m7.group(2) is not None
4805
4806       # skip link markdown
4807       # TODO(ensonic): consider adding more uri schemes (ftp, ...)
4808       if re.search(r'https?', tag):
4809         logging.info("skipping link '%s'", tag)
4810       else:
4811         # for TEXT_LEVEL_ELEMENTS, we want to keep them as-is in the paragraph
4812         # instead of creation a markdown block.
4813         scanning_for_end_of_text_level_tag = (
4814             md_block["type"] == "paragraph" and
4815             'start' in md_block and
4816             not md_block["closed"])
4817         logging.info("markup found '%s', scanning %s ?", tag, scanning_for_end_of_text_level_tag)
4818         if not MD_TEXT_LEVEL_ELEMENTS[tag] and not scanning_for_end_of_text_level_tag:
4819           md_blocks.append(md_block)
4820
4821           if is_self_closing:
4822             logging.info("self-closing docbook '%s'", tag)
4823             md_block = {'type': "self-closing tag",
4824                         'text': deindented_line,
4825                         }
4826             is_self_closing = 0
4827             continue
4828
4829           logging.info("new markup '%s'", tag)
4830           md_block = {'type'   : "markup",
4831                       'text'   : deindented_line,
4832                       'start'  : "<" + tag + ">",
4833                       'end'    : "</" + tag + ">",
4834                       'closed' : 0,
4835                       'depth'  : 0,
4836                       }
4837           if re.search(r'<\/%s>' % tag, deindented_line):
4838             md_block["closed"] = 1
4839
4840           continue
4841         else:
4842           if MD_TEXT_LEVEL_ELEMENTS[tag]:
4843             logging.info("text level docbook '%s' in '%s' state", tag, md_block["type"])
4844             # TODO(ensonic): handle nesting
4845             if scanning_for_end_of_text_level_tag:
4846               if not re.search(r'<\/%s>' % tag, deindented_line):
4847                 logging.info("new text level markup '%s'", tag)
4848                 md_block["start"] = "<" + tag + ">"
4849                 md_block["end"] = "</" + tag + ">"
4850                 md_block["closed"] = 0
4851                 logging.info("scanning for end of '%s'", tag)
4852
4853             else:
4854               if re.search(md_block["end"], deindented_line):
4855                 md_block["closed"] = 1
4856                 logging.info("found end of '%s'", tag)
4857
4858     elif m8:
4859       # li
4860       md_blocks.append(md_block)
4861       lines = m8.group(2)
4862       indentation = m8.group(1)
4863       lines = re.sub(r'^[ ]{0,4}', '', lines)
4864       md_block = {'type': "li",
4865                   'ordered': 0,
4866                   'indentation': indentation,
4867                   'marker': "[*+-]",
4868                   'first': 1,
4869                   'last': 1,
4870                   'lines': [lines],
4871                   }
4872       continue
4873     elif m9:
4874       md_blocks.append(md_block)
4875       md_block = {'type': "quote",
4876                   'lines': [m9.group(1)],
4877                   }
4878       continue
4879
4880
4881     # list item
4882     m10 = re.search(r'^([ ]{0,4})\d+[.][ ]+(.*)', line)
4883     if m10:
4884       md_blocks.append(md_block)
4885       lines = m10.group(2)
4886       indentation = m10.group(1)
4887       lines = re.sub(r'^[ ]{0,4}', '', lines)
4888
4889       md_block = {'type': "li",
4890                   'ordered': 1,
4891                   'indentation': indentation,
4892                   'marker': "\\d+[.]",
4893                   'first': 1,
4894                   'last': 1,
4895                   'lines': [lines],
4896                   }
4897
4898       continue
4899
4900
4901     # paragraph
4902     if md_block["type"] == "paragraph":
4903       if md_block["interrupted"]:
4904         md_blocks.append(md_block)
4905         md_block = {'type': "paragraph",
4906                     'interrupted': 0,
4907                     'text': line,
4908                     }
4909         logging.info("new paragraph due to interrupted")
4910       else:
4911         md_block["text"] += "\n" + line
4912         logging.info("add to paragraph")
4913
4914     else:
4915       md_blocks.append(md_block)
4916       md_block = {'type': "paragraph",
4917                   'text': line,
4918                   }
4919       logging.info("new paragraph due to different block type")
4920
4921   md_blocks.append(md_block)
4922
4923   return md_blocks[1:]
4924
4925
4926 def MarkDownParseSpanElementsInner(text, markersref):
4927   markup = ''
4928   markers = {}
4929   for i in markersref:
4930       markers[i] = 1
4931
4932   while text != '':
4933     closest_marker = ''
4934     # closest_marker_index = 0
4935     closest_marker_position = -1
4936     text_marker = ''
4937     # i = 0
4938     offset = 0
4939     markers_rest = []
4940
4941     for marker, use in markers.items():
4942       if not use:
4943         continue
4944
4945
4946       marker_position = text.find(marker)
4947
4948       if marker_position < 0:
4949         markers[marker] = 0
4950         continue
4951
4952
4953       if closest_marker == '' or marker_position < closest_marker_position:
4954         closest_marker = marker
4955         # closest_marker_index = i
4956         closest_marker_position = marker_position
4957
4958     if closest_marker_position >= 0:
4959       text_marker = text[closest_marker_position:]
4960
4961
4962     if text_marker == '':
4963       markup += text
4964       text = ''
4965       continue # last
4966
4967
4968     markup += text[0:closest_marker_position]
4969     text = text[closest_marker_position:]
4970     markers_rest = []
4971     for key in markers.keys():
4972         if markers[key]:
4973             if key == closest_marker:
4974                 markers_rest.append([])
4975             else:
4976                 markers_rest.append(key)
4977         else:
4978             markers_rest.append([])
4979
4980     if closest_marker == "![" or closest_marker == "[":
4981       element = {}
4982
4983       m = re.search(r'\[((?:[^][]|(?R))*)\]', text)
4984       if ']' in text and m:
4985
4986         element = {"!": text[0] == "!",
4987                    "a": m.group(1),
4988                    }
4989
4990         offset = len(m.group(0))
4991         if element["!"]:
4992           offset+=1
4993
4994
4995         remaining_text = text[offset:]
4996         m2 = re.search(r'''^\([ ]*([^)'"]*?)(?:[ ]+['"](.+?)['"])?[ ]*\)''', remaining_text)
4997         m3 = re.search(r'^\s*\[([^\]<]*?)\]', remaining_text)
4998         if m2:
4999           element["»"] = m2.group(1)
5000           if m2.group(2):
5001             element["#"] = m2.group(2)
5002             offset += len(m2.group(0))
5003         elif m3:
5004           element["ref"] = m3.group(1)
5005           offset += len(m3.group(0))
5006         else:
5007           element = None
5008
5009       if element:
5010         if element["»"]:
5011           element["»"] = element["»"].replace('&', '&amp;')
5012           element["»"] = element["»"].replace('<', '&lt;')
5013
5014         if element["!"]:
5015           markup += '<inlinemediaobject><imageobject><imagedata fileref="' + element['»'] + '"></imagedata></imageobject>'
5016
5017           if 'a' in element:
5018             markup += "<textobject><phrase>" + element["a"] + "</phrase></textobject>"
5019
5020
5021             markup += "</inlinemediaobject>"
5022         elif element["ref"]:
5023           element["a"] = MarkDownParseSpanElementsInner(element["a"], markers_rest)
5024           markup += '<link linkend="' + element['ref'] + '"'
5025
5026           if '#' in element["#"]:
5027             # title attribute not supported
5028
5029
5030             markup += ">" + element["a"] + "</link>"
5031         else:
5032           element["a"] = MarkDownParseSpanElementsInner(element["a"], markers_rest)
5033           markup += '<ulink url="' + element['»'] + '"'
5034
5035           if '#' in element:
5036             # title attribute not supported
5037             markup += ">" + element["a"] + "</ulink>"
5038
5039       else:
5040         markup += closest_marker
5041         if closest_marker == "![":
5042           offset = 2
5043         else:
5044           offset = 1
5045
5046
5047     elif closest_marker == "<":
5048       m4 = re.search(r'^<(https?:[\/]{2}[^\s]+?)>', text)
5049       m5 = re.search(r'^<([A-Za-z0-9._-]+?@[A-Za-z0-9._-]+?)>', text)
5050       m6 = re.search(r'^<[^>]+?>', text)
5051       if m4:
5052         element_url = m4.group(1).replace('&', '&amp;').replace('<', '&lt;')
5053
5054         markup += '<ulink url="' + element_url + '">' + element_url + '</ulink>'
5055         offset = len(m4.group(0))
5056       elif m5:
5057         markup += "<ulink url=\"mailto:" + m5.group(1) + "\">" + m5.group(1) + "</ulink>"
5058         offset = len(m5.group(0))
5059       elif m6:
5060         markup += m6.group(0)
5061         offset = len(m6.group(0))
5062       else:
5063         markup += "&lt;"
5064         offset = 1
5065
5066     elif closest_marker == "\\":
5067       special_char = text[1]
5068       if MD_ESCAPABLE_CHARS[special_char] or \
5069           MD_GTK_ESCAPABLE_CHARS[special_char]:
5070         markup += special_char
5071         offset = 2
5072       else:
5073         markup += "\\"
5074         offset = 1
5075
5076     elif closest_marker == "`":
5077       m7 = re.search(r'^(`+)([^`]+?)\1(?!`)', text)
5078       if m7:
5079         element_text = m7.group(2)
5080         markup += "<literal>" + element_text + "</literal>"
5081         offset = len(m7.group(0))
5082       else:
5083         markup += "`"
5084         offset = 1
5085
5086     elif closest_marker == "@":
5087       # Convert '@param()'
5088       # FIXME: we could make those also links ($symbol.$2), but that would be less
5089       # useful as the link target is a few lines up or down
5090       m7 = re.search(r'^(\A|[^\\])\@(\w+((\.|->)\w+)*)\s*\(\)', text)
5091       m8 = re.search(r'^(\A|[^\\])\@(\w+((\.|->)\w+)*)', text)
5092       m9 = re.search(r'^\\\@', text)
5093       if m7:
5094         markup += m7.group(1) + "<parameter>" + m7.group(2) + "()</parameter>\n"
5095         offset = len(m7.group(0))
5096       elif m8:
5097         # Convert '@param', but not '\@param'.
5098         markup += m8.group(1) + "<parameter>" + m8.group(2) + "</parameter>\n"
5099         offset = len(m8.group(0))
5100       elif m9:
5101         markup += r"\@"
5102         offset = len(m9.group(0))
5103       else:
5104         markup += "@"
5105         offset = 1
5106
5107     elif closest_marker == "#":
5108       m10 = re.search(r'^(\A|[^\\])#([\w\-:\.]+[\w]+)\s*\(\)', text)
5109       m11 = re.search(r'^(\A|[^\\])#([\w\-:\.]+[\w]+)', text)
5110       m12 = re.search(r'^\\#', text)
5111       if m10:
5112         # handle #Object.func()
5113         markup += m10.group(1) + MakeXRef(m10.group(2), tagify(m10.group(2) + "()", "function"))
5114         offset = len(m10.group(0))
5115       elif m11:
5116         # Convert '#symbol', but not '\#symbol'.
5117         markup += m11.group(1) + MakeHashXRef(m11.group(2), "type")
5118         offset = len(m11.group(0))
5119       elif m12:
5120         markup += "#"
5121         offset = len(m12.group(0))
5122       else:
5123         markup += "#"
5124         offset = 1
5125
5126     elif closest_marker == "%":
5127       m12 = re.search(r'^(\A|[^\\])\%(-?\w+)', text)
5128       m13 = re.search(r'^\\%', text)
5129       if m12:
5130         # Convert '%constant', but not '\%constant'.
5131         # Also allow negative numbers, e.g. %-1.
5132         markup += m12.group(1) + MakeXRef(m12.group(2), tagify(m12.group(2), "literal"))
5133         offset = len(m12.group(0))
5134       elif m13:
5135         markup += r"\%"
5136         offset = len(m13.group(0))
5137       else:
5138         markup += "%"
5139         offset = 1
5140
5141
5142
5143     if offset > 0:
5144       text = text[offset:]
5145
5146   return markup
5147
5148
5149 def MarkDownParseSpanElements(text):
5150     markers = ["\\", "<", "![", "[", "`", "%", "#", "@"]
5151
5152     text = MarkDownParseSpanElementsInner(text, markers)
5153
5154     # Convert 'function()' or 'macro()'.
5155     # if there is abc_*_def() we don't want to make a link to _def()
5156     # FIXME: also handle abc(def(....)) : but that would need to be done recursively :/
5157     def f(m):
5158         return m.group(1) + MakeXRef(m.group(2), tagify(m.group(2) + "()", "function")) + ';'
5159     text = re.sub(r'([^\*.\w])(\w+)\s*\(\)', f, text)
5160     return text
5161
5162
5163 def ReplaceEntities(text, symbol):
5164     entities = [["&lt;", "<"],
5165                 ["&gt;", ">"],
5166                 ["&ast;", "*"],
5167                 ["&num;", "#"],
5168                 ["&percnt;", "%"],
5169                 ["&colon;", ":"],
5170                 ["&quot;", '"'],
5171                 ["&apos;", "'"],
5172                 ["&nbsp;", " "],
5173                 ["&amp;", "&"], # Do this last, or the others get messed up.
5174                ]
5175
5176
5177     # Expand entities in <programlisting> even inside CDATA since
5178     # we changed the definition of |[ to add CDATA
5179     for i in entities:
5180         text = re.sub(i[0], i[1], text)
5181     return text
5182
5183
5184 def MarkDownOutputDocBook(blocksref, symbol, context):
5185   output = ''
5186   blocks = blocksref
5187
5188   for block in blocks:
5189     #$output += "\n<!-- beg type='" . $block->{"type"} . "'-->\n"
5190
5191     if block["type"] == "paragraph":
5192       text = MarkDownParseSpanElements(block["text"])
5193       if context == "li" and output == '':
5194         if block["interrupted"]:
5195           output += "\n<para>%s</para>\n" % text
5196         else:
5197           output += "<para>%s</para>" % text
5198           if len(blocks) > 0:
5199             output += "\n"
5200       else:
5201         output += "<para>%s</para>\n" % text
5202
5203     elif block["type"] == "heading":
5204
5205       title = MarkDownParseSpanElements(block["text"])
5206
5207       if block["level"] == 1:
5208         tag = "refsect2"
5209       else:
5210         tag = "refsect3"
5211
5212
5213       text = MarkDownParseLines(block["lines"], symbol, "heading")
5214       if id in block:
5215         output += "<%s id=\"%s\">" % (tag, block["id"])
5216       else:
5217         output += "<%s>" % tag
5218
5219
5220       output += "<title>%s</title>%s</%s>\n" % (title, text, tag)
5221     elif block["type"] == "li":
5222       tag = "itemizedlist"
5223
5224       if block["first"]:
5225         if block["ordered"]:
5226           tag = "orderedlist"
5227
5228         output += "<%s>\n" % tag
5229
5230
5231       if block["interrupted"]:
5232         block["lines"].append('')
5233
5234
5235       text = MarkDownParseLines(block["lines"], symbol, "li")
5236       output += "<listitem>" + text + "</listitem>\n"
5237       if block["last"]:
5238         if block["ordered"]:
5239           tag = "orderedlist"
5240         output += "</$tag>\n"
5241
5242     elif block["type"] == "quote":
5243       text = MarkDownParseLines(block["lines"], symbol, "quote")
5244       output += "<blockquote>\n%s</blockquote>\n" % text
5245     elif block["type"] == "code":
5246       tag = "programlisting"
5247
5248       if block["language"]:
5249         if block["language"] == "plain":
5250           output += "<informalexample><screen><![CDATA[\n"
5251           tag = "screen"
5252         else:
5253           output += '<informalexample><programlisting language="' + block['language'] + '"><![CDATA[\n'
5254
5255       else:
5256         output += "<informalexample><programlisting><![CDATA[\n"
5257
5258       for line in block["lines"]:
5259         output += ReplaceEntities(line, symbol) + "\n"
5260
5261       output += "]]></$tag></informalexample>\n"
5262     elif block["type"] == "markup":
5263       text = ExpandAbbreviations(symbol, block["text"])
5264       output += text+"\n"
5265     else:
5266       output += block["text"]+"\n"
5267
5268     #$output += "\n<!-- end type='" . $block->{"type"} . "'-->\n"
5269
5270   return output
5271
5272
5273 def MarkDownParseLines(linesref, symbol, context):
5274     lines = linesref
5275
5276     blocks = MarkDownParseBlocks(lines, symbol, context)
5277     output = MarkDownOutputDocBook(blocks, symbol, context)
5278
5279     return output
5280
5281
5282 def MarkDownParse(text, symbol):
5283     # take out some variability in line endings
5284     text = re.sub(r'\r\n', '\n', text)
5285     text = re.sub(r'\r', '\n', text)
5286
5287     # split lines
5288     lines = text.split('\n')
5289     text = MarkDownParseLines(lines, symbol, '')
5290
5291     return text
5292
5293
5294 #############################################################################
5295 # Function    : ReadDeclarationsFile
5296 # Description : This reads in a file containing the function/macro/enum etc.
5297 #                declarations.
5298 #
5299 #                Note that in some cases there are several declarations with
5300 #                the same name, e.g. for conditional macros. In this case we
5301 #                set a flag in the %DeclarationConditional hash so the
5302 #                declaration is not shown in the docs.
5303 #
5304 #                If a macro and a function have the same name, e.g. for
5305 #                gtk_object_ref, the function declaration takes precedence.
5306 #
5307 #                Some opaque structs are just declared with 'typedef struct
5308 #                _name name;' in which case the declaration may be empty.
5309 #                The structure may have been found later in the header, so
5310 #                that overrides the empty declaration.
5311 #
5312 # Arguments   : $file - the declarations file to read
5313 #                $override - if declarations in this file should override
5314 #                        any current declaration.
5315 #############################################################################
5316
5317 def ReadDeclarationsFile(ifile, override):
5318
5319     if override == 0:
5320         global Declarations, DeclarationTypes, DeclarationConditional, DeclarationOutput
5321         Declarations = {}
5322         DeclarationTypes = {}
5323         DeclarationConditional = {}
5324         DeclarationOutput = {}
5325
5326
5327     INPUT = open(ifile)
5328     declaration_type = ''
5329     declaration_name = None
5330     declaration = None
5331     is_deprecated = 0
5332     line_number = 0
5333     for line in INPUT:
5334         line_number += 1
5335         if not declaration_type:
5336             m1 = re.search(r'^<([^>]+)>', line)
5337             if m1:
5338                 declaration_type = m1.group(1)
5339                 declaration_name = ''
5340                 logging.info("Found declaration: %s", declaration_type)
5341                 declaration = ''
5342
5343         else:
5344             m2 = re.search(r'^<NAME>(.*)</NAME>', line)
5345             m3 = re.search(r'^<DEPRECATED/>', line)
5346             m4 = re.search(r'^</$declaration_type>', line)
5347             if m2:
5348                 declaration_name = m2.group(1)
5349             elif m3:
5350                 is_deprecated = True
5351             elif m4:
5352                 logging.info("Found end of declaration: %s", declaration_name)
5353                 # Check that the declaration has a name
5354                 if declaration_name == '':
5355                     common.LogWarning(ifile, line_number, declaration_type + " has no name.\n")
5356
5357
5358                 # If the declaration is an empty typedef struct _XXX XXX
5359                 # set the flag to indicate the struct has a typedef.
5360                 if declaration_type == 'STRUCT' or declaration_type == 'UNION' \
5361                     and re.search(r'^\s*$', declaration):
5362                     logging.info("Struct has typedef: " + declaration_name)
5363                     StructHasTypedef[declaration_name] = 1
5364
5365
5366                 # Check if the symbol is already defined.
5367                 if declaration_name in Declarations and override == 0:
5368                     # Function declarations take precedence.
5369                     if DeclarationTypes[declaration_name] == 'FUNCTION':
5370                         # Ignore it.
5371                         pass
5372                     elif declaration_type == 'FUNCTION':
5373                         if is_deprecated:
5374                             Deprecated[declaration_name] = ''
5375
5376                         Declarations[declaration_name] = declaration
5377                         DeclarationTypes[declaration_name] = declaration_type
5378                     elif DeclarationTypes[declaration_name] == declaration_type:
5379                         # If the existing declaration is empty, or is just a
5380                         # forward declaration of a struct, override it.
5381                         if declaration_type == 'STRUCT' or declaration_type == 'UNION':
5382                             if re.search(r'^\s*((struct|union)\s+\w+\s*;)?\s*$', Declarations[declaration_name]):
5383                                 if is_deprecated:
5384                                     Deprecated[declaration_name] = ''
5385                                 Declarations[declaration_name] = declaration
5386                             elif re.search(r'^\s*((struct|union)\s+\w+\s*;)?\s*$', declaration):
5387                                 # Ignore an empty or forward declaration.
5388                                 pass
5389                             else:
5390                                 common.LogWarning(ifile, line_number, "Structure %s has multiple definitions." % declaration_name)
5391
5392                         else:
5393                             # set flag in %DeclarationConditional hash for
5394                             # multiply defined macros/typedefs.
5395                             DeclarationConditional[declaration_name] = 1
5396
5397                     else:
5398                         common.LogWarning(ifile, line_number, declaration_name + " has multiple definitions.")
5399
5400                 else:
5401                     if is_deprecated:
5402                         Deprecated[declaration_name] = ''
5403
5404                     Declarations[declaration_name] = declaration
5405                     DeclarationTypes[declaration_name] = declaration_type
5406
5407
5408                 declaration_type = ''
5409                 is_deprecated = False
5410             else:
5411                 declaration += line
5412     INPUT.close()
5413
5414
5415 #############################################################################
5416 # Function    : ReadSignalsFile
5417 # Description : This reads in an existing file which contains information on
5418 #                all GTK signals. It creates the arrays @SignalNames and
5419 #                @SignalPrototypes containing info on the signals. The first
5420 #                line of the SignalPrototype is the return type of the signal
5421 #                handler. The remaining lines are the parameters passed to it.
5422 #                The last parameter, "gpointer user_data" is always the same
5423 #                so is not included.
5424 # Arguments   : $file - the file containing the signal handler prototype
5425 #                        information.
5426 #############################################################################
5427
5428 def ReadSignalsFile(ifile):
5429
5430     in_signal = 0
5431     signal_object = None
5432     signal_name = None
5433     signal_returns = None
5434     signal_flags = None
5435     signal_prototype = None
5436
5437     # Reset the signal info.
5438     global SignalObjects, SignalNames, SignalReturns, SignalFlags, SignalPrototypes
5439     SignalObjects = []
5440     SignalNames = []
5441     SignalReturns = []
5442     SignalFlags = []
5443     SignalPrototypes = []
5444
5445     if not os.path.isfile(ifile):
5446         return
5447
5448     INPUT = open(ifile)
5449     line_number = 0
5450     for line in INPUT:
5451         line_number += 1
5452         if not in_signal:
5453             if re.search(r'^<SIGNAL>', line):
5454                 in_signal = 1
5455                 signal_object = ''
5456                 signal_name = ''
5457                 signal_returns = ''
5458                 signal_prototype = ''
5459
5460         else:
5461             m = re.search(r'^<NAME>(.*)<\/NAME>', line)
5462             m2 = re.search(r'^<RETURNS>(.*)<\/RETURNS>', line)
5463             m3 = re.search(r'^<FLAGS>(.*)<\/FLAGS>', line)
5464             if m:
5465                 signal_name = m.group(1)
5466                 m1_2 = re.search(r'^(.*)::(.*)$', signal_name)
5467                 if m1_2:
5468                     signal_object = m1_2.group(1)
5469                     signal_name = m1_2.group(2).replace('_', '-')
5470                     logging.info("Found signal: %s", signal_name)
5471                 else:
5472                     common.LogWarning(ifile, line_number, "Invalid signal name: %s." % signal_name)
5473
5474             elif m2:
5475                 signal_returns = m2.group(1)
5476             elif m3:
5477                 signal_flags = m3.group(1)
5478             elif re.search(r'^</SIGNAL>', line):
5479                 logging.info("Found end of signal: %s::%s\nReturns: %s\n%s", signal_object, signal_name, signal_returns, signal_prototype)
5480                 SignalObjects.append(signal_object)
5481                 SignalNames.append(signal_name)
5482                 SignalReturns.append(signal_returns)
5483                 SignalFlags.append(signal_flags)
5484                 SignalPrototypes.append(signal_prototype)
5485                 in_signal = False
5486             else:
5487                 signal_prototype += line
5488     INPUT.close()
5489
5490
5491
5492 #############################################################################
5493 # Function    : ReadTemplateFile
5494 # Description : This reads in the manually-edited documentation file
5495 #               corresponding to the file currently being created, so we can
5496 #               insert the documentation at the appropriate places.
5497 #               It outputs %SymbolTypes, %SymbolDocs and %SymbolParams, which
5498 #               is a hash of arrays.
5499 # Arguments   : $docsfile - the template file to read in.
5500 #               $skip_unused_params - 1 if the unused parameters should be
5501 #                 skipped.
5502 #############################################################################
5503
5504 def ReadTemplateFile(docsfile, skip_unused_params):
5505
5506     template = docsfile + ".sgml"
5507     if not os.path.isfile(template):
5508         logging.info("File doesn't exist: " + template)
5509         return 0
5510
5511
5512     # start with empty hashes, we merge the source comment for each file
5513     # afterwards
5514     global SymbolDocs, SymbolTypes, SymbolParams
5515     SymbolDocs = {}
5516     SymbolTypes = {}
5517     SymbolParams = {}
5518
5519     current_type = ''        # Type of symbol being read.
5520     current_symbol = ''        # Name of symbol being read.
5521     symbol_doc = ''                # Description of symbol being read.
5522     params = []                        # Parameter names and descriptions of current
5523                                 #   function/macro/function typedef.
5524     current_param = -1        # Index of parameter currently being read.
5525                                 #   Note that the param array contains pairs
5526                                 #   of param name & description.
5527     in_unused_params = 0        # True if we are reading in the unused params.
5528     in_deprecated = 0
5529     in_since = 0
5530     in_stability = 0
5531
5532     DOCS = open(template)
5533
5534     logging.info("reading template " + template)
5535
5536     line_number = 0
5537     for line in DOCS:
5538         line_number += 1
5539         m1 = re.search(r'^<!-- ##### ([A-Z_]+) (\S+) ##### -->', line)
5540         if m1:
5541             stype = m1.group(1)
5542             symbol = m1.group(2)
5543             if symbol == "Title" \
5544                 or symbol == "Short_Description" \
5545                 or symbol == "Long_Description" \
5546                 or symbol == "See_Also" \
5547                 or symbol == "Stability_Level" \
5548                 or symbol == "Include" \
5549                 or symbol == "Image":
5550
5551                 symbol = docsfile + ":" + symbol
5552
5553
5554             logging.info("Found symbol: " + symbol)
5555             # Remember file and line for the symbol
5556             SymbolSourceFile[symbol] = template
5557             SymbolSourceLine[symbol] = line_number
5558
5559             # Store previous symbol, but remove any trailing blank lines.
5560             if current_symbol != '':
5561                 symbol_doc = symbol_doc.rstrip()
5562                 SymbolTypes[current_symbol] = current_type
5563                 SymbolDocs[current_symbol] = symbol_doc
5564
5565                 # Check that the stability level is valid.
5566                 if StabilityLevel[current_symbol]:
5567                     StabilityLevel[current_symbol] = ParseStabilityLevel(StabilityLevel[current_symbol], template, line_number, "Stability level for " + current_symbol)
5568
5569                 if current_param >= 0:
5570                     SymbolParams[current_symbol] = [params]
5571                 else:
5572                     # Delete any existing params in case we are overriding a
5573                     # previously read template.
5574                     del SymbolParams[current_symbol]
5575
5576
5577             current_type = stype
5578             current_symbol = symbol
5579             current_param = -1
5580             in_unused_params = 0
5581             in_deprecated = 0
5582             in_since = 0
5583             in_stability = 0
5584             symbol_doc = ''
5585             params = []
5586
5587         elif re.search(r'^<!-- # Unused Parameters # -->', line):
5588             logging.info("Found unused parameters\n")
5589             in_unused_params = True
5590             continue
5591
5592         elif in_unused_params and skip_unused_params:
5593             # When outputting the DocBook we skip unused parameters.
5594             logging.info("Skipping unused param: " + line)
5595             continue
5596
5597         else:
5598             # Check if param found. Need to handle "..." and "format...".
5599             m2 = re.search(r'^\@([\w\.]+):\040?', line)
5600             if m2:
5601                 line = re.sub(r'^\@([\w\.]+):\040?', '', line)
5602                 param_name = m2.group(1)
5603                 param_desc = line
5604                 # Allow variations of 'Returns'
5605                 if re.search(r'^[Rr]eturns?$', param_name):
5606                     param_name = "Returns"
5607
5608                 # Allow varargs variations
5609                 if re.search(r'^.*\.\.\.$', param_name):
5610                     param_name = "..."
5611
5612
5613                 # strip trailing whitespaces and blank lines
5614                 line = re.sub(r'\s+\n$', '\n', line, flags=re.M)
5615                 line = re.sub(r'\n+$', '\n', line, flags=re.M|re.S)
5616                 logging.info("Found param for symbol %s : '%s'= '%s'", current_symbol, param_name, line)
5617
5618                 if param_name == "Deprecated":
5619                     in_deprecated = True
5620                     Deprecated[current_symbol] = line
5621                 elif param_name == "Since":
5622                     in_since = True
5623                     Since[current_symbol] = line.strip()
5624                 elif param_name == "Stability":
5625                     in_stability = True
5626                     StabilityLevel[current_symbol] = line
5627                 else:
5628                     params.append(param_name)
5629                     params.append(param_desc)
5630                     current_param += PARAM_FIELD_COUNT
5631
5632             else:
5633                 # strip trailing whitespaces and blank lines
5634                 line = re.sub(r'\s+\n$', '\n', line, flags=re.M)
5635                 line = re.sub(r'\n+$', '\n', line, flags=re.M|re.S)
5636
5637                 if re.search(r'^\s+$', line):
5638                     if in_deprecated:
5639                         Deprecated[current_symbol] += line
5640                     elif in_since:
5641                         common.LogWarning(template, line_number, "multi-line since docs found")
5642                         #$Since{$current_symbol} += $_
5643                     elif in_stability:
5644                         StabilityLevel[current_symbol] += line
5645                     elif current_param >= 0:
5646                         params[current_param] += line
5647                     else:
5648                         symbol_doc += line
5649
5650     # Remember to finish the current symbol doccs.
5651     if current_symbol != '':
5652
5653         symbol_doc = re.sub(r'\s+$', '', symbol_doc)
5654         SymbolTypes[current_symbol] = current_type
5655         SymbolDocs[current_symbol] = symbol_doc
5656
5657         # Check that the stability level is valid.
5658         if StabilityLevel[current_symbol]:
5659             StabilityLevel[current_symbol] = ParseStabilityLevel(StabilityLevel[current_symbol], template, line_number, "Stability level for " + current_symbol)
5660
5661         if current_param >= 0:
5662             SymbolParams[current_symbol] = [params]
5663         else:
5664             # Delete any existing params in case we are overriding a
5665             # previously read template.
5666             del SymbolParams[current_symbol]
5667
5668     DOCS.close()
5669     return 1
5670
5671
5672
5673 #############################################################################
5674 # Function    : ReadObjectHierarchy
5675 # Description : This reads in the $MODULE-hierarchy.txt file containing all
5676 #               the GtkObject subclasses described in this module (and their
5677 #               ancestors).
5678 #               It places them in the @Objects array, and places their level
5679 #               in the object hierarchy in the @ObjectLevels array, at the
5680 #               same index. GtkObject, the root object, has a level of 1.
5681 #
5682 #               This also generates tree_index.sgml as it goes along.
5683 #
5684 # Arguments   : none
5685 #############################################################################
5686
5687 def ReadObjectHierarchy():
5688
5689     global Objects, ObjectLevels
5690     Objects = []
5691     ObjectLevels = []
5692
5693     if not os.path.isfile(OBJECT_TREE_FILE):
5694         return
5695
5696     INPUT = open(OBJECT_TREE_FILE)
5697
5698     # Only emit objects if they are supposed to be documented, or if
5699     # they have documented children. To implement this, we maintain a
5700     # stack of pending objects which will be emitted if a documented
5701     # child turns up.
5702     pending_objects = []
5703     pending_levels = []
5704     root = None
5705     tree = []
5706     for line in INPUT:
5707         m1 = re.search(r'\S+', line)
5708         if m1:
5709             gobject = m1.group(0)
5710             level = len(line[0:m1.start()]) / 2 + 1
5711             xref = ''
5712
5713             if level == 1:
5714                 root = gobject
5715
5716
5717             while pending_levels and pending_levels[-1] >= level:
5718                 pending_objects.pop()
5719                 pending_levels.pop()
5720
5721
5722             pending_objects.append(gobject)
5723             pending_levels.append(level)
5724
5725             if gobject in KnownSymbols:
5726                 while len(pending_levels) > 0:
5727                     gobject = pending_objects[0]
5728                     pending_objects = pending_objects[1:]
5729                     level = pending_levels[0]
5730                     pending_levels = pending_levels[1:]
5731                     xref = MakeXRef(gobject)
5732
5733                     tree.append(' ' * (level * 4) + xref)
5734                     Objects.append(gobject)
5735                     ObjectLevels.append(level)
5736                     ObjectRoots[gobject] = root
5737
5738
5739             #else
5740             #    common.LogWarning($OBJECT_TREE_FILE, line_number, "unknown type $object")
5741             #
5742
5743
5744     INPUT.close()
5745
5746     # FIXME: use xml
5747     # my $old_tree_index = "$DB_OUTPUT_DIR/tree_index.$xml"
5748     old_tree_index = os.path.join(DB_OUTPUT_DIR, "tree_index.sgml")
5749     new_tree_index = os.path.join(DB_OUTPUT_DIR, "tree_index.new")
5750
5751     OUTPUT = open(new_tree_index, 'w')
5752
5753     OUTPUT.write(MakeDocHeader("screen") + "\n<screen>\n" + AddTreeLineArt(tree) + "\n</screen>\n")
5754     OUTPUT.close()
5755
5756     common.UpdateFileIfChanged(old_tree_index, new_tree_index, 0)
5757
5758     OutputObjectList()
5759
5760
5761 #############################################################################
5762 # Function    : ReadInterfaces
5763 # Description : This reads in the $MODULE.interfaces file.
5764 #
5765 # Arguments   : none
5766 #############################################################################
5767
5768 def ReadInterfaces():
5769     global Interfaces
5770     Interfaces = {}
5771
5772     if not os.path.isfile(INTERFACES_FILE):
5773         return
5774
5775     INPUT = open(INTERFACES_FILE)
5776
5777     for line in INPUT:
5778         line = line.strip()
5779         (gobject, ifaces) = line.split()
5780         if gobject in KnownSymbols and KnownSymbols[gobject] == 1:
5781             knownIfaces = []
5782
5783             # filter out private interfaces, but leave foreign interfaces
5784             for iface in ifaces:
5785                 if iface not in KnownSymbols or KnownSymbols[iface] == 1:
5786                     knownIfaces.append(iface)
5787
5788
5789
5790             Interfaces[gobject] = ' '.join(knownIfaces)
5791             logging.info("Interfaces for %s: %s\n", gobject, Interfaces[gobject])
5792         else:
5793             logging.info("skipping interfaces for unknown symbol: %s", gobject)
5794
5795     INPUT.close()
5796
5797
5798 #############################################################################
5799 # Function    : ReadPrerequisites
5800 # Description : This reads in the $MODULE.prerequisites file.
5801 #
5802 # Arguments   : none
5803 #############################################################################
5804
5805 def ReadPrerequisites():
5806     global Prerequisites
5807     Prerequisites = {}
5808
5809     if not os.path.isfile(PREREQUISITES_FILE):
5810         return
5811
5812     INPUT = open(PREREQUISITES_FILE)
5813
5814     for line in INPUT:
5815         line = line.strip()
5816         (iface, prereqs) = line.split()
5817         if iface in KnownSymbols and KnownSymbols[iface] == 1:
5818             knownPrereqs = []
5819
5820             # filter out private prerequisites, but leave foreign prerequisites
5821             for prereq in prereqs:
5822                 if prereq not in KnownSymbols or KnownSymbols[prereq] == 1:
5823                     knownPrereqs.append(prereq)
5824
5825             Prerequisites[iface] = ' '.join(knownPrereqs)
5826
5827     INPUT.close()
5828
5829
5830 #############################################################################
5831 # Function    : ReadArgsFile
5832 # Description : This reads in an existing file which contains information on
5833 #                all GTK args. It creates the arrays @ArgObjects, @ArgNames,
5834 #                @ArgTypes, @ArgFlags, @ArgNicks and @ArgBlurbs containing info
5835 #               on the args.
5836 # Arguments   : $file - the file containing the arg information.
5837 #############################################################################
5838
5839 def ReadArgsFile(ifile):
5840     in_arg = False
5841     arg_object = None
5842     arg_name = None
5843     arg_type = None
5844     arg_flags = None
5845     arg_nick = None
5846     arg_blurb = None
5847     arg_default = None
5848     arg_range = None
5849
5850     # Reset the args info.
5851     global ArgObjects, ArgNames, ArgTypes, ArgFlags, ArgNicks, ArgBlurbs, ArgDefaults, ArgRanges
5852     ArgObjects = []
5853     ArgNames = []
5854     ArgTypes = []
5855     ArgFlags = []
5856     ArgNicks = []
5857     ArgBlurbs = []
5858     ArgDefaults = []
5859     ArgRanges = []
5860
5861     if not os.path.isfile(ifile):
5862         return
5863
5864     INPUT = open(ifile)
5865     line_number = 0
5866     for line in INPUT:
5867         line_number += 1
5868         if not in_arg:
5869             if re.search(r'^<ARG>', line):
5870                 in_arg = True
5871                 arg_object = ''
5872                 arg_name = ''
5873                 arg_type = ''
5874                 arg_flags = ''
5875                 arg_nick = ''
5876                 arg_blurb = ''
5877                 arg_default = ''
5878                 arg_range = ''
5879
5880         else:
5881             m1 = re.search(r'^<NAME>(.*)<\/NAME>', line)
5882             m2 = re.search(r'^<TYPE>(.*)<\/TYPE>', line)
5883             m3 = re.search(r'^<RANGE>(.*)<\/RANGE>', line)
5884             m4 = re.search(r'^<FLAGS>(.*)<\/FLAGS>', line)
5885             m5 = re.search(r'^<NICK>(.*)<\/NICK>', line)
5886             m6 = re.search(r'^<BLURB>(.*)<\/BLURB>', line)
5887             m7 = re.search(r'^<DEFAULT>(.*)<\/DEFAULT>', line)
5888             if m1:
5889                 arg_name = m1.group(1)
5890                 m1_1 = re.search(r'^(.*)::(.*)$', arg_name)
5891                 if m1_1:
5892                     arg_object = m1_1.group(1)
5893                     arg_name = m1_1.group(2).replace('_', '-')
5894                     logging.info("Found arg: %s", arg_name)
5895                 else:
5896                     common.LogWarning(ifile, line_number, "Invalid argument name: " + arg_name)
5897
5898             elif m2:
5899                 arg_type = m2.group(1)
5900             elif m3:
5901                 arg_range = m3.group(1)
5902             elif m4:
5903                 arg_flags = m4.group(1)
5904             elif m5:
5905                 arg_nick = m5.group(1)
5906             elif m6:
5907                 arg_blurb = m6.group(1)
5908                 if arg_blurb == "(null)":
5909                     arg_blurb = ''
5910                     common.LogWarning(ifile, line_number, "Property %s:%s has no documentation." % (arg_object, arg_name))
5911
5912             elif m7:
5913                 arg_default = m7.group(1)
5914             elif re.search(r'^</ARG>', line):
5915                 logging.info("Found end of arg: %s::%s\n%s : %s\n", arg_object, arg_name, arg_type, arg_flags)
5916                 ArgObjects.append(arg_object)
5917                 ArgNames.append(arg_name)
5918                 ArgTypes.append(arg_type)
5919                 ArgRanges.append(arg_range)
5920                 ArgFlags.append(arg_flags)
5921                 ArgNicks.append(arg_nick)
5922                 ArgBlurbs.append(arg_blurb)
5923                 ArgDefaults.append(arg_default)
5924                 in_arg = False
5925
5926     INPUT.close()
5927
5928
5929 #############################################################################
5930 # Function    : AddTreeLineArt
5931 # Description : Add unicode lineart to a pre-indented string array and returns
5932 #               it as as multiline string.
5933 # Arguments   : @tree - array of indented strings.
5934 #############################################################################
5935
5936 def AddTreeLineArt(tree):
5937     # iterate bottom up over the tree
5938     for i in range(len(tree) - 1, -1, -1):
5939         # count leading spaces
5940         m = re.search(r'^([^<A-Za-z]*)', tree[i])
5941         tree[i] = m.group(0)
5942         indent = len(m.group(1))
5943         # replace with ╰───, if place of ╰ is not space insert ├
5944         if indent > 4:
5945             if tree[i][indent-4] == " ":
5946                 tree[i] = tree[i][:indent-4] + "--- " + tree[i][indent:]
5947             else:
5948                 tree[i] = tree[i][:indent-4] + "+-- " + tree[i][indent:]
5949
5950             # go lines up while space and insert |
5951             j = i - 1
5952             while j >= 0 and tree[j][indent-4] == ' ':
5953                 tree[j][indent] = '|'
5954                 j -= 1
5955
5956
5957     res = "\n".join(tree)
5958     # unicode chars for: ╰──
5959     res = re.sub(r'---', '<phrase role=\"lineart\">&#9584;&#9472;&#9472;</phrase>', res)
5960     # unicde chars for: ├──
5961     res = re.sub(r'\+--', '<phrase role=\"lineart\">&#9500;&#9472;&#9472;</phrase>', res)
5962     # unicode char for: │
5963     res = re.sub(r'\|', '<phrase role=\"lineart\">&#9474;</phrase>', res)
5964
5965     return res
5966
5967
5968
5969 #############################################################################
5970 # Function    : CheckIsObject
5971 # Description : Returns 1 if the given name is a GObject or a subclass.
5972 #                It uses the global @Objects array.
5973 #                Note that the @Objects array only contains classes in the
5974 #                current module and their ancestors - not all GObject classes.
5975 # Arguments   : $name - the name to check.
5976 #############################################################################
5977
5978 def CheckIsObject(name):
5979     root = ObjectRoots[name]
5980     # Let GBoxed pass as an object here to get -struct appended to the id
5981     # and prevent conflicts with sections.
5982     return root and root != 'GEnum' and root != 'GFlags'
5983
5984
5985
5986 #############################################################################
5987 # Function    : MakeReturnField
5988 # Description : Pads a string to $RETURN_TYPE_FIELD_WIDTH.
5989 # Arguments   : $str - the string to pad.
5990 #############################################################################
5991
5992 def MakeReturnField(s):
5993
5994     return str + (' ' * (RETURN_TYPE_FIELD_WIDTH - len(s)))
5995
5996
5997 #############################################################################
5998 # Function    : GetSymbolSourceFile
5999 # Description : Get the filename where the symbol docs where taken from.
6000 # Arguments   : $symbol - the symbol name
6001 #############################################################################
6002
6003 def GetSymbolSourceFile(symbol):
6004     if symbol in SourceSymbolSourceFile:
6005         return SourceSymbolSourceFile[symbol]
6006     if symbol in SymbolSourceFile:
6007         return SymbolSourceFile[symbol]
6008     return ''
6009
6010 #############################################################################
6011 # Function    : GetSymbolSourceLine
6012 # Description : Get the file line where the symbol docs where taken from.
6013 # Arguments   : $symbol - the symbol name
6014 #############################################################################
6015
6016 def GetSymbolSourceLine(symbol):
6017     if symbol in SourceSymbolSourceLine:
6018         return SourceSymbolSourceLine[symbol]
6019     if symbol in SymbolSourceLine[symbol]:
6020         return SymbolSourceLine[symbol]
6021     return 0
6022
6023
6024 if __name__ == '__main__':
6025     Run()