gtkdoc/scan.py

   1 # -*- python -*-
   2 #
   3 # gtk-doc - GTK DocBook documentation generator.
   4 # Copyright (C) 1998  Damon Chaplin
   5 #               2007-2016  Stefan Sauer
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  20 #
  21
  22 """
  23 Extracts declarations of functions, macros, enums, structs and unions from
  24 header files.
  25
  26 It is called with a module name, an optional source directory, an optional
  27 output directory, and the header files to scan.
  28
  29 It outputs all declarations found to a file named '$MODULE-decl.txt', and the
  30 list of decarations to another file '$MODULE-decl-list.txt'.
  31
  32 This second list file is typically copied to '$MODULE-sections.txt' and
  33 organized into sections ready to output the XML pages.
  34 """
  35
  36 from __future__ import print_function
  37 from six import iteritems, iterkeys
  38
  39 import logging
  40 import os
  41 import re
  42 import shutil
  43
  44 from . import common
  45
  46 # do not read files twice; checking it here permits to give both srcdir and
  47 # builddir as --source-dir without fear of duplicities
  48 seen_headers = {}
  49
  50
  51 def Run(options):
  52     # logging.basicConfig(level=logging.INFO)
  53
  54     if not os.path.isdir(options.output_dir):
  55         os.mkdir(options.output_dir)
  56
  57     base_filename = os.path.join(options.output_dir, options.module)
  58     old_decl_list = base_filename + '-decl-list.txt'
  59     new_decl_list = base_filename + '-decl-list.new'
  60     old_decl = base_filename + '-decl.txt'
  61     new_decl = base_filename + '-decl.new'
  62     old_types = base_filename + '.types'
  63     new_types = base_filename + '.types.new'
  64     sections_file = base_filename + '-sections.txt'
  65
  66     # If this is the very first run then we create the .types file automatically.
  67     if not os.path.exists(sections_file) and not os.path.exists(old_types):
  68         options.rebuild_types = True
  69
  70     section_list = {}
  71     decl_list = []
  72     get_types = []
  73
  74     for file in options.headers:
  75         ScanHeader(file, section_list, decl_list, get_types, options)
  76
  77     for dir in options.source_dir:
  78         ScanHeaders(dir, section_list, decl_list, get_types, options)
  79
  80     with common.open_text(new_decl_list, 'w') as f:
  81         for section in sorted(iterkeys(section_list)):
  82             f.write(section_list[section])
  83
  84     with common.open_text(new_decl, 'w') as f:
  85         for decl in decl_list:
  86             f.write(decl)
  87
  88     if options.rebuild_types:
  89         with common.open_text(new_types, 'w') as f:
  90             for func in sorted(get_types):
  91                 f.write(func + '\n')
  92
  93         # remove the file if empty
  94         if len(get_types) == 0:
  95             os.unlink(new_types)
  96             if os.path.exists(old_types):
  97                 os.rename(old_types, old_types + '.bak')
  98         else:
  99             common.UpdateFileIfChanged(old_types, new_types, True)
 100
 101     common.UpdateFileIfChanged(old_decl_list, new_decl_list, True)
 102     common.UpdateFileIfChanged(old_decl, new_decl, True)
 103
 104     # If there is no MODULE-sections.txt file yet or we are asked to rebuild it,
 105     # we copy the MODULE-decl-list.txt file into its place. The user can tweak it
 106     # later if they want.
 107     if options.rebuild_sections or not os.path.exists(sections_file):
 108         new_sections_file = base_filename + '-sections.new'
 109         shutil.copyfile(old_decl_list, new_sections_file)
 110         common.UpdateFileIfChanged(sections_file, new_sections_file, False)
 111
 112     # If there is no MODULE-overrides.txt file we create an empty one
 113     # because EXTRA_DIST in gtk-doc.make requires it.
 114     overrides_file = base_filename + '-overrides.txt'
 115     if not os.path.exists(overrides_file):
 116         open(overrides_file, 'w').close()
 117
 118
 119 #
 120 # Function    : ScanHeaders
 121 # Description : This scans a directory tree looking for header files.
 122 #
 123 # Arguments   : $source_dir - the directory to scan.
 124 #               $section_list - a reference to the hashmap of sections.
 125 #
 126
 127 def ScanHeaders(source_dir, section_list, decl_list, get_types, options):
 128     logging.info('Scanning source directory: %s', source_dir)
 129
 130     # This array holds any subdirectories found.
 131     subdirs = []
 132
 133     for file in sorted(os.listdir(source_dir)):
 134         if file.startswith('.'):
 135             continue
 136         fullname = os.path.join(source_dir, file)
 137         if os.path.isdir(fullname):
 138             subdirs.append(file)
 139         elif file.endswith('.h'):
 140             ScanHeader(fullname, section_list, decl_list, get_types, options)
 141
 142     # Now recursively scan the subdirectories.
 143     for dir in subdirs:
 144         matchstr = r'(\s|^)' + re.escape(dir) + r'(\s|$)'
 145         if re.search(matchstr, options.ignore_headers):
 146             continue
 147         ScanHeaders(os.path.join(source_dir, dir), section_list, decl_list,
 148                     get_types, options)
 149
 150
 151 #
 152 # Function    : ScanHeader
 153 # Description : This scans a header file, looking for declarations of
 154 #                functions, macros, typedefs, structs and unions, which it
 155 #                outputs to the decl_list.
 156 # Arguments   : $input_file - the header file to scan.
 157 #               $section_list - a map of sections.
 158 #               $decl_list - a list of declarations
 159 # Returns     : it adds declarations to the appropriate list.
 160 #
 161
 162 def ScanHeader(input_file, section_list, decl_list, get_types, options):
 163     global seen_headers
 164     slist = []                 # Holds the resulting list of declarations.
 165     title = ''                 # Holds the title of the section
 166     in_comment = 0             # True if we are in a comment.
 167     in_declaration = ''        # The type of declaration we are in, e.g.
 168                                #   'function' or 'macro'.
 169     skip_block = 0             # True if we should skip a block.
 170     symbol = None              # The current symbol being declared.
 171     decl = ''                  # Holds the declaration of the current symbol.
 172     ret_type = None            # For functions and function typedefs this
 173                                #   holds the function's return type.
 174     pre_previous_line = ''     # The pre-previous line read in - some Gnome
 175                                #   functions have the return type on one
 176                                #   line, the function name on the next,
 177                                #   and the rest of the declaration after.
 178     previous_line = ''         # The previous line read in - some Gnome
 179                                #   functions have the return type on one line
 180                                #   and the rest of the declaration after.
 181     first_macro = 1            # Used to try to skip the standard #ifdef XXX
 182                                # define XXX at the start of headers.
 183     level = None               # Used to handle structs/unions which contain
 184                                #   nested structs or unions.
 185     internal = 0               # Set to 1 for internal symbols, we need to
 186                                #   fully parse, but don't add them to docs
 187     forward_decls = {}         # Dict of forward declarations, we skip
 188                                #   them if we find the real declaration
 189                                #   later.
 190     doc_comments = {}          # Dict of doc-comments we found.
 191                                # The key is lowercase symbol name, val=1.
 192
 193     file_basename = None
 194
 195     deprecated_conditional_nest = 0
 196     ignore_conditional_nest = 0
 197
 198     deprecated = ''
 199     doc_comment = ''
 200
 201     # Don't scan headers twice
 202     canonical_input_file = os.path.realpath(input_file)
 203     if canonical_input_file in seen_headers:
 204         logging.info('File already scanned: %s', input_file)
 205         return
 206
 207     seen_headers[canonical_input_file] = 1
 208
 209     file_basename = os.path.split(input_file)[1][:-2]  # filename ends in .h
 210
 211     # Check if the basename is in the list of headers to ignore.
 212     matchstr = r'(\s|^)' + re.escape(file_basename) + r'\.h(\s|$)'
 213     if re.search(matchstr, options.ignore_headers):
 214         logging.info('File ignored: %s', input_file)
 215         return
 216
 217     # Check if the full name is in the list of headers to ignore.
 218     matchstr = r'(\s|^)' + re.escape(input_file) + r'(\s|$)'
 219     if re.search(matchstr, options.ignore_headers):
 220         logging.info('File ignored: %s', input_file)
 221         return
 222
 223     if not os.path.exists(input_file):
 224         logging.warning('File does not exist: %s', input_file)
 225         return
 226
 227     logging.info('Scanning %s', input_file)
 228
 229     for line in common.open_text(input_file):
 230         # If this is a private header, skip it.
 231         if re.search(r'%^\s*/\*\s*<\s*private_header\s*>\s*\*/', line):
 232             return
 233
 234         # Skip to the end of the current comment.
 235         if in_comment:
 236             logging.info('Comment: %s', line.strip())
 237             doc_comment += line
 238             if re.search(r'\*/', line):
 239                 m = re.search(r'\* ([a-zA-Z][a-zA-Z0-9_]+):', doc_comment)
 240                 if m:
 241                     doc_comments[m.group(1).lower()] = 1
 242                 in_comment = 0
 243                 doc_comment = ''
 244             continue
 245
 246         # Keep a count of #if, #ifdef, #ifndef nesting,
 247         # and if we enter a deprecation-symbol-bracketed
 248         # zone, take note.
 249         m = re.search(r'^\s*#\s*if(?:n?def\b|\s+!?\s*defined\s*\()\s*(\w+)', line)
 250         if m:
 251             define_name = m.group(1)
 252             if deprecated_conditional_nest < 1 and re.search(options.deprecated_guards, define_name):
 253                 deprecated_conditional_nest = 1
 254             elif deprecated_conditional_nest >= 1:
 255                 deprecated_conditional_nest += 1
 256             if ignore_conditional_nest == 0 and '__GTK_DOC_IGNORE__' in define_name:
 257                 ignore_conditional_nest = 1
 258             elif ignore_conditional_nest > 0:
 259                 ignore_conditional_nest = 1
 260
 261         elif re.search(r'^\s*#\sif', line):
 262             if deprecated_conditional_nest >= 1:
 263                 deprecated_conditional_nest += 1
 264
 265             if ignore_conditional_nest > 0:
 266                 ignore_conditional_nest += 1
 267         elif re.search(r'^\s*#endif', line):
 268             if deprecated_conditional_nest >= 1:
 269                 deprecated_conditional_nest -= 1
 270
 271             if ignore_conditional_nest > 0:
 272                 ignore_conditional_nest -= 1
 273
 274         # If we find a line containing _DEPRECATED, we hope that this is
 275         # attribute based deprecation and also treat this as a deprecation
 276         # guard, unless it's a macro definition.
 277         if deprecated_conditional_nest == 0 and '_DEPRECATED' in line:
 278             m = re.search(r'^\s*#\s*(if*|define)', line)
 279             if not (m or in_declaration == 'enum'):
 280                 logging.info('Found deprecation annotation (decl: "%s"): "%s"',
 281                              in_declaration, line.strip())
 282                 deprecated_conditional_nest += 0.1
 283
 284         # set flag that is used later when we do AddSymbolToList
 285         if deprecated_conditional_nest > 0:
 286             deprecated = '<DEPRECATED/>\n'
 287         else:
 288             deprecated = ''
 289
 290         if ignore_conditional_nest:
 291             continue
 292
 293         if not in_declaration:
 294             # Skip top-level comments.
 295             m = re.search(r'^\s*/\*', line)
 296             if m:
 297                 re.sub(r'^\s*/\*', '', line)
 298                 if re.search(r'\*/', line):
 299                     logging.info('Found one-line comment: %s', line.strip())
 300                 else:
 301                     in_comment = 1
 302                     doc_comment = line
 303                     logging.info('Found start of comment: %s', line.strip())
 304                 continue
 305
 306             logging.info('no decl: %s', line.strip())
 307
 308             # avoid generating regex with |'' (matching no string)
 309             ignore_decorators = ''
 310             if options.ignore_decorators:
 311                 ignore_decorators = '|' + options.ignore_decorators
 312
 313             m = re.search(r'^\s*#\s*define\s+(\w+)', line)
 314             #                   $1                                $3            $4             $5
 315             m2 = re.search(
 316                 r'^\s*typedef\s+((const\s+|G_CONST_RETURN\s+)?\w+)(\s+const)?\s*(\**)\s*\(\*\s*(\w+)\)\s*\(', line)
 317             #                    $1                                $3            $4             $5
 318             m3 = re.search(r'^\s*((const\s+|G_CONST_RETURN\s+)?\w+)(\s+const)?\s*(\**)\s*\(\*\s*(\w+)\)\s*\(', line)
 319             #                    $1            $2
 320             m4 = re.search(r'^\s*(\**)\s*\(\*\s*(\w+)\)\s*\(', line)
 321             #                              $1                                $3
 322             m5 = re.search(r'^\s*typedef\s*((const\s+|G_CONST_RETURN\s+)?\w+)(\s+const)?\s*', previous_line)
 323             #                                              $1                                $3            $4             $5
 324             m6 = re.search(
 325                 r'^\s*(?:\b(?:extern|G_INLINE_FUNC%s)\s*)*((const\s+|G_CONST_RETURN\s+)?\w+)(\s+const)?\s*(\**)\s*\(\*\s*(\w+)\)\s*\(' % ignore_decorators, line)
 326             m7 = re.search(r'^\s*enum\s+_?(\w+)\s+\{', line)
 327             m8 = re.search(r'^\s*typedef\s+enum', line)
 328             m9 = re.search(r'^\s*typedef\s+(struct|union)\s+_(\w+)\s+\2\s*;', line)
 329             m10 = re.search(r'^\s*(struct|union)\s+(\w+)\s*;', line)
 330             m11 = re.search(r'^\s*typedef\s+(struct|union)\s*\w*\s*{', line)
 331             m12 = re.search(r'^\s*typedef\s+(?:struct|union)\s+\w+[\s\*]+(\w+)\s*;', line)
 332             m13 = re.search(r'^\s*(G_GNUC_EXTENSION\s+)?typedef\s+(.+[\s\*])(\w+)(\s*\[[^\]]+\])*\s*;', line)
 333             m14 = re.search(
 334                 r'^\s*(extern|[A-Za-z_]+VAR%s)\s+((const\s+|signed\s+|unsigned\s+|long\s+|short\s+)*\w+)(\s+\*+|\*+|\s)\s*(const\s+)*([A-Za-z]\w*)\s*;' % ignore_decorators, line)
 335             m15 = re.search(
 336                 r'^\s*((const\s+|signed\s+|unsigned\s+|long\s+|short\s+)*\w+)(\s+\*+|\*+|\s)\s*(const\s+)*([A-Za-z]\w*)\s*\=', line)
 337             m16 = re.search(r'.*G_DECLARE_(FINAL_TYPE|DERIVABLE_TYPE|INTERFACE)\s*\(', line)
 338             #                                             $1                                                                                                    $2                                                     $3
 339             m17 = re.search(
 340                 r'^\s*(?:\b(?:extern|G_INLINE_FUNC%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|long\s+|short\s+|struct\s+|union\s+|enum\s+)*\w+)([\s*]+(?:\s*(?:\*+|\bconst\b|\bG_CONST_RETURN\b))*)\s*(_[A-Za-z]\w*)\s*\(' % ignore_decorators, line)
 341             #                                             $1                                                                                                    $2                                                     $3
 342             m18 = re.search(
 343                 r'^\s*(?:\b(?:extern|G_INLINE_FUNC%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|long\s+|short\s+|struct\s+|union\s+|enum\s+)*\w+)([\s*]+(?:\s*(?:\*+|\bconst\b|\bG_CONST_RETURN\b))*)\s*([A-Za-z]\w*)\s*\(' % ignore_decorators, line)
 344             m19 = re.search(r'^\s*([A-Za-z]\w*)\s*\(', line)
 345             m20 = re.search(r'^\s*\(', line)
 346             m21 = re.search(r'^\s*struct\s+_?(\w+)', line)
 347             m22 = re.search(r'^\s*union\s+_(\w+)', line)
 348
 349             # MACROS
 350
 351             if m:
 352                 symbol = m.group(1)
 353                 decl = line
 354                 # We assume all macros which start with '_' are private.
 355                 # We also try to skip the first macro if it looks like the
 356                 # standard #ifndef HEADER_FILE #define HEADER_FILE etc.
 357                 # And we only want TRUE & FALSE defined in GLib.
 358                 if not symbol.startswith('_') \
 359                     and (not re.search(r'#ifndef\s+' + symbol, previous_line)
 360                          or first_macro == 0) \
 361                     and ((symbol != 'TRUE' and symbol != 'FALSE')
 362                          or options.module == 'glib'):
 363                     in_declaration = 'macro'
 364                     logging.info('Macro: "%s"', symbol)
 365                 else:
 366                     logging.info('skipping Macro: "%s"', symbol)
 367                     in_declaration = 'macro'
 368                     internal = 1
 369                 first_macro = 0
 370
 371             # TYPEDEF'D FUNCTIONS (i.e. user functions)
 372             elif m2:
 373                 p3 = m2.group(3) or ''
 374                 ret_type = "%s%s %s" % (m2.group(1), p3, m2.group(4))
 375                 symbol = m2.group(5)
 376                 decl = line[m2.end():]
 377                 in_declaration = 'user_function'
 378                 logging.info('user function (1): "%s", Returns: "%s"', symbol, ret_type)
 379
 380             elif re.search(r'^\s*typedef\s*', previous_line) and m3:
 381                 p3 = m3.group(3) or ''
 382                 ret_type = '%s%s %s' % (m3.group(1), p3, m3.group(4))
 383                 symbol = m3.group(5)
 384                 decl = line[m3.end():]
 385                 in_declaration = 'user_function'
 386                 logging.info('user function (2): "%s", Returns: "%s"', symbol, ret_type)
 387
 388             elif re.search(r'^\s*typedef\s*', previous_line) and m4:
 389                 ret_type = m4.group(1)
 390                 symbol = m4.group(2)
 391                 decl = line[m4.end():]
 392                 if m5:
 393                     p3 = m5.group(3) or ''
 394                     ret_type = "%s%s %s" % (m5.group(1), p3, ret_type)
 395                     in_declaration = 'user_function'
 396                     logging.info('user function (3): "%s", Returns: "%s"', symbol, ret_type)
 397
 398             # FUNCTION POINTER VARIABLES
 399             elif m6:
 400                 p3 = m6.group(3) or ''
 401                 ret_type = '%s%s %s' % (m6.group(1), p3, m6.group(4))
 402                 symbol = m6.group(5)
 403                 decl = line[m6.end():]
 404                 in_declaration = 'user_function'
 405                 logging.info('function pointer variable: "%s", Returns: "%s"', symbol, ret_type)
 406
 407             # ENUMS
 408
 409             elif m7:
 410                 re.sub(r'^\s*enum\s+_?(\w+)\s+\{', r'enum \1 {', line)
 411                 # We assume that 'enum _<enum_name> {' is really the
 412                 # declaration of enum <enum_name>.
 413                 symbol = m7.group(1)
 414                 decl = line
 415                 in_declaration = 'enum'
 416                 logging.info('plain enum: "%s"', symbol)
 417
 418             elif re.search(r'^\s*typedef\s+enum\s+_?(\w+)\s+\1\s*;', line):
 419                 # We skip 'typedef enum <enum_name> _<enum_name>;' as the enum will
 420                 # be declared elsewhere.
 421                 logging.info('skipping enum typedef: "%s"', line)
 422             elif m8:
 423                 symbol = ''
 424                 decl = line
 425                 in_declaration = 'enum'
 426                 logging.info('typedef enum: -')
 427
 428             # STRUCTS AND UNIONS
 429
 430             elif m9:
 431                 # We've found a 'typedef struct _<name> <name>;'
 432                 # This could be an opaque data structure, so we output an
 433                 # empty declaration. If the structure is actually found that
 434                 # will override this.
 435                 structsym = m9.group(1).upper()
 436                 logging.info('%s typedef: "%s"', structsym, m9.group(2))
 437                 forward_decls[m9.group(2)] = '<%s>\n<NAME>%s</NAME>\n%s</%s>\n' % (
 438                     structsym, m9.group(2), deprecated, structsym)
 439
 440             elif re.search(r'^\s*(?:struct|union)\s+_(\w+)\s*;', line):
 441                 # Skip private structs/unions.
 442                 logging.info('private struct/union')
 443
 444             elif m10:
 445                 # Do a similar thing for normal structs as for typedefs above.
 446                 # But we output the declaration as well in this case, so we
 447                 # can differentiate it from a typedef.
 448                 structsym = m10.group(1).upper()
 449                 logging.info('%s:%s', structsym, m10.group(2))
 450                 forward_decls[m10.group(2)] = '<%s>\n<NAME>%s</NAME>\n%s%s</%s>\n' % (
 451                     structsym, m10.group(2), line, deprecated, structsym)
 452
 453             elif m11:
 454                 symbol = ''
 455                 decl = line
 456                 level = 0
 457                 in_declaration = m11.group(1)
 458                 logging.info('typedef struct/union "%s"', in_declaration)
 459
 460             # OTHER TYPEDEFS
 461
 462             elif m12:
 463                 logging.info('Found struct/union(*) typedef "%s": "%s"', m12.group(1), line)
 464                 if AddSymbolToList(slist, m12.group(1)):
 465                     decl_list.append('<TYPEDEF>\n<NAME>%s</NAME>\n%s%s</TYPEDEF>\n' % (m12.group(1), deprecated, line))
 466
 467             elif m13:
 468                 if m13.group(2).split()[0] not in ('struct', 'union'):
 469                     logging.info('Found typedef: "%s"', line)
 470                     if AddSymbolToList(slist, m13.group(3)):
 471                         decl_list.append(
 472                             '<TYPEDEF>\n<NAME>%s</NAME>\n%s%s</TYPEDEF>\n' % (m13.group(3), deprecated, line))
 473             elif re.search(r'^\s*typedef\s+', line):
 474                 logging.info('Skipping typedef: "%s"', line)
 475
 476             # VARIABLES (extern'ed variables)
 477
 478             elif m14:
 479                 symbol = m14.group(6)
 480                 line = re.sub(r'^\s*([A-Za-z_]+VAR)\b', r'extern', line)
 481                 decl = line
 482                 logging.info('Possible extern var "%s": "%s"', symbol, decl)
 483                 if AddSymbolToList(slist, symbol):
 484                     decl_list.append('<VARIABLE>\n<NAME>%s</NAME>\n%s%s</VARIABLE>\n' % (symbol, deprecated, decl))
 485
 486             # VARIABLES
 487
 488             elif m15:
 489                 symbol = m15.group(5)
 490                 decl = line
 491                 logging.info('Possible global var" %s": "%s"', symbol, decl)
 492                 if AddSymbolToList(slist, symbol):
 493                     decl_list.append('<VARIABLE>\n<NAME>%s</NAME>\n%s%s</VARIABLE>\n' % (symbol, deprecated, decl))
 494
 495             # G_DECLARE_*
 496
 497             elif m16:
 498                 in_declaration = 'g-declare'
 499                 symbol = 'G_DECLARE_' + m16.group(1)
 500                 decl = line[m16.end():]
 501
 502             # FUNCTIONS
 503
 504             # We assume that functions which start with '_' are private, so
 505             # we skip them.
 506             elif m17:
 507                 ret_type = m17.group(1)
 508                 if m17.group(2):
 509                     ret_type += ' ' + m17.group(2)
 510                 symbol = m17.group(3)
 511                 decl = line[m17.end():]
 512                 logging.info('internal Function: "%s", Returns: "%s""%s"', symbol, m17.group(1), m17.group(2))
 513                 in_declaration = 'function'
 514                 internal = 1
 515                 if line.strip().startswith('G_INLINE_FUNC'):
 516                     logging.info('skip block after inline function')
 517                     # now we we need to skip a whole { } block
 518                     skip_block = 1
 519
 520             elif m18:
 521                 ret_type = m18.group(1)
 522                 if m18.group(2):
 523                     ret_type += ' ' + m18.group(2)
 524                 symbol = m18.group(3)
 525                 decl = line[m18.end():]
 526                 logging.info('Function (1): "%s", Returns: "%s""%s"', symbol, m18.group(1), m18.group(2))
 527                 in_declaration = 'function'
 528                 if line.strip().startswith('G_INLINE_FUNC'):
 529                     logging.info('skip block after inline function')
 530                     # now we we need to skip a whole { } block
 531                     skip_block = 1
 532
 533             # Try to catch function declarations which have the return type on
 534             # the previous line. But we don't want to catch complete functions
 535             # which have been declared G_INLINE_FUNC, e.g. g_bit_nth_lsf in
 536             # glib, or 'static inline' functions.
 537             elif m19:
 538                 symbol = m19.group(1)
 539                 decl = line[m19.end():]
 540
 541                 previous_line_words = previous_line.strip().split()
 542
 543                 if not previous_line.strip().startswith('G_INLINE_FUNC'):
 544                     if not previous_line_words or previous_line_words[0] != 'static':
 545                         #                                           $1                                                                                                   $2
 546                         pm = re.search(r'^\s*(?:\b(?:extern%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|long\s+|short\s+|struct\s+|union\s+|enum\s+)*\w+)((?:\s*(?:\*+|\bconst\b|\bG_CONST_RETURN\b))*)\s*$' %
 547                                        ignore_decorators, previous_line)
 548                         if pm:
 549                             ret_type = pm.group(1)
 550                             if pm.group(2):
 551                                 ret_type += ' ' + pm.group(2)
 552                             logging.info('Function  (2): "%s", Returns: "%s"', symbol, ret_type)
 553                             in_declaration = 'function'
 554                     else:
 555                         logging.info('skip block after inline function')
 556                         # now we we need to skip a whole { } block
 557                         skip_block = 1
 558                         #                                                        $1                                                                                                    $2
 559                         pm = re.search(r'^\s*(?:\b(?:extern|static|inline%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|long\s+|short\s+|struct\s+|union\s+|enum\s+)*\w+)((?:\s*(?:\*+|\bconst\b|\bG_CONST_RETURN\b))*)\s*$' %
 560                                        ignore_decorators, previous_line)
 561                         if pm:
 562                             ret_type = pm.group(1)
 563                             if pm.group(2):
 564                                 ret_type += ' ' + pm.group(2)
 565                             logging.info('Function  (3): "%s", Returns: "%s"', symbol, ret_type)
 566                             in_declaration = 'function'
 567                 else:
 568                     if not previous_line_words or previous_line_words[0] != 'static':
 569                         logging.info('skip block after inline function')
 570                         # now we we need to skip a whole { } block
 571                         skip_block = 1
 572                         #                                                         $1                                                                                                    $2
 573                         pm = re.search(r'^\s*(?:\b(?:extern|G_INLINE_FUNC%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|long\s+|short\s+|struct\s+|union\s+|enum\s+)*\w+)((?:\s*(?:\*+|\bconst\b|\bG_CONST_RETURN\b))*)\s*$' %
 574                                        ignore_decorators, previous_line)
 575                         if pm:
 576                             ret_type = pm.group(1)
 577                             if pm.group(2):
 578                                 ret_type += ' ' + pm.group(2)
 579                             logging.info('Function (4): "%s", Returns: "%s"', symbol, ret_type)
 580                             in_declaration = 'function'
 581
 582             # Try to catch function declarations with the return type and name
 583             # on the previous line(s), and the start of the parameters on this.
 584             elif m20:
 585                 decl = line[m20.end():]
 586                 pm = re.search(
 587                     r'^\s*(?:\b(?:extern|G_INLINE_FUNC%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|enum\s+)*\w+)(\s+\*+|\*+|\s)\s*([A-Za-z]\w*)\s*$' % ignore_decorators, previous_line)
 588                 ppm = re.search(r'^\s*(?:\b(?:extern|G_INLINE_FUNC%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|struct\s+|union\s+|enum\s+)*\w+(?:\**\s+\**(?:const|G_CONST_RETURN))?(?:\s+|\s*\*+))\s*$' %
 589                                 ignore_decorators, pre_previous_line)
 590                 if pm:
 591                     ret_type = pm.group(1) + ' ' + pm.group(2)
 592                     symbol = pm.group(3)
 593                     in_declaration = 'function'
 594                     logging.info('Function (5): "%s", Returns: "%s"', symbol, ret_type)
 595
 596                 elif re.search(r'^\s*\w+\s*$', previous_line) and ppm:
 597                     ret_type = ppm.group(1)
 598                     ret_type = re.sub(r'\s*\n', '', ret_type, flags=re.MULTILINE)
 599                     in_declaration = 'function'
 600
 601                     symbol = previous_line
 602                     symbol = re.sub(r'^\s+', '', symbol)
 603                     symbol = re.sub(r'\s*\n', '', symbol, flags=re.MULTILINE)
 604                     logging.info('Function (6): "%s", Returns: "%s"', symbol, ret_type)
 605
 606             #} elsif (m/^extern\s+/) {
 607                 # print "DEBUG: Skipping extern: $_"
 608
 609             # STRUCTS
 610             elif re.search(r'^\s*struct\s+_?(\w+)\s*\*', line):
 611                 # Skip 'struct _<struct_name> *', since it could be a
 612                 # return type on its own line.
 613                 pass
 614             elif m21:
 615                 # We assume that 'struct _<struct_name>' is really the
 616                 # declaration of struct <struct_name>.
 617                 symbol = m21.group(1)
 618                 decl = line
 619                 # we will find the correct level as below we do $level += tr/{//
 620                 level = 0
 621                 in_declaration = 'struct'
 622                 logging.info('Struct(_): "%s"', symbol)
 623
 624             # UNIONS
 625             elif re.search(r'^\s*union\s+_(\w+)\s*\*', line):
 626                 # Skip 'union _<union_name> *' (see above)
 627                 pass
 628             elif m22:
 629                 symbol = m22.group(1)
 630                 decl = line
 631                 level = 0
 632                 in_declaration = 'union'
 633                 logging.info('Union(_): "%s"', symbol)
 634         else:
 635             logging.info('in decl: skip=%s %s', skip_block, line.strip())
 636             # If we were already in the middle of a declaration, we simply add
 637             # the current line onto the end of it.
 638             if skip_block == 0:
 639                 decl += line
 640             else:
 641                 # Remove all nested pairs of curly braces.
 642                 brace_remover = r'{[^{]*}'
 643                 bm = re.search(brace_remover, line)
 644                 while bm:
 645                     line = re.sub(brace_remover, '', line)
 646                     bm = re.search(brace_remover, line)
 647                 # Then hope at most one remains in the line...
 648                 bm = re.search(r'(.*?){', line)
 649                 if bm:
 650                     if skip_block == 1:
 651                         decl += bm.group(1)
 652                     skip_block += 1
 653                 elif '}' in line:
 654                     skip_block -= 1
 655                     if skip_block == 1:
 656                         # this is a hack to detect the end of declaration
 657                         decl += ';'
 658                         skip_block = 0
 659                         logging.info('2: ---')
 660
 661                 else:
 662                     if skip_block == 1:
 663                         decl += line
 664
 665         if in_declaration == "g-declare":
 666             dm = re.search(r'\s*(\w+)\s*,\s*(\w+)\s*,\s*(\w+)\s*,\s*(\w+)\s*,\s*(\w+)\s*\).*$', decl)
 667             # FIXME the original code does s// stuff here and we don't. Is it necessary?
 668             if dm:
 669                 ModuleObjName = dm.group(1)
 670                 module_obj_name = dm.group(2)
 671                 if options.rebuild_types:
 672                     get_types.append(module_obj_name + '_get_type')
 673                 forward_decls[ModuleObjName] = '<STRUCT>\n<NAME>%s</NAME>\n%s</STRUCT>\n' % (ModuleObjName, deprecated)
 674                 if symbol.startswith('G_DECLARE_DERIVABLE'):
 675                     forward_decls[ModuleObjName + 'Class'] = '<STRUCT>\n<NAME>%sClass</NAME>\n%s</STRUCT>\n' % (
 676                         ModuleObjName, deprecated)
 677                 if symbol.startswith('G_DECLARE_INTERFACE'):
 678                     forward_decls[ModuleObjName + 'Interface'] = '<STRUCT>\n<NAME>%sInterface</NAME>\n%s</STRUCT>\n' % (
 679                         ModuleObjName, deprecated)
 680                 in_declaration = ''
 681
 682         # Note that sometimes functions end in ') G_GNUC_PRINTF (2, 3);' or
 683         # ') __attribute__ (...);'.
 684         if in_declaration == 'function':
 685             regex = r'\)\s*(G_GNUC_.*|.*DEPRECATED.*%s\s*|__attribute__\s*\(.*\)\s*)*;.*$' % ignore_decorators
 686             pm = re.search(regex, decl, flags=re.MULTILINE)
 687             if pm:
 688                 logging.info('scrubbing:[%s]', decl.strip())
 689                 decl = re.sub(regex, '', decl, flags=re.MULTILINE)
 690                 logging.info('scrubbed:[%s]', decl.strip())
 691                 if internal == 0:
 692                     decl = re.sub(r'/\*.*?\*/', '', decl, flags=re.MULTILINE)   # remove comments.
 693                     decl = re.sub(r'\s*\n\s*(?!$)', ' ', decl, flags=re.MULTILINE)
 694                                   # consolidate whitespace at start/end of lines.
 695                     decl = decl.strip()
 696                     ret_type = re.sub(r'/\*.*?\*/', '', ret_type)               # remove comments in ret type.
 697                     if AddSymbolToList(slist, symbol):
 698                         decl_list.append('<FUNCTION>\n<NAME>%s</NAME>\n%s<RETURNS>%s</RETURNS>\n%s\n</FUNCTION>\n' %
 699                                          (symbol, deprecated, ret_type, decl))
 700                         if options.rebuild_types:
 701                             # check if this looks like a get_type function and if so remember
 702                             if symbol.endswith('_get_type') and 'GType' in ret_type and re.search(r'^(void|)$', decl):
 703                                 logging.info(
 704                                     "Adding get-type: [%s] [%s] [%s]\tfrom %s", ret_type, symbol, decl, input_file)
 705                                 get_types.append(symbol)
 706                 else:
 707                     internal = 0
 708                 deprecated_conditional_nest = int(deprecated_conditional_nest)
 709                 in_declaration = ''
 710                 skip_block = 0
 711
 712         if in_declaration == 'user_function':
 713             if re.search(r'\).*$', decl):
 714                 decl = re.sub(r'\).*$', '', decl)
 715                 if AddSymbolToList(slist, symbol):
 716                     decl_list.append('<USER_FUNCTION>\n<NAME>%s</NAME>\n%s<RETURNS>%s</RETURNS>\n%s</USER_FUNCTION>\n' %
 717                                      (symbol, deprecated, ret_type, decl))
 718                 deprecated_conditional_nest = int(deprecated_conditional_nest)
 719                 in_declaration = ''
 720
 721         if in_declaration == 'macro':
 722             if not re.search(r'\\\s*$', decl):
 723                 if internal == 0:
 724                     if AddSymbolToList(slist, symbol):
 725                         decl_list.append('<MACRO>\n<NAME>%s</NAME>\n%s%s</MACRO>\n' % (symbol, deprecated, decl))
 726                 else:
 727                     internal = 0
 728                 deprecated_conditional_nest = int(deprecated_conditional_nest)
 729                 in_declaration = ''
 730
 731         if in_declaration == 'enum':
 732             em = re.search(r'\}\s*(\w+)?;\s*$', decl)
 733             if em:
 734                 if symbol == '':
 735                     symbol = em.group(1)
 736                 if AddSymbolToList(slist, symbol):
 737                     decl_list.append('<ENUM>\n<NAME>%s</NAME>\n%s%s</ENUM>\n' % (symbol, deprecated, decl))
 738                 deprecated_conditional_nest = int(deprecated_conditional_nest)
 739                 in_declaration = ''
 740
 741         # We try to handle nested stucts/unions, but unmatched brackets in
 742         # comments will cause problems.
 743         if in_declaration == 'struct' or in_declaration == 'union':
 744             sm = re.search(r'\n\}\s*(\w*);\s*$', decl)
 745             if level <= 1 and sm:
 746                 if symbol == '':
 747                     symbol = sm.group(1)
 748
 749                 bm = re.search(r'^(\S+)(Class|Iface|Interface)\b', symbol)
 750                 if bm:
 751                     objectname = bm.group(1)
 752                     logging.info('Found object: "%s"', objectname)
 753                     title = '<TITLE>%s</TITLE>' % objectname
 754
 755                 logging.info('Store struct: "%s"', symbol)
 756                 if AddSymbolToList(slist, symbol):
 757                     structsym = in_declaration.upper()
 758                     decl_list.append('<%s>\n<NAME>%s</NAME>\n%s%s</%s>\n' %
 759                                      (structsym, symbol, deprecated, decl, structsym))
 760                     if symbol in forward_decls:
 761                         del forward_decls[symbol]
 762                 deprecated_conditional_nest = int(deprecated_conditional_nest)
 763                 in_declaration = ''
 764             else:
 765                 # We use tr to count the brackets in the line, and adjust
 766                 # $level accordingly.
 767                 level += line.count('{')
 768                 level -= line.count('}')
 769                 logging.info('struct/union level : %d', level)
 770
 771         pre_previous_line = previous_line
 772         previous_line = line
 773
 774     # print remaining forward declarations
 775     for symbol in sorted(iterkeys(forward_decls)):
 776         if forward_decls[symbol]:
 777             AddSymbolToList(slist, symbol)
 778             decl_list.append(forward_decls[symbol])
 779
 780     # add title
 781     slist = [title] + slist
 782
 783     logging.info("Scanning %s done", input_file)
 784
 785     # Try to separate the standard macros and functions, placing them at the
 786     # end of the current section, in a subsection named 'Standard'.
 787     # do this in a loop to catch object, enums and flags
 788     klass = lclass = prefix = lprefix = None
 789     standard_decl = []
 790     liststr = '\n'.join(s for s in slist if s) + '\n'
 791     while True:
 792         m = re.search(r'^(\S+)_IS_(\S*)_CLASS\n', liststr, flags=re.MULTILINE)
 793         m2 = re.search(r'^(\S+)_IS_(\S*)\n', liststr, flags=re.MULTILINE)
 794         m3 = re.search(r'^(\S+?)_(\S*)_get_type\n', liststr, flags=re.MULTILINE)
 795         if m:
 796             prefix = m.group(1)
 797             lprefix = prefix.lower()
 798             klass = m.group(2)
 799             lclass = klass.lower()
 800             logging.info("Found gobject type '%s_%s' from is_class macro", prefix, klass)
 801         elif m2:
 802             prefix = m2.group(1)
 803             lprefix = prefix.lower()
 804             klass = m2.group(2)
 805             lclass = klass.lower()
 806             logging.info("Found gobject type '%s_%s' from is_ macro", prefix, klass)
 807         elif m3:
 808             lprefix = m3.group(1)
 809             prefix = lprefix.upper()
 810             lclass = m3.group(2)
 811             klass = lclass.upper()
 812             logging.info("Found gobject type '%s_%s' from get_type function", prefix, klass)
 813         else:
 814             break
 815
 816         cclass = lclass
 817         cclass = cclass.replace('_', '')
 818         mtype = lprefix + cclass
 819
 820         liststr, standard_decl = replace_once(liststr, standard_decl, r'^%sPrivate\n' % mtype)
 821
 822         # We only leave XxYy* in the normal section if they have docs
 823         if mtype not in doc_comments:
 824             logging.info("  Hide instance docs for %s", mtype)
 825             liststr, standard_decl = replace_once(liststr, standard_decl, r'^%s\n' % mtype)
 826
 827         if mtype + 'class' not in doc_comments:
 828             logging.info("  Hide class docs for %s", mtype)
 829             liststr, standard_decl = replace_once(liststr, standard_decl, r'^%sClass\n' % mtype)
 830
 831         if mtype + 'interface' not in doc_comments:
 832             logging.info("  Hide iface docs for %s", mtype)
 833             liststr, standard_decl = replace_once(liststr, standard_decl, r'%sInterface\n' % mtype)
 834
 835         if mtype + 'iface' not in doc_comments:
 836             logging.info("  Hide iface docs for " + mtype)
 837             liststr, standard_decl = replace_once(liststr, standard_decl, r'%sIface\n' % mtype)
 838
 839         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_IS_%s\n' % klass)
 840         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_TYPE_%s\n' % klass)
 841         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_%s_get_type\n' % lclass)
 842         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_%s_CLASS\n' % klass)
 843         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_IS_%s_CLASS\n' % klass)
 844         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_%s_GET_CLASS\n' % klass)
 845         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_%s_GET_IFACE\n' % klass)
 846         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_%s_GET_INTERFACE\n' % klass)
 847         # We do this one last, otherwise it tends to be caught by the IS_$class macro
 848         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_%s\n' % klass)
 849
 850     logging.info('Decl:%s---', liststr)
 851     logging.info('Std :%s---', ''.join(sorted(standard_decl)))
 852     if len(standard_decl):
 853         # sort the symbols
 854         liststr += '<SUBSECTION Standard>\n' + ''.join(sorted(standard_decl))
 855
 856     if liststr != '':
 857         if file_basename not in section_list:
 858             section_list[file_basename] = ''
 859         section_list[file_basename] += "<SECTION>\n<FILE>%s</FILE>\n%s</SECTION>\n\n" % (file_basename, liststr)
 860
 861
 862 def replace_once(liststr, standard_decl, regex):
 863     mre = re.search(regex, liststr,  flags=re.IGNORECASE | re.MULTILINE)
 864     if mre:
 865         standard_decl.append(mre.group(0))
 866         liststr = re.sub(regex, '', liststr, flags=re.IGNORECASE | re.MULTILINE)
 867     return liststr, standard_decl
 868
 869
 870 def replace_all(liststr, standard_decl, regex):
 871     mre = re.search(regex, liststr, flags=re.MULTILINE)
 872     while mre:
 873         standard_decl.append(mre.group(0))
 874         liststr = re.sub(regex, '', liststr, flags=re.MULTILINE)
 875         mre = re.search(regex, liststr, flags=re.MULTILINE)
 876     return liststr, standard_decl
 877
 878
 879 def AddSymbolToList(slist, symbol):
 880     """ Adds symbol to list of declaration if not already present.
 881
 882     Args:
 883         slist: The list of symbols.
 884         symbol: The symbol to add to the list.
 885     """
 886     if symbol in slist:
 887          # logging.info('Symbol %s already in list. skipping', symbol)
 888          # we return False to skip outputting another entry to -decl.txt
 889          # this is to avoid redeclarations (e.g. in conditional sections).
 890         return False
 891     slist.append(symbol)
 892     return True