gtkdoc/scan.py

   1 # -*- python -*-
   2 #
   3 # gtk-doc - GTK DocBook documentation generator.
   4 # Copyright (C) 1998  Damon Chaplin
   5 #               2007-2016  Stefan Sauer
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  20 #
  21
  22 """
  23 Extracts declarations of functions, macros, enums, structs and unions from
  24 header files.
  25
  26 It is called with a module name, an optional source directory, an optional
  27 output directory, and the header files to scan.
  28
  29 It outputs all declarations found to a file named '$MODULE-decl.txt', and the
  30 list of decarations to another file '$MODULE-decl-list.txt'.
  31
  32 This second list file is typically copied to '$MODULE-sections.txt' and
  33 organized into sections ready to output the XML pages.
  34 """
  35
  36 from __future__ import print_function
  37 from six import iteritems, iterkeys
  38
  39 import logging
  40 import os
  41 import re
  42
  43 from . import common
  44
  45 # do not read files twice; checking it here permits to give both srcdir and
  46 # builddir as --source-dir without fear of duplicities
  47 seen_headers = {}
  48
  49
  50 def Run(options):
  51     # logging.basicConfig(level=logging.INFO)
  52
  53     if not os.path.isdir(options.output_dir):
  54         os.mkdir(options.output_dir)
  55
  56     base_filename = os.path.join(options.output_dir, options.module)
  57     old_decl_list = base_filename + '-decl-list.txt'
  58     new_decl_list = base_filename + '-decl-list.new'
  59     old_decl = base_filename + '-decl.txt'
  60     new_decl = base_filename + '-decl.new'
  61     old_types = base_filename + '.types'
  62     new_types = base_filename + '.types.new'
  63     sections_file = base_filename + '-sections.txt'
  64
  65     # If this is the very first run then we create the .types file automatically.
  66     if not os.path.exists(sections_file) and not os.path.exists(old_types):
  67         options.rebuild_types = True
  68
  69     section_list = {}
  70     decl_list = []
  71     get_types = []
  72
  73     for file in options.headers:
  74         ScanHeader(file, section_list, decl_list, get_types, options)
  75
  76     for dir in options.source_dir:
  77         ScanHeaders(dir, section_list, decl_list, get_types, options)
  78
  79     with open(new_decl_list, 'w') as f:
  80         for section in sorted(iterkeys(section_list)):
  81             f.write(section_list[section])
  82
  83     with open(new_decl, 'w') as f:
  84         for decl in decl_list:
  85             f.write(decl)
  86
  87     if options.rebuild_types:
  88         with open(new_types, 'w') as f:
  89             for func in sorted(get_types):
  90                 f.write(func + '\n')
  91
  92         # remove the file if empty
  93         if len(get_types) == 0:
  94             os.unlink(new_types)
  95             if os.path.exists(old_types):
  96                 os.rename(old_types, old_types + '.bak')
  97         else:
  98             common.UpdateFileIfChanged(old_types, new_types, True)
  99
 100     common.UpdateFileIfChanged(old_decl_list, new_decl_list, True)
 101     common.UpdateFileIfChanged(old_decl, new_decl, True)
 102
 103     # If there is no MODULE-sections.txt file yet or we are asked to rebuild it,
 104     # we copy the MODULE-decl-list.txt file into its place. The user can tweak it
 105     # later if they want.
 106     if options.rebuild_sections or not os.path.exists(sections_file):
 107         common.UpdateFileIfChanged(sections_file, old_decl_list, False)
 108
 109     # If there is no MODULE-overrides.txt file we create an empty one
 110     # because EXTRA_DIST in gtk-doc.make requires it.
 111     overrides_file = base_filename + '-overrides.txt'
 112     if not os.path.exists(overrides_file):
 113         open(overrides_file, 'w').close()
 114
 115
 116 #
 117 # Function    : ScanHeaders
 118 # Description : This scans a directory tree looking for header files.
 119 #
 120 # Arguments   : $source_dir - the directory to scan.
 121 #               $section_list - a reference to the hashmap of sections.
 122 #
 123
 124 def ScanHeaders(source_dir, section_list, decl_list, get_types, options):
 125     logging.info('Scanning source directory: %s', source_dir)
 126
 127     # This array holds any subdirectories found.
 128     subdirs = []
 129
 130     for file in sorted(os.listdir(source_dir)):
 131         if file.startswith('.'):
 132             continue
 133         fullname = os.path.join(source_dir, file)
 134         if os.path.isdir(fullname):
 135             subdirs.append(file)
 136         elif file.endswith('.h'):
 137             ScanHeader(fullname, section_list, decl_list, get_types, options)
 138
 139     # Now recursively scan the subdirectories.
 140     for dir in subdirs:
 141         matchstr = r'(\s|^)' + re.escape(dir) + r'(\s|$)'
 142         if re.search(matchstr, options.ignore_headers):
 143             continue
 144         ScanHeaders(os.path.join(source_dir, dir), section_list, decl_list,
 145                     get_types, options)
 146
 147
 148 #
 149 # Function    : ScanHeader
 150 # Description : This scans a header file, looking for declarations of
 151 #                functions, macros, typedefs, structs and unions, which it
 152 #                outputs to the decl_list.
 153 # Arguments   : $input_file - the header file to scan.
 154 #               $section_list - a map of sections.
 155 #               $decl_list - a list of declarations
 156 # Returns     : it adds declarations to the appropriate list.
 157 #
 158
 159 def ScanHeader(input_file, section_list, decl_list, get_types, options):
 160     global seen_headers
 161     slist = []                  # Holds the resulting list of declarations.
 162     title = ''                 # Holds the title of the section
 163     in_comment = 0                  # True if we are in a comment.
 164     in_declaration = ''          # The type of declaration we are in, e.g.
 165                               #   'function' or 'macro'.
 166     skip_block = 0                  # True if we should skip a block.
 167     symbol = None                  # The current symbol being declared.
 168     decl = ''                          # Holds the declaration of the current symbol.
 169     ret_type = None                  # For functions and function typedefs this
 170                               #   holds the function's return type.
 171     pre_previous_line = ''   # The pre-previous line read in - some Gnome
 172                               #   functions have the return type on one
 173                               #   line, the function name on the next,
 174                               #   and the rest of the declaration after.
 175     previous_line = ''          # The previous line read in - some Gnome
 176                               #   functions have the return type on one line
 177                               #   and the rest of the declaration after.
 178     first_macro = 1          # Used to try to skip the standard #ifdef XXX
 179                               # define XXX at the start of headers.
 180     level = None                          # Used to handle structs/unions which contain
 181                               #   nested structs or unions.
 182     internal = 0             # Set to 1 for internal symbols, we need to
 183                                 #   fully parse, but don't add them to docs
 184     forward_decls = {}         # hashtable of forward declarations, we skip
 185                                 #   them if we find the real declaration
 186                                 #   later.
 187     doc_comments = {}          # hastable of doc-comment we found. We can
 188                                 # use that to put element to the right
 189                                 # sction in the generated section-file
 190
 191     file_basename = None
 192
 193     deprecated_conditional_nest = 0
 194     ignore_conditional_nest = 0
 195
 196     deprecated = ''
 197     doc_comment = ''
 198
 199     # Don't scan headers twice
 200     canonical_input_file = os.path.realpath(input_file)
 201     if canonical_input_file in seen_headers:
 202         logging.info('File already scanned: %s', input_file)
 203         return
 204
 205     seen_headers[canonical_input_file] = 1
 206
 207     file_basename = os.path.split(input_file)[1][:-2]  # filename ends in .h
 208
 209     # Check if the basename is in the list of headers to ignore.
 210     matchstr = r'(\s|^)' + re.escape(file_basename) + r'\.h(\s|$)'
 211     if re.search(matchstr, options.ignore_headers):
 212         logging.info('File ignored: %s', input_file)
 213         return
 214
 215     # Check if the full name is in the list of headers to ignore.
 216     matchstr = r'(\s|^)' + re.escape(input_file) + r'(\s|$)'
 217     if re.search(matchstr, options.ignore_headers):
 218         logging.info('File ignored: %s', input_file)
 219         return
 220
 221     if not os.path.exists(input_file):
 222         logging.warning('File does not exist: %s', input_file)
 223         return
 224
 225     logging.info('Scanning %s', input_file)
 226
 227     for line in open(input_file):
 228         # If this is a private header, skip it.
 229         if re.search(r'%^\s*/\*\s*<\s*private_header\s*>\s*\*/', line):
 230             return
 231
 232         # Skip to the end of the current comment.
 233         if in_comment:
 234             logging.info('Comment: %s', line)
 235             doc_comment += line
 236             if re.search(r'\*/', line):
 237                 m = re.search(r'\* ([a-zA-Z][a-zA-Z0-9_]+):/', doc_comment)
 238                 if m:
 239                     doc_comments[m.group(1).lower()] = 1
 240                 in_comment = 0
 241                 doc_comment = ''
 242             continue
 243
 244         # Keep a count of #if, #ifdef, #ifndef nesting,
 245         # and if we enter a deprecation-symbol-bracketed
 246         # zone, take note.
 247         m = re.search(r'^\s*#\s*if(?:n?def\b|\s+!?\s*defined\s*\()\s*(\w+)', line)
 248         if m:
 249             define_name = m.group(1)
 250             if deprecated_conditional_nest < 1 and re.search(options.deprecated_guards, define_name):
 251                 deprecated_conditional_nest = 1
 252             elif deprecated_conditional_nest >= 1:
 253                 deprecated_conditional_nest += 1
 254             if ignore_conditional_nest == 0 and '__GTK_DOC_IGNORE__' in define_name:
 255                 ignore_conditional_nest = 1
 256             elif ignore_conditional_nest > 0:
 257                 ignore_conditional_nest = 1
 258
 259         elif re.search(r'^\s*#\sif', line):
 260             if deprecated_conditional_nest >= 1:
 261                 deprecated_conditional_nest += 1
 262
 263             if ignore_conditional_nest > 0:
 264                 ignore_conditional_nest += 1
 265         elif re.search(r'^\s*#endif', line):
 266             if deprecated_conditional_nest >= 1:
 267                 deprecated_conditional_nest -= 1
 268
 269             if ignore_conditional_nest > 0:
 270                 ignore_conditional_nest -= 1
 271
 272         # If we find a line containing _DEPRECATED, we hope that this is
 273         # attribute based deprecation and also treat this as a deprecation
 274         # guard, unless it's a macro definition.
 275         if deprecated_conditional_nest == 0 and '_DEPRECATED' in line:
 276             m = re.search(r'^\s*#\s*(if*|define)', line)
 277             if not (m or in_declaration == 'enum'):
 278                 logging.info('Found deprecation annotation (decl: "%s"): "%s"', in_declaration, line)
 279                 deprecated_conditional_nest += 0.1
 280
 281         # set flag that is used later when we do AddSymbolToList
 282         if deprecated_conditional_nest > 0:
 283             deprecated = '<DEPRECATED/>\n'
 284         else:
 285             deprecated = ''
 286
 287         if ignore_conditional_nest:
 288             continue
 289
 290         if not in_declaration:
 291             # Skip top-level comments.
 292             m = re.search(r'^\s*/\*', line)
 293             if m:
 294                 re.sub(r'^\s*/\*', '', line)
 295                 if re.search(r'\*/', line):
 296                     logging.info('Found one-line comment: %s', line)
 297                 else:
 298                     in_comment = 1
 299                     doc_comment = line
 300                     logging.info('Found start of comment: %s', line)
 301                 continue
 302
 303             logging.info('no decl: %s', line.strip())
 304
 305             # avoid generating regex with |'' (matching no string)
 306             ignore_decorators = ''
 307             if options.ignore_decorators:
 308                 ignore_decorators = '|' + options.ignore_decorators
 309
 310             m = re.search(r'^\s*#\s*define\s+(\w+)', line)
 311             #                   $1                                $3            $4             $5
 312             m2 = re.search(
 313                 r'^\s*typedef\s+((const\s+|G_CONST_RETURN\s+)?\w+)(\s+const)?\s*(\**)\s*\(\*\s*(\w+)\)\s*\(', line)
 314             #                    $1                                $3            $4             $5
 315             m3 = re.search(r'^\s*((const\s+|G_CONST_RETURN\s+)?\w+)(\s+const)?\s*(\**)\s*\(\*\s*(\w+)\)\s*\(', line)
 316             #                    $1            $2
 317             m4 = re.search(r'^\s*(\**)\s*\(\*\s*(\w+)\)\s*\(', line)
 318             #                              $1                                $3
 319             m5 = re.search(r'^\s*typedef\s*((const\s+|G_CONST_RETURN\s+)?\w+)(\s+const)?\s*', previous_line)
 320             #                                              $1                                $3            $4             $5
 321             m6 = re.search(
 322                 r'^\s*(?:\b(?:extern|G_INLINE_FUNC%s)\s*)*((const\s+|G_CONST_RETURN\s+)?\w+)(\s+const)?\s*(\**)\s*\(\*\s*(\w+)\)\s*\(' % ignore_decorators, line)
 323             m7 = re.search(r'^\s*enum\s+_?(\w+)\s+\{', line)
 324             m8 = re.search(r'^\s*typedef\s+enum', line)
 325             m9 = re.search(r'^\s*typedef\s+(struct|union)\s+_(\w+)\s+\2\s*;', line)
 326             m10 = re.search(r'^\s*(struct|union)\s+(\w+)\s*;', line)
 327             m11 = re.search(r'^\s*typedef\s+(struct|union)\s*\w*\s*{', line)
 328             m12 = re.search(r'^\s*typedef\s+(?:struct|union)\s+\w+[\s\*]+(\w+)\s*;', line)
 329             m13 = re.search(r'^\s*(G_GNUC_EXTENSION\s+)?typedef\s+(.+[\s\*])(\w+)(\s*\[[^\]]+\])*\s*;', line)
 330             m14 = re.search(
 331                 r'^\s*(extern|[A-Za-z_]+VAR%s)\s+((const\s+|signed\s+|unsigned\s+|long\s+|short\s+)*\w+)(\s+\*+|\*+|\s)\s*(const\s+)*([A-Za-z]\w*)\s*;' % ignore_decorators, line)
 332             m15 = re.search(
 333                 r'^\s*((const\s+|signed\s+|unsigned\s+|long\s+|short\s+)*\w+)(\s+\*+|\*+|\s)\s*(const\s+)*([A-Za-z]\w*)\s*\=', line)
 334             m16 = re.search(r'.*G_DECLARE_(FINAL_TYPE|DERIVABLE_TYPE|INTERFACE)\s*\(', line)
 335             #                                             $1                                                                                                    $2                                                     $3
 336             m17 = re.search(
 337                 r'^\s*(?:\b(?:extern|G_INLINE_FUNC%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|long\s+|short\s+|struct\s+|union\s+|enum\s+)*\w+)([\s*]+(?:\s*(?:\*+|\bconst\b|\bG_CONST_RETURN\b))*)\s*(_[A-Za-z]\w*)\s*\(' % ignore_decorators, line)
 338             #                                             $1                                                                                                    $2                                                     $3
 339             m18 = re.search(
 340                 r'^\s*(?:\b(?:extern|G_INLINE_FUNC%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|long\s+|short\s+|struct\s+|union\s+|enum\s+)*\w+)([\s*]+(?:\s*(?:\*+|\bconst\b|\bG_CONST_RETURN\b))*)\s*([A-Za-z]\w*)\s*\(' % ignore_decorators, line)
 341             m19 = re.search(r'^\s*([A-Za-z]\w*)\s*\(', line)
 342             m20 = re.search(r'^\s*\(', line)
 343             m21 = re.search(r'^\s*struct\s+_?(\w+)', line)
 344             m22 = re.search(r'^\s*union\s+_(\w+)', line)
 345
 346             # MACROS
 347
 348             if m:
 349                 symbol = m.group(1)
 350                 decl = line
 351                 # We assume all macros which start with '_' are private.
 352                 # We also try to skip the first macro if it looks like the
 353                 # standard #ifndef HEADER_FILE #define HEADER_FILE etc.
 354                 # And we only want TRUE & FALSE defined in GLib.
 355                 if not symbol.startswith('_') \
 356                     and (not re.search(r'#ifndef\s+' + symbol, previous_line)
 357                          or first_macro == 0) \
 358                     and ((symbol != 'TRUE' and symbol != 'FALSE')
 359                          or options.module == 'glib'):
 360                     in_declaration = 'macro'
 361                     logging.info('Macro: "%s"', symbol)
 362                 else:
 363                     logging.info('skipping Macro: "%s"', symbol)
 364                     in_declaration = 'macro'
 365                     internal = 1
 366                 first_macro = 0
 367
 368             # TYPEDEF'D FUNCTIONS (i.e. user functions)
 369             elif m2:
 370                 p3 = m2.group(3) or ''
 371                 ret_type = "%s%s %s" % (m2.group(1), p3, m2.group(4))
 372                 symbol = m2.group(5)
 373                 decl = line[m2.end():]
 374                 in_declaration = 'user_function'
 375                 logging.info('user function (1): "%s", Returns: "%s"', symbol, ret_type)
 376
 377             elif re.search(r'^\s*typedef\s*', previous_line) and m3:
 378                 p3 = m3.group(3) or ''
 379                 ret_type = '%s%s %s' % (m3.group(1), p3, m3.group(4))
 380                 symbol = m3.group(5)
 381                 decl = line[m3.end():]
 382                 in_declaration = 'user_function'
 383                 logging.info('user function (2): "%s", Returns: "%s"', symbol, ret_type)
 384
 385             elif re.search(r'^\s*typedef\s*', previous_line) and m4:
 386                 ret_type = m4.group(1)
 387                 symbol = m4.group(2)
 388                 decl = line[m4.end():]
 389                 if m5:
 390                     p3 = m5.group(3) or ''
 391                     ret_type = "%s%s %s" % (m5.group(1), p3, ret_type)
 392                     in_declaration = 'user_function'
 393                     logging.info('user function (3): "%s", Returns: "%s"', symbol, ret_type)
 394
 395             # FUNCTION POINTER VARIABLES
 396             elif m6:
 397                 p3 = m6.group(3) or ''
 398                 ret_type = '%s%s %s' % (m6.group(1), p3, m6.group(4))
 399                 symbol = m6.group(5)
 400                 decl = line[m6.end():]
 401                 in_declaration = 'user_function'
 402                 logging.info('function pointer variable: "%s", Returns: "%s"', symbol, ret_type)
 403
 404             # ENUMS
 405
 406             elif m7:
 407                 re.sub(r'^\s*enum\s+_?(\w+)\s+\{', r'enum \1 {', line)
 408                 # We assume that 'enum _<enum_name> {' is really the
 409                 # declaration of enum <enum_name>.
 410                 symbol = m7.group(1)
 411                 decl = line
 412                 in_declaration = 'enum'
 413                 logging.info('plain enum: "%s"', symbol)
 414
 415             elif re.search(r'^\s*typedef\s+enum\s+_?(\w+)\s+\1\s*;', line):
 416                 # We skip 'typedef enum <enum_name> _<enum_name>;' as the enum will
 417                 # be declared elsewhere.
 418                 logging.info('skipping enum typedef: "%s"', line)
 419             elif m8:
 420                 symbol = ''
 421                 decl = line
 422                 in_declaration = 'enum'
 423                 logging.info('typedef enum: -')
 424
 425             # STRUCTS AND UNIONS
 426
 427             elif m9:
 428                 # We've found a 'typedef struct _<name> <name>;'
 429                 # This could be an opaque data structure, so we output an
 430                 # empty declaration. If the structure is actually found that
 431                 # will override this.
 432                 structsym = m9.group(1).upper()
 433                 logging.info('%s typedef: "%s"', structsym, m9.group(2))
 434                 forward_decls[m9.group(2)] = '<%s>\n<NAME>%s</NAME>\n%s</%s>\n' % (
 435                     structsym, m9.group(2), deprecated, structsym)
 436
 437             elif re.search(r'^\s*(?:struct|union)\s+_(\w+)\s*;', line):
 438                 # Skip private structs/unions.
 439                 logging.info('private struct/union')
 440
 441             elif m10:
 442                 # Do a similar thing for normal structs as for typedefs above.
 443                 # But we output the declaration as well in this case, so we
 444                 # can differentiate it from a typedef.
 445                 structsym = m10.group(1).upper()
 446                 logging.info('%s:%s', structsym, m10.group(2))
 447                 forward_decls[m10.group(2)] = '<%s>\n<NAME>%s</NAME>\n%s%s</%s>\n' % (
 448                     structsym, m10.group(2), line, deprecated, structsym)
 449
 450             elif m11:
 451                 symbol = ''
 452                 decl = line
 453                 level = 0
 454                 in_declaration = m11.group(1)
 455                 logging.info('typedef struct/union "%s"', in_declaration)
 456
 457             # OTHER TYPEDEFS
 458
 459             elif m12:
 460                 logging.info('Found struct/union(*) typedef "%s": "%s"', m12.group(1), line)
 461                 if AddSymbolToList(slist, m12.group(1)):
 462                     decl_list.append('<TYPEDEF>\n<NAME>%s</NAME>\n%s%s</TYPEDEF>\n' % (m12.group(1), deprecated, line))
 463
 464             elif m13:
 465                 if m13.group(2).split()[0] not in ('struct', 'union'):
 466                     logging.info('Found typedef: "%s"', line)
 467                     if AddSymbolToList(slist, m13.group(3)):
 468                         decl_list.append(
 469                             '<TYPEDEF>\n<NAME>%s</NAME>\n%s%s</TYPEDEF>\n' % (m13.group(3), deprecated, line))
 470             elif re.search(r'^\s*typedef\s+', line):
 471                 logging.info('Skipping typedef: "%s"', line)
 472
 473             # VARIABLES (extern'ed variables)
 474
 475             elif m14:
 476                 symbol = m14.group(6)
 477                 line = re.sub(r'^\s*([A-Za-z_]+VAR)\b', r'extern', line)
 478                 decl = line
 479                 logging.info('Possible extern var "%s": "%s"', symbol, decl)
 480                 if AddSymbolToList(slist, symbol):
 481                     decl_list.append('<VARIABLE>\n<NAME>%s</NAME>\n%s%s</VARIABLE>\n' % (symbol, deprecated, decl))
 482
 483             # VARIABLES
 484
 485             elif m15:
 486                 symbol = m15.group(5)
 487                 decl = line
 488                 logging.info('Possible global var" %s": "%s"', symbol, decl)
 489                 if AddSymbolToList(slist, symbol):
 490                     decl_list.append('<VARIABLE>\n<NAME>%s</NAME>\n%s%s</VARIABLE>\n' % (symbol, deprecated, decl))
 491
 492             # G_DECLARE_*
 493
 494             elif m16:
 495                 in_declaration = 'g-declare'
 496                 symbol = 'G_DECLARE_' + m16.group(1)
 497                 decl = line[m16.end():]
 498
 499             # FUNCTIONS
 500
 501             # We assume that functions which start with '_' are private, so
 502             # we skip them.
 503             elif m17:
 504                 ret_type = m17.group(1)
 505                 if m17.group(2):
 506                     ret_type += ' ' + m17.group(2)
 507                 symbol = m17.group(3)
 508                 decl = line[m17.end():]
 509                 logging.info('internal Function: "%s", Returns: "%s""%s"', symbol, m17.group(1), m17.group(2))
 510                 in_declaration = 'function'
 511                 internal = 1
 512                 if line.strip().startswith('G_INLINE_FUNC'):
 513                     logging.info('skip block after inline function')
 514                     # now we we need to skip a whole { } block
 515                     skip_block = 1
 516
 517             elif m18:
 518                 ret_type = m18.group(1)
 519                 if m18.group(2):
 520                     ret_type += ' ' + m18.group(2)
 521                 symbol = m18.group(3)
 522                 decl = line[m18.end():]
 523                 logging.info('Function (1): "%s", Returns: "%s""%s"', symbol, m18.group(1), m18.group(2))
 524                 in_declaration = 'function'
 525                 if line.strip().startswith('G_INLINE_FUNC'):
 526                     logging.info('skip block after inline function')
 527                     # now we we need to skip a whole { } block
 528                     skip_block = 1
 529
 530             # Try to catch function declarations which have the return type on
 531             # the previous line. But we don't want to catch complete functions
 532             # which have been declared G_INLINE_FUNC, e.g. g_bit_nth_lsf in
 533             # glib, or 'static inline' functions.
 534             elif m19:
 535                 symbol = m19.group(1)
 536                 decl = line[m19.end():]
 537
 538                 previous_line_words = previous_line.strip().split()
 539
 540                 if not previous_line.strip().startswith('G_INLINE_FUNC'):
 541                     if not previous_line_words or previous_line_words[0] != 'static':
 542                         #                                           $1                                                                                                   $2
 543                         pm = re.search(r'^\s*(?:\b(?:extern%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|long\s+|short\s+|struct\s+|union\s+|enum\s+)*\w+)((?:\s*(?:\*+|\bconst\b|\bG_CONST_RETURN\b))*)\s*$' %
 544                                        ignore_decorators, previous_line)
 545                         if pm:
 546                             ret_type = pm.group(1)
 547                             if pm.group(2):
 548                                 ret_type += ' ' + pm.group(2)
 549                             logging.info('Function  (2): "%s", Returns: "%s"', symbol, ret_type)
 550                             in_declaration = 'function'
 551                     else:
 552                         logging.info('skip block after inline function')
 553                         # now we we need to skip a whole { } block
 554                         skip_block = 1
 555                         #                                                        $1                                                                                                    $2
 556                         pm = re.search(r'^\s*(?:\b(?:extern|static|inline%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|long\s+|short\s+|struct\s+|union\s+|enum\s+)*\w+)((?:\s*(?:\*+|\bconst\b|\bG_CONST_RETURN\b))*)\s*$' %
 557                                        ignore_decorators, previous_line)
 558                         if pm:
 559                             ret_type = pm.group(1)
 560                             if pm.group(2):
 561                                 ret_type += ' ' + pm.group(2)
 562                             logging.info('Function  (3): "%s", Returns: "%s"', symbol, ret_type)
 563                             in_declaration = 'function'
 564                 else:
 565                     if not previous_line_words or previous_line_words[0] != 'static':
 566                         logging.info('skip block after inline function')
 567                         # now we we need to skip a whole { } block
 568                         skip_block = 1
 569                         #                                                         $1                                                                                                    $2
 570                         pm = re.search(r'^\s*(?:\b(?:extern|G_INLINE_FUNC%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|long\s+|short\s+|struct\s+|union\s+|enum\s+)*\w+)((?:\s*(?:\*+|\bconst\b|\bG_CONST_RETURN\b))*)\s*$' %
 571                                        ignore_decorators, previous_line)
 572                         if pm:
 573                             ret_type = pm.group(1)
 574                             if pm.group(2):
 575                                 ret_type += ' ' + pm.group(2)
 576                             logging.info('Function (4): "%s", Returns: "%s"', symbol, ret_type)
 577                             in_declaration = 'function'
 578
 579             # Try to catch function declarations with the return type and name
 580             # on the previous line(s), and the start of the parameters on this.
 581             elif m20:
 582                 decl = line[m20.end():]
 583                 pm = re.search(
 584                     r'^\s*(?:\b(?:extern|G_INLINE_FUNC%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|enum\s+)*\w+)(\s+\*+|\*+|\s)\s*([A-Za-z]\w*)\s*$' % ignore_decorators, previous_line)
 585                 ppm = re.search(r'^\s*(?:\b(?:extern|G_INLINE_FUNC%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|struct\s+|union\s+|enum\s+)*\w+(?:\**\s+\**(?:const|G_CONST_RETURN))?(?:\s+|\s*\*+))\s*$' %
 586                                 ignore_decorators, pre_previous_line)
 587                 if pm:
 588                     ret_type = pm.group(1) + ' ' + pm.group(2)
 589                     symbol = pm.group(3)
 590                     in_declaration = 'function'
 591                     logging.info('Function (5): "%s", Returns: "%s"', symbol, ret_type)
 592
 593                 elif re.search(r'^\s*\w+\s*$', previous_line) and ppm:
 594                     ret_type = ppm.group(1)
 595                     ret_type = re.sub(r'\s*\n', '', ret_type, flags=re.MULTILINE)
 596                     in_declaration = 'function'
 597
 598                     symbol = previous_line
 599                     symbol = re.sub(r'^\s+', '', symbol)
 600                     symbol = re.sub(r'\s*\n', '', symbol, flags=re.MULTILINE)
 601                     logging.info('Function (6): "%s", Returns: "%s"', symbol, ret_type)
 602
 603             #} elsif (m/^extern\s+/) {
 604                 # print "DEBUG: Skipping extern: $_"
 605
 606             # STRUCTS
 607             elif re.search(r'^\s*struct\s+_?(\w+)\s*\*', line):
 608                 # Skip 'struct _<struct_name> *', since it could be a
 609                 # return type on its own line.
 610                 pass
 611             elif m21:
 612                 # We assume that 'struct _<struct_name>' is really the
 613                 # declaration of struct <struct_name>.
 614                 symbol = m21.group(1)
 615                 decl = line
 616                 # we will find the correct level as below we do $level += tr/{//
 617                 level = 0
 618                 in_declaration = 'struct'
 619                 logging.info('Struct(_): "%s"', symbol)
 620
 621             # UNIONS
 622             elif re.search(r'^\s*union\s+_(\w+)\s*\*', line):
 623                 # Skip 'union _<union_name> *' (see above)
 624                 pass
 625             elif m22:
 626                 symbol = m22.group(1)
 627                 decl = line
 628                 level = 0
 629                 in_declaration = 'union'
 630                 logging.info('Union(_): "%s"', symbol)
 631         else:
 632             logging.info('in decl: skip=%s %s', skip_block, line.strip())
 633             # If we were already in the middle of a declaration, we simply add
 634             # the current line onto the end of it.
 635             if skip_block == 0:
 636                 decl += line
 637             else:
 638                 # Remove all nested pairs of curly braces.
 639                 brace_remover = r'{[^{]*}'
 640                 bm = re.search(brace_remover, line)
 641                 while bm:
 642                     line = re.sub(brace_remover, '', line)
 643                     bm = re.search(brace_remover, line)
 644                 # Then hope at most one remains in the line...
 645                 bm = re.search(r'(.*?){', line)
 646                 if bm:
 647                     if skip_block == 1:
 648                         decl += bm.group(1)
 649                     skip_block += 1
 650                 elif '}' in line:
 651                     skip_block -= 1
 652                     if skip_block == 1:
 653                         # this is a hack to detect the end of declaration
 654                         decl += ';'
 655                         skip_block = 0
 656                         logging.info('2: ---')
 657
 658                 else:
 659                     if skip_block == 1:
 660                         decl += line
 661
 662         if in_declaration == "g-declare":
 663             dm = re.search(r'\s*(\w+)\s*,\s*(\w+)\s*,\s*(\w+)\s*,\s*(\w+)\s*,\s*(\w+)\s*\).*$', decl)
 664             # FIXME the original code does s// stuff here and we don't. Is it necessary?
 665             if dm:
 666                 ModuleObjName = dm.group(1)
 667                 module_obj_name = dm.group(2)
 668                 if options.rebuild_types:
 669                     get_types.append(module_obj_name + '_get_type')
 670                 forward_decls[ModuleObjName] = '<STRUCT>\n<NAME>%s</NAME>\n%s</STRUCT>\n' % (ModuleObjName, deprecated)
 671                 if symbol.startswith('G_DECLARE_DERIVABLE'):
 672                     forward_decls[ModuleObjName + 'Class'] = '<STRUCT>\n<NAME>%sClass</NAME>\n%s</STRUCT>\n' % (
 673                         ModuleObjName, deprecated)
 674                 if symbol.startswith('G_DECLARE_INTERFACE'):
 675                     forward_decls[ModuleObjName + 'Interface'] = '<STRUCT>\n<NAME>%sInterface</NAME>\n%s</STRUCT>\n' % (
 676                         ModuleObjName, deprecated)
 677                 in_declaration = ''
 678
 679         # Note that sometimes functions end in ') G_GNUC_PRINTF (2, 3);' or
 680         # ') __attribute__ (...);'.
 681         if in_declaration == 'function':
 682             regex = r'\)\s*(G_GNUC_.*|.*DEPRECATED.*%s\s*|__attribute__\s*\(.*\)\s*)*;.*$' % ignore_decorators
 683             pm = re.search(regex, decl, flags=re.MULTILINE)
 684             if pm:
 685                 logging.info('scrubbing:[%s]', decl)
 686                 decl = re.sub(regex, '', decl, flags=re.MULTILINE)
 687                 logging.info('scrubbed:[%s]', decl)
 688                 if internal == 0:
 689                     decl = re.sub(r'/\*.*?\*/', '', decl, flags=re.MULTILINE)   # remove comments.
 690                     decl = re.sub(r'\s*\n\s*(?!$)', ' ', decl, flags=re.MULTILINE)
 691                                   # consolidate whitespace at start/end of lines.
 692                     decl = decl.strip()
 693                     ret_type = re.sub(r'/\*.*?\*/', '', ret_type)               # remove comments in ret type.
 694                     if AddSymbolToList(slist, symbol):
 695                         decl_list.append('<FUNCTION>\n<NAME>%s</NAME>\n%s<RETURNS>%s</RETURNS>\n%s\n</FUNCTION>\n' %
 696                                          (symbol, deprecated, ret_type, decl))
 697                         if options.rebuild_types:
 698                             # check if this looks like a get_type function and if so remember
 699                             if symbol.endswith('_get_type') and 'GType' in ret_type and re.search(r'^(void|)$', decl):
 700                                 logging.info(
 701                                     "Adding get-type: [%s] [%s] [%s]\tfrom %s", ret_type, symbol, decl, input_file)
 702                                 get_types.append(symbol)
 703                 else:
 704                     internal = 0
 705                 deprecated_conditional_nest = int(deprecated_conditional_nest)
 706                 in_declaration = ''
 707                 skip_block = 0
 708
 709         if in_declaration == 'user_function':
 710             if re.search(r'\).*$', decl):
 711                 decl = re.sub(r'\).*$', '', decl)
 712                 if AddSymbolToList(slist, symbol):
 713                     decl_list.append('<USER_FUNCTION>\n<NAME>%s</NAME>\n%s<RETURNS>%s</RETURNS>\n%s</USER_FUNCTION>\n' %
 714                                      (symbol, deprecated, ret_type, decl))
 715                 deprecated_conditional_nest = int(deprecated_conditional_nest)
 716                 in_declaration = ''
 717
 718         if in_declaration == 'macro':
 719             if not re.search(r'\\\s*$', decl):
 720                 if internal == 0:
 721                     if AddSymbolToList(slist, symbol):
 722                         decl_list.append('<MACRO>\n<NAME>%s</NAME>\n%s%s</MACRO>\n' % (symbol, deprecated, decl))
 723                 else:
 724                     internal = 0
 725                 deprecated_conditional_nest = int(deprecated_conditional_nest)
 726                 in_declaration = ''
 727
 728         if in_declaration == 'enum':
 729             em = re.search(r'\}\s*(\w+)?;\s*$', decl)
 730             if em:
 731                 if symbol == '':
 732                     symbol = em.group(1)
 733                 if AddSymbolToList(slist, symbol):
 734                     decl_list.append('<ENUM>\n<NAME>%s</NAME>\n%s%s</ENUM>\n' % (symbol, deprecated, decl))
 735                 deprecated_conditional_nest = int(deprecated_conditional_nest)
 736                 in_declaration = ''
 737
 738         # We try to handle nested stucts/unions, but unmatched brackets in
 739         # comments will cause problems.
 740         if in_declaration == 'struct' or in_declaration == 'union':
 741             sm = re.search(r'\n\}\s*(\w*);\s*$', decl)
 742             if level <= 1 and sm:
 743                 if symbol == '':
 744                     symbol = sm.group(1)
 745
 746                 bm = re.search(r'^(\S+)(Class|Iface|Interface)\b', symbol)
 747                 if bm:
 748                     objectname = bm.group(1)
 749                     logging.info('Found object: "%s"', objectname)
 750                     title = '<TITLE>%s</TITLE>' % objectname
 751
 752                 logging.info('Store struct: "%s"', symbol)
 753                 if AddSymbolToList(slist, symbol):
 754                     structsym = in_declaration.upper()
 755                     decl_list.append('<%s>\n<NAME>%s</NAME>\n%s%s</%s>\n' %
 756                                      (structsym, symbol, deprecated, decl, structsym))
 757                     if symbol in forward_decls:
 758                         del forward_decls[symbol]
 759                 deprecated_conditional_nest = int(deprecated_conditional_nest)
 760                 in_declaration = ''
 761             else:
 762                 # We use tr to count the brackets in the line, and adjust
 763                 # $level accordingly.
 764                 level += line.count('{')
 765                 level -= line.count('}')
 766                 logging.info('struct/union level : %d', level)
 767
 768         pre_previous_line = previous_line
 769         previous_line = line
 770
 771     # print remaining forward declarations
 772     for symbol in sorted(iterkeys(forward_decls)):
 773         if forward_decls[symbol]:
 774             AddSymbolToList(slist, symbol)
 775             decl_list.append(forward_decls[symbol])
 776
 777     # add title
 778     slist = [title] + slist
 779
 780     logging.info("Scanning %s done", input_file)
 781
 782     # Try to separate the standard macros and functions, placing them at the
 783     # end of the current section, in a subsection named 'Standard'.
 784     # do this in a loop to catch object, enums and flags
 785     klass = lclass = prefix = lprefix = None
 786     standard_decl = []
 787     liststr = '\n'.join(s for s in slist if s) + '\n'
 788     while True:
 789         m = re.search(r'^(\S+)_IS_(\S*)_CLASS\n', liststr, flags=re.MULTILINE)
 790         m2 = re.search(r'^(\S+)_IS_(\S*)\n', liststr, flags=re.MULTILINE)
 791         m3 = re.search(r'^(\S+?)_(\S*)_get_type\n', liststr, flags=re.MULTILINE)
 792         if m:
 793             prefix = m.group(1)
 794             lprefix = prefix.lower()
 795             klass = m.group(2)
 796             lclass = klass.lower()
 797             logging.info("Found gobject type '%s_%s' from is_class macro", prefix, klass)
 798         elif m2:
 799             prefix = m2.group(1)
 800             lprefix = prefix.lower()
 801             klass = m2.group(2)
 802             lclass = klass.lower()
 803             logging.info("Found gobject type '%s_%s' from is_ macro", prefix, klass)
 804         elif m3:
 805             lprefix = m3.group(1)
 806             prefix = lprefix.upper()
 807             lclass = m3.group(2)
 808             klass = lclass.upper()
 809             logging.info("Found gobject type '%s_%s' from get_type function", prefix, klass)
 810         else:
 811             break
 812
 813         cclass = lclass
 814         cclass = cclass.replace('_', '')
 815         mtype = lprefix + cclass
 816
 817         liststr, standard_decl = replace_once(liststr, standard_decl, r'^%sPrivate\n' % mtype)
 818
 819         # We only leave XxYy* in the normal section if they have docs
 820         if mtype not in doc_comments:
 821             logging.info("  Hide instance docs for %s", mtype)
 822             liststr, standard_decl = replace_once(liststr, standard_decl, r'^%s\n' % mtype)
 823
 824         if mtype + 'class' not in doc_comments:
 825             logging.info("  Hide class docs for %s", mtype)
 826             liststr, standard_decl = replace_once(liststr, standard_decl, r'^%sClass\n' % mtype)
 827
 828         if mtype + 'interface' not in doc_comments:
 829             logging.info("  Hide iface docs for %s", mtype)
 830             liststr, standard_decl = replace_once(liststr, standard_decl, r'%sInterface\n' % mtype)
 831
 832         if mtype + 'iface' not in doc_comments:
 833             logging.info("  Hide iface docs for " + mtype)
 834             liststr, standard_decl = replace_once(liststr, standard_decl, r'%sIface\n' % mtype)
 835
 836         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_IS_%s\n' % klass)
 837         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_TYPE_%s\n' % klass)
 838         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_%s_get_type\n' % lclass)
 839         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_%s_CLASS\n' % klass)
 840         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_IS_%s_CLASS\n' % klass)
 841         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_%s_GET_CLASS\n' % klass)
 842         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_%s_GET_IFACE\n' % klass)
 843         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_%s_GET_INTERFACE\n' % klass)
 844         # We do this one last, otherwise it tends to be caught by the IS_$class macro
 845         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_%s\n' % klass)
 846
 847     logging.info('Decl:%s---', liststr)
 848     logging.info('Std :%s---', ''.join(sorted(standard_decl)))
 849     if len(standard_decl):
 850         # sort the symbols
 851         liststr += '<SUBSECTION Standard>\n' + ''.join(sorted(standard_decl))
 852
 853     if liststr != '':
 854         if file_basename not in section_list:
 855             section_list[file_basename] = ''
 856         section_list[file_basename] += "<SECTION>\n<FILE>%s</FILE>\n%s</SECTION>\n\n" % (file_basename, liststr)
 857
 858
 859 def replace_once(liststr, standard_decl, regex):
 860     mre = re.search(regex, liststr,  flags=re.IGNORECASE | re.MULTILINE)
 861     if mre:
 862         standard_decl.append(mre.group(0))
 863         liststr = re.sub(regex, '', liststr, flags=re.IGNORECASE | re.MULTILINE)
 864     return liststr, standard_decl
 865
 866
 867 def replace_all(liststr, standard_decl, regex):
 868     mre = re.search(regex, liststr, flags=re.MULTILINE)
 869     while mre:
 870         standard_decl.append(mre.group(0))
 871         liststr = re.sub(regex, '', liststr, flags=re.MULTILINE)
 872         mre = re.search(regex, liststr, flags=re.MULTILINE)
 873     return liststr, standard_decl
 874
 875
 876 def AddSymbolToList(slist, symbol):
 877     """ Adds symbol to list of declaration if not already present.
 878
 879     Args:
 880         slist: The list of symbols.
 881         symbol: The symbol to add to the list.
 882     """
 883     if symbol in slist:
 884          # logging.info('Symbol %s already in list. skipping', symbol)
 885          # we return False to skip outputting another entry to -decl.txt
 886          # this is to avoid redeclarations (e.g. in conditional sections).
 887         return False
 888     slist.append(symbol)
 889     return True