gtkdoc/scan.py

   1 # -*- python -*-
   2 #
   3 # gtk-doc - GTK DocBook documentation generator.
   4 # Copyright (C) 1998  Damon Chaplin
   5 #               2007-2016  Stefan Sauer
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  20 #
  21
  22 """
  23 Extracts declarations of functions, macros, enums, structs and unions from
  24 header files.
  25
  26 It is called with a module name, an optional source directory, an optional
  27 output directory, and the header files to scan.
  28
  29 It outputs all declarations found to a file named '$MODULE-decl.txt', and the
  30 list of decarations to another file '$MODULE-decl-list.txt'.
  31
  32 This second list file is typically copied to '$MODULE-sections.txt' and
  33 organized into sections ready to output the XML pages.
  34 """
  35
  36 import logging
  37 import os
  38 import re
  39 import shutil
  40
  41 from . import common
  42
  43 # do not read files twice; checking it here permits to give both srcdir and
  44 # builddir as --source-dir without fear of duplicities
  45 seen_headers = {}
  46
  47
  48 def Run(options):
  49     logging.info('options: %s', str(options.__dict__))
  50
  51     if not os.path.isdir(options.output_dir):
  52         os.mkdir(options.output_dir)
  53
  54     base_filename = os.path.join(options.output_dir, options.module)
  55     old_decl_list = base_filename + '-decl-list.txt'
  56     new_decl_list = base_filename + '-decl-list.new'
  57     old_decl = base_filename + '-decl.txt'
  58     new_decl = base_filename + '-decl.new'
  59     old_types = base_filename + '.types'
  60     new_types = base_filename + '.types.new'
  61     sections_file = base_filename + '-sections.txt'
  62
  63     # If this is the very first run then we create the .types file automatically.
  64     if not os.path.exists(sections_file) and not os.path.exists(old_types):
  65         options.rebuild_types = True
  66
  67     section_list = {}
  68     decl_list = []
  69     get_types = []
  70
  71     for file in options.headers:
  72         ScanHeader(file, section_list, decl_list, get_types, options)
  73
  74     for dir in options.source_dir:
  75         ScanHeaders(dir, section_list, decl_list, get_types, options)
  76
  77     with open(new_decl_list, 'w', encoding='utf-8') as f:
  78         for section in sorted(section_list.keys()):
  79             f.write(section_list[section])
  80
  81     with open(new_decl, 'w', encoding='utf-8') as f:
  82         for decl in decl_list:
  83             f.write(decl)
  84
  85     if options.rebuild_types:
  86         with open(new_types, 'w', encoding='utf-8') as f:
  87             for func in sorted(get_types):
  88                 f.write(func + '\n')
  89
  90         # remove the file if empty
  91         if len(get_types) == 0:
  92             os.unlink(new_types)
  93             if os.path.exists(old_types):
  94                 os.rename(old_types, old_types + '.bak')
  95         else:
  96             common.UpdateFileIfChanged(old_types, new_types, True)
  97
  98     common.UpdateFileIfChanged(old_decl_list, new_decl_list, True)
  99     common.UpdateFileIfChanged(old_decl, new_decl, True)
 100
 101     # If there is no MODULE-sections.txt file yet or we are asked to rebuild it,
 102     # we copy the MODULE-decl-list.txt file into its place. The user can tweak it
 103     # later if they want.
 104     if options.rebuild_sections or not os.path.exists(sections_file):
 105         new_sections_file = base_filename + '-sections.new'
 106         shutil.copyfile(old_decl_list, new_sections_file)
 107         common.UpdateFileIfChanged(sections_file, new_sections_file, False)
 108
 109     # If there is no MODULE-overrides.txt file we create an empty one
 110     # because EXTRA_DIST in gtk-doc.make requires it.
 111     overrides_file = base_filename + '-overrides.txt'
 112     if not os.path.exists(overrides_file):
 113         open(overrides_file, 'w', encoding='utf-8').close()
 114
 115
 116 #
 117 # Function    : ScanHeaders
 118 # Description : This scans a directory tree looking for header files.
 119 #
 120 # Arguments   : $source_dir - the directory to scan.
 121 #               $section_list - a reference to the hashmap of sections.
 122 #
 123
 124 def ScanHeaders(source_dir, section_list, decl_list, get_types, options):
 125     logging.info('Scanning source directory: %s', source_dir)
 126
 127     # This array holds any subdirectories found.
 128     subdirs = []
 129
 130     for file in sorted(os.listdir(source_dir)):
 131         if file.startswith('.'):
 132             continue
 133         fullname = os.path.join(source_dir, file)
 134         if os.path.isdir(fullname):
 135             subdirs.append(file)
 136         elif file.endswith('.h'):
 137             ScanHeader(fullname, section_list, decl_list, get_types, options)
 138
 139     # Now recursively scan the subdirectories.
 140     for dir in subdirs:
 141         matchstr = r'(\s|^)' + re.escape(dir) + r'(\s|$)'
 142         if re.search(matchstr, options.ignore_headers):
 143             continue
 144         ScanHeaders(os.path.join(source_dir, dir), section_list, decl_list,
 145                     get_types, options)
 146
 147
 148 #
 149 # Function    : ScanHeader
 150 # Description : This scans a header file, looking for declarations of
 151 #                functions, macros, typedefs, structs and unions, which it
 152 #                outputs to the decl_list.
 153 # Arguments   : $input_file - the header file to scan.
 154 #               $section_list - a map of sections.
 155 #               $decl_list - a list of declarations
 156 # Returns     : it adds declarations to the appropriate list.
 157 #
 158
 159 def ScanHeader(input_file, section_list, decl_list, get_types, options):
 160     global seen_headers
 161     slist = []                 # Holds the resulting list of declarations.
 162     title = ''                 # Holds the title of the section
 163     in_comment = 0             # True if we are in a comment.
 164     in_declaration = ''        # The type of declaration we are in, e.g.
 165                                #   'function' or 'macro'.
 166     skip_block = 0             # True if we should skip a block.
 167     symbol = None              # The current symbol being declared.
 168     decl = ''                  # Holds the declaration of the current symbol.
 169     ret_type = None            # For functions and function typedefs this
 170                                #   holds the function's return type.
 171     pre_previous_line = ''     # The pre-previous line read in - some Gnome
 172                                #   functions have the return type on one
 173                                #   line, the function name on the next,
 174                                #   and the rest of the declaration after.
 175     previous_line = ''         # The previous line read in - some Gnome
 176                                #   functions have the return type on one line
 177                                #   and the rest of the declaration after.
 178     first_macro = 1            # Used to try to skip the standard #ifdef XXX
 179                                # define XXX at the start of headers.
 180     level = None               # Used to handle structs/unions which contain
 181                                #   nested structs or unions.
 182     internal = 0               # Set to 1 for internal symbols, we need to
 183                                #   fully parse, but don't add them to docs
 184     forward_decls = {}         # Dict of forward declarations, we skip
 185                                #   them if we find the real declaration
 186                                #   later.
 187     doc_comments = {}          # Dict of doc-comments we found.
 188                                # The key is lowercase symbol name, val=1.
 189
 190     file_basename = None
 191
 192     deprecated_conditional_nest = 0
 193     ignore_conditional_nest = 0
 194
 195     deprecated = ''
 196     doc_comment = ''
 197
 198     # Don't scan headers twice
 199     canonical_input_file = os.path.realpath(input_file)
 200     if canonical_input_file in seen_headers:
 201         logging.info('File already scanned: %s', input_file)
 202         return
 203
 204     seen_headers[canonical_input_file] = 1
 205
 206     file_basename = os.path.split(input_file)[1][:-2]  # filename ends in .h
 207
 208     # Check if the basename is in the list of headers to ignore.
 209     matchstr = r'(\s|^)' + re.escape(file_basename) + r'\.h(\s|$)'
 210     if re.search(matchstr, options.ignore_headers):
 211         logging.info('File ignored: %s', input_file)
 212         return
 213
 214     # Check if the full name is in the list of headers to ignore.
 215     matchstr = r'(\s|^)' + re.escape(input_file) + r'(\s|$)'
 216     if re.search(matchstr, options.ignore_headers):
 217         logging.info('File ignored: %s', input_file)
 218         return
 219
 220     if not os.path.exists(input_file):
 221         logging.warning('File does not exist: %s', input_file)
 222         return
 223
 224     logging.info('Scanning %s', input_file)
 225
 226     for line in open(input_file, 'r', encoding='utf-8'):
 227         # If this is a private header, skip it.
 228         if re.search(r'^\s*/\*\s*<\s*private_header\s*>\s*\*/', line):
 229             return
 230
 231         # Skip to the end of the current comment.
 232         if in_comment:
 233             logging.info('Comment: %s', line.strip())
 234             doc_comment += line
 235             if re.search(r'\*/', line):
 236                 m = re.search(r'\* ([a-zA-Z][a-zA-Z0-9_]+):', doc_comment)
 237                 if m:
 238                     doc_comments[m.group(1).lower()] = 1
 239                 in_comment = 0
 240                 doc_comment = ''
 241             continue
 242
 243         # Keep a count of #if, #ifdef, #ifndef nesting,
 244         # and if we enter a deprecation-symbol-bracketed
 245         # zone, take note.
 246         m = re.search(r'^\s*#\s*if(?:n?def\b|\s+!?\s*defined\s*\()\s*(\w+)', line)
 247         if m:
 248             define_name = m.group(1)
 249             if deprecated_conditional_nest < 1 and re.search(options.deprecated_guards, define_name):
 250                 deprecated_conditional_nest = 1
 251             elif deprecated_conditional_nest >= 1:
 252                 deprecated_conditional_nest += 1
 253             if ignore_conditional_nest == 0 and '__GTK_DOC_IGNORE__' in define_name:
 254                 ignore_conditional_nest = 1
 255             elif ignore_conditional_nest > 0:
 256                 ignore_conditional_nest = 1
 257
 258         elif re.search(r'^\s*#\sif', line):
 259             if deprecated_conditional_nest >= 1:
 260                 deprecated_conditional_nest += 1
 261
 262             if ignore_conditional_nest > 0:
 263                 ignore_conditional_nest += 1
 264         elif re.search(r'^\s*#endif', line):
 265             if deprecated_conditional_nest >= 1:
 266                 deprecated_conditional_nest -= 1
 267
 268             if ignore_conditional_nest > 0:
 269                 ignore_conditional_nest -= 1
 270
 271         # If we find a line containing _DEPRECATED, we hope that this is
 272         # attribute based deprecation and also treat this as a deprecation
 273         # guard, unless it's a macro definition.
 274         if deprecated_conditional_nest == 0 and '_DEPRECATED' in line:
 275             m = re.search(r'^\s*#\s*(if*|define)', line)
 276             if not (m or in_declaration == 'enum'):
 277                 logging.info('Found deprecation annotation (decl: "%s"): "%s"',
 278                              in_declaration, line.strip())
 279                 deprecated_conditional_nest += 0.1
 280
 281         # set flag that is used later when we do AddSymbolToList
 282         if deprecated_conditional_nest > 0:
 283             deprecated = '<DEPRECATED/>\n'
 284         else:
 285             deprecated = ''
 286
 287         if ignore_conditional_nest:
 288             continue
 289
 290         if not in_declaration:
 291             # Skip top-level comments.
 292             m = re.search(r'^\s*/\*', line)
 293             if m:
 294                 re.sub(r'^\s*/\*', '', line)
 295                 if re.search(r'\*/', line):
 296                     logging.info('Found one-line comment: %s', line.strip())
 297                 else:
 298                     in_comment = 1
 299                     doc_comment = line
 300                     logging.info('Found start of comment: %s', line.strip())
 301                 continue
 302
 303             logging.info('no decl: %s', line.strip())
 304
 305             # avoid generating regex with |'' (matching no string)
 306             ignore_decorators = ''
 307             if options.ignore_decorators:
 308                 ignore_decorators = '|' + options.ignore_decorators
 309
 310             m = re.search(r'^\s*#\s*define\s+(\w+)', line)
 311             #                   $1                                $3            $4             $5
 312             m2 = re.search(
 313                 r'^\s*typedef\s+((const\s+|G_CONST_RETURN\s+)?\w+)(\s+const)?\s*(\**)\s*\(\*\s*(\w+)\)\s*\(', line)
 314             #                    $1                                $3            $4             $5
 315             m3 = re.search(r'^\s*((const\s+|G_CONST_RETURN\s+)?\w+)(\s+const)?\s*(\**)\s*\(\*\s*(\w+)\)\s*\(', line)
 316             #                    $1            $2
 317             m4 = re.search(r'^\s*(\**)\s*\(\*\s*(\w+)\)\s*\(', line)
 318             #                              $1                                $3
 319             m5 = re.search(r'^\s*typedef\s*((const\s+|G_CONST_RETURN\s+)?\w+)(\s+const)?\s*', previous_line)
 320             #                                              $1                                $3            $4             $5
 321             m6 = re.search(
 322                 r'^\s*(?:\b(?:extern|G_INLINE_FUNC%s)\s*)*((const\s+|G_CONST_RETURN\s+)?\w+)(\s+const)?\s*(\**)\s*\(\*\s*(\w+)\)\s*\(' % ignore_decorators, line)
 323             m7 = re.search(r'^\s*enum\s+_?(\w+)\s+\{', line)
 324             m8 = re.search(r'^\s*typedef\s+enum', line)
 325             m9 = re.search(r'^\s*typedef\s+(struct|union)\s+_(\w+)\s+\2\s*;', line)
 326             m10 = re.search(r'^\s*(struct|union)\s+(\w+)\s*;', line)
 327             m11 = re.search(r'^\s*typedef\s+(struct|union)\s*\w*\s*{', line)
 328             m12 = re.search(r'^\s*typedef\s+(?:struct|union)\s+\w+[\s\*]+(\w+)\s*;', line)
 329             m13 = re.search(r'^\s*(G_GNUC_EXTENSION\s+)?typedef\s+(.+[\s\*])(\w+)(\s*\[[^\]]+\])*\s*;', line)
 330             m14 = re.search(
 331                 r'^\s*(extern|[A-Za-z_]+VAR%s)\s+((const\s+|signed\s+|unsigned\s+|long\s+|short\s+)*\w+)(\s+\*+|\*+|\s)\s*(const\s+)*([A-Za-z]\w*)\s*;' % ignore_decorators, line)
 332             m15 = re.search(
 333                 r'^\s*((const\s+|signed\s+|unsigned\s+|long\s+|short\s+)*\w+)(\s+\*+|\*+|\s)\s*(const\s+)*([A-Za-z]\w*)\s*\=', line)
 334             m16 = re.search(r'.*G_DECLARE_(FINAL_TYPE|DERIVABLE_TYPE|INTERFACE)\s*\(', line)
 335             #                                             $1                                                                                                    $2                                                     $3
 336             m17 = re.search(
 337                 r'^\s*(?:\b(?:extern|G_INLINE_FUNC%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|long\s+|short\s+|struct\s+|union\s+|enum\s+)*\w+)([\s*]+(?:\s*(?:\*+|\bconst\b|\bG_CONST_RETURN\b))*)\s*(_[A-Za-z]\w*)\s*\(' % ignore_decorators, line)
 338             #                                             $1                                                                                                    $2                                                     $3
 339             m18 = re.search(
 340                 r'^\s*(?:\b(?:extern|G_INLINE_FUNC%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|long\s+|short\s+|struct\s+|union\s+|enum\s+)*\w+)([\s*]+(?:\s*(?:\*+|\bconst\b|\bG_CONST_RETURN\b))*)\s*([A-Za-z]\w*)\s*\(' % ignore_decorators, line)
 341             m19 = re.search(r'^\s*([A-Za-z]\w*)\s*\(', line)
 342             m20 = re.search(r'^\s*\(', line)
 343             m21 = re.search(r'^\s*struct\s+_?(\w+)', line)
 344             m22 = re.search(r'^\s*union\s+_(\w+)', line)
 345
 346             # MACROS
 347
 348             if m:
 349                 symbol = m.group(1)
 350                 decl = line
 351                 # We assume all macros which start with '_' are private.
 352                 # We also try to skip the first macro if it looks like the
 353                 # standard #ifndef HEADER_FILE #define HEADER_FILE etc.
 354                 # And we only want TRUE & FALSE defined in GLib.
 355                 if not symbol.startswith('_') \
 356                     and (not re.search(r'#ifndef\s+' + symbol, previous_line)
 357                          or first_macro == 0) \
 358                     and ((symbol != 'TRUE' and symbol != 'FALSE')
 359                          or options.module == 'glib'):
 360                     in_declaration = 'macro'
 361                     logging.info('Macro: "%s"', symbol)
 362                 else:
 363                     logging.info('skipping Macro: "%s"', symbol)
 364                     in_declaration = 'macro'
 365                     internal = 1
 366                 first_macro = 0
 367
 368             # TYPEDEF'D FUNCTIONS (i.e. user functions)
 369             elif m2:
 370                 p3 = m2.group(3) or ''
 371                 ret_type = "%s%s %s" % (m2.group(1), p3, m2.group(4))
 372                 symbol = m2.group(5)
 373                 decl = line[m2.end():]
 374                 in_declaration = 'user_function'
 375                 logging.info('user function (1): "%s", Returns: "%s"', symbol, ret_type)
 376
 377             elif re.search(r'^\s*typedef\s*', previous_line) and m3:
 378                 p3 = m3.group(3) or ''
 379                 ret_type = '%s%s %s' % (m3.group(1), p3, m3.group(4))
 380                 symbol = m3.group(5)
 381                 decl = line[m3.end():]
 382                 in_declaration = 'user_function'
 383                 logging.info('user function (2): "%s", Returns: "%s"', symbol, ret_type)
 384
 385             elif re.search(r'^\s*typedef\s*', previous_line) and m4:
 386                 ret_type = m4.group(1)
 387                 symbol = m4.group(2)
 388                 decl = line[m4.end():]
 389                 if m5:
 390                     p3 = m5.group(3) or ''
 391                     ret_type = "%s%s %s" % (m5.group(1), p3, ret_type)
 392                     in_declaration = 'user_function'
 393                     logging.info('user function (3): "%s", Returns: "%s"', symbol, ret_type)
 394
 395             # FUNCTION POINTER VARIABLES
 396             elif m6:
 397                 p3 = m6.group(3) or ''
 398                 ret_type = '%s%s %s' % (m6.group(1), p3, m6.group(4))
 399                 symbol = m6.group(5)
 400                 decl = line[m6.end():]
 401                 in_declaration = 'user_function'
 402                 logging.info('function pointer variable: "%s", Returns: "%s"', symbol, ret_type)
 403
 404             # ENUMS
 405
 406             elif m7:
 407                 re.sub(r'^\s*enum\s+_?(\w+)\s+\{', r'enum \1 {', line)
 408                 # We assume that 'enum _<enum_name> {' is really the
 409                 # declaration of enum <enum_name>.
 410                 symbol = m7.group(1)
 411                 decl = line
 412                 in_declaration = 'enum'
 413                 logging.info('plain enum: "%s"', symbol)
 414
 415             elif re.search(r'^\s*typedef\s+enum\s+_?(\w+)\s+\1\s*;', line):
 416                 # We skip 'typedef enum <enum_name> _<enum_name>;' as the enum will
 417                 # be declared elsewhere.
 418                 logging.info('skipping enum typedef: "%s"', line)
 419             elif m8:
 420                 symbol = ''
 421                 decl = line
 422                 in_declaration = 'enum'
 423                 logging.info('typedef enum: -')
 424
 425             # STRUCTS AND UNIONS
 426
 427             elif m9:
 428                 # We've found a 'typedef struct _<name> <name>;'
 429                 # This could be an opaque data structure, so we output an
 430                 # empty declaration. If the structure is actually found that
 431                 # will override this.
 432                 structsym = m9.group(1).upper()
 433                 logging.info('%s typedef: "%s"', structsym, m9.group(2))
 434                 forward_decls[m9.group(2)] = '<%s>\n<NAME>%s</NAME>\n%s</%s>\n' % (
 435                     structsym, m9.group(2), deprecated, structsym)
 436
 437             elif re.search(r'^\s*(?:struct|union)\s+_(\w+)\s*;', line):
 438                 # Skip private structs/unions.
 439                 logging.info('private struct/union')
 440
 441             elif m10:
 442                 # Do a similar thing for normal structs as for typedefs above.
 443                 # But we output the declaration as well in this case, so we
 444                 # can differentiate it from a typedef.
 445                 structsym = m10.group(1).upper()
 446                 logging.info('%s:%s', structsym, m10.group(2))
 447                 forward_decls[m10.group(2)] = '<%s>\n<NAME>%s</NAME>\n%s%s</%s>\n' % (
 448                     structsym, m10.group(2), line, deprecated, structsym)
 449
 450             elif m11:
 451                 symbol = ''
 452                 decl = line
 453                 level = 0
 454                 in_declaration = m11.group(1)
 455                 logging.info('typedef struct/union "%s"', in_declaration)
 456
 457             # OTHER TYPEDEFS
 458
 459             elif m12:
 460                 logging.info('Found struct/union(*) typedef "%s": "%s"', m12.group(1), line)
 461                 if AddSymbolToList(slist, m12.group(1)):
 462                     decl_list.append('<TYPEDEF>\n<NAME>%s</NAME>\n%s%s</TYPEDEF>\n' % (m12.group(1), deprecated, line))
 463
 464             elif m13:
 465                 if m13.group(2).split()[0] not in ('struct', 'union'):
 466                     logging.info('Found typedef: "%s"', line)
 467                     if AddSymbolToList(slist, m13.group(3)):
 468                         decl_list.append(
 469                             '<TYPEDEF>\n<NAME>%s</NAME>\n%s%s</TYPEDEF>\n' % (m13.group(3), deprecated, line))
 470             elif re.search(r'^\s*typedef\s+', line):
 471                 logging.info('Skipping typedef: "%s"', line)
 472
 473             # VARIABLES (extern'ed variables)
 474
 475             elif m14:
 476                 symbol = m14.group(6)
 477                 line = re.sub(r'^\s*([A-Za-z_]+VAR)\b', r'extern', line)
 478                 decl = line
 479                 logging.info('Possible extern var "%s": "%s"', symbol, decl)
 480                 if AddSymbolToList(slist, symbol):
 481                     decl_list.append('<VARIABLE>\n<NAME>%s</NAME>\n%s%s</VARIABLE>\n' % (symbol, deprecated, decl))
 482
 483             # VARIABLES
 484
 485             elif m15:
 486                 symbol = m15.group(5)
 487                 decl = line
 488                 logging.info('Possible global var" %s": "%s"', symbol, decl)
 489                 if AddSymbolToList(slist, symbol):
 490                     decl_list.append('<VARIABLE>\n<NAME>%s</NAME>\n%s%s</VARIABLE>\n' % (symbol, deprecated, decl))
 491
 492             # G_DECLARE_*
 493
 494             elif m16:
 495                 in_declaration = 'g-declare'
 496                 symbol = 'G_DECLARE_' + m16.group(1)
 497                 decl = line[m16.end():]
 498
 499             # FUNCTIONS
 500
 501             # We assume that functions which start with '_' are private, so
 502             # we skip them.
 503             elif m17:
 504                 ret_type = m17.group(1)
 505                 if m17.group(2):
 506                     ret_type += ' ' + m17.group(2)
 507                 symbol = m17.group(3)
 508                 decl = line[m17.end():]
 509                 logging.info('internal Function: "%s", Returns: "%s""%s"', symbol, m17.group(1), m17.group(2))
 510                 in_declaration = 'function'
 511                 internal = 1
 512                 if line.strip().startswith('G_INLINE_FUNC'):
 513                     logging.info('skip block after inline function')
 514                     # now we we need to skip a whole { } block
 515                     skip_block = 1
 516
 517             elif m18:
 518                 ret_type = m18.group(1)
 519                 if m18.group(2):
 520                     ret_type += ' ' + m18.group(2)
 521                 symbol = m18.group(3)
 522                 decl = line[m18.end():]
 523                 logging.info('Function (1): "%s", Returns: "%s""%s"', symbol, m18.group(1), m18.group(2))
 524                 in_declaration = 'function'
 525                 if line.strip().startswith('G_INLINE_FUNC'):
 526                     logging.info('skip block after inline function')
 527                     # now we we need to skip a whole { } block
 528                     skip_block = 1
 529
 530             # Try to catch function declarations which have the return type on
 531             # the previous line. But we don't want to catch complete functions
 532             # which have been declared G_INLINE_FUNC, e.g. g_bit_nth_lsf in
 533             # glib, or 'static inline' functions.
 534             elif m19:
 535                 symbol = m19.group(1)
 536                 decl = line[m19.end():]
 537
 538                 previous_line_strip = previous_line.strip()
 539                 previous_line_words = previous_line_strip.split()
 540
 541                 if not previous_line_strip.startswith('G_INLINE_FUNC'):
 542                     if not previous_line_words or previous_line_words[0] != 'static':
 543                         #                                          $ 1                                                                                                   $2
 544                         pm = re.search(r'^\s*(?:\b(?:extern%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|long\s+|short\s+|struct\s+|union\s+|enum\s+)*\w+)((?:\s*(?:\*+|\bconst\b|\bG_CONST_RETURN\b))*)\s*$' %
 545                                        ignore_decorators, previous_line)
 546                         if pm:
 547                             ret_type = pm.group(1)
 548                             if pm.group(2):
 549                                 ret_type += ' ' + pm.group(2)
 550                             logging.info('Function  (2): "%s", Returns: "%s"', symbol, ret_type)
 551                             in_declaration = 'function'
 552                     else:
 553                         logging.info('skip block after inline function')
 554                         # now we we need to skip a whole { } block
 555                         skip_block = 1
 556                         #                                                        $1                                                                                                    $2
 557                         pm = re.search(r'^\s*(?:\b(?:extern|static|inline%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|long\s+|short\s+|struct\s+|union\s+|enum\s+)*\w+)((?:\s*(?:\*+|\bconst\b|\bG_CONST_RETURN\b))*)\s*$' %
 558                                        ignore_decorators, previous_line)
 559                         if pm:
 560                             ret_type = pm.group(1)
 561                             if pm.group(2):
 562                                 ret_type += ' ' + pm.group(2)
 563                             logging.info('Function  (3): "%s", Returns: "%s"', symbol, ret_type)
 564                             in_declaration = 'function'
 565                 else:
 566                     if not previous_line_words or previous_line_words[0] != 'static':
 567                         logging.info('skip block after inline function')
 568                         # now we we need to skip a whole { } block
 569                         skip_block = 1
 570                         #                                                         $1                                                                                                   $2
 571                         pm = re.search(r'^\s*(?:\b(?:extern|G_INLINE_FUNC%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|long\s+|short\s+|struct\s+|union\s+|enum\s+)*\w+)((?:\s*(?:\*+|\bconst\b|\bG_CONST_RETURN\b))*)\s*$' %
 572                                        ignore_decorators, previous_line)
 573                         if pm:
 574                             ret_type = pm.group(1)
 575                             if pm.group(2):
 576                                 ret_type += ' ' + pm.group(2)
 577                             logging.info('Function (4): "%s", Returns: "%s"', symbol, ret_type)
 578                             in_declaration = 'function'
 579
 580             # Try to catch function declarations with the return type and name
 581             # on the previous line(s), and the start of the parameters on this.
 582             elif m20:
 583                 decl = line[m20.end():]
 584                 pm = re.search(
 585                     r'^\s*(?:\b(?:extern|G_INLINE_FUNC%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|enum\s+)*\w+)(\s+\*+|\*+|\s)\s*([A-Za-z]\w*)\s*$' % ignore_decorators, previous_line)
 586                 ppm = re.search(r'^\s*(?:\b(?:extern|G_INLINE_FUNC%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|struct\s+|union\s+|enum\s+)*\w+(?:\**\s+\**(?:const|G_CONST_RETURN))?(?:\s+|\s*\*+))\s*$' %
 587                                 ignore_decorators, pre_previous_line)
 588                 if pm:
 589                     ret_type = pm.group(1) + ' ' + pm.group(2)
 590                     symbol = pm.group(3)
 591                     in_declaration = 'function'
 592                     logging.info('Function (5): "%s", Returns: "%s"', symbol, ret_type)
 593
 594                 elif re.search(r'^\s*\w+\s*$', previous_line) and ppm:
 595                     ret_type = ppm.group(1)
 596                     ret_type = re.sub(r'\s*\n', '', ret_type, flags=re.MULTILINE)
 597                     in_declaration = 'function'
 598
 599                     symbol = previous_line
 600                     symbol = re.sub(r'^\s+', '', symbol)
 601                     symbol = re.sub(r'\s*\n', '', symbol, flags=re.MULTILINE)
 602                     logging.info('Function (6): "%s", Returns: "%s"', symbol, ret_type)
 603
 604             # } elsif (m/^extern\s+/) {
 605                 # print "DEBUG: Skipping extern: $_"
 606
 607             # STRUCTS
 608             elif re.search(r'^\s*struct\s+_?(\w+)\s*\*', line):
 609                 # Skip 'struct _<struct_name> *', since it could be a
 610                 # return type on its own line.
 611                 pass
 612             elif m21:
 613                 # We assume that 'struct _<struct_name>' is really the
 614                 # declaration of struct <struct_name>.
 615                 symbol = m21.group(1)
 616                 decl = line
 617                 # we will find the correct level as below we do $level += tr/{//
 618                 level = 0
 619                 in_declaration = 'struct'
 620                 logging.info('Struct(_): "%s"', symbol)
 621
 622             # UNIONS
 623             elif re.search(r'^\s*union\s+_(\w+)\s*\*', line):
 624                 # Skip 'union _<union_name> *' (see above)
 625                 pass
 626             elif m22:
 627                 symbol = m22.group(1)
 628                 decl = line
 629                 level = 0
 630                 in_declaration = 'union'
 631                 logging.info('Union(_): "%s"', symbol)
 632         else:
 633             logging.info('in decl: skip=%s %s', skip_block, line.strip())
 634             # If we were already in the middle of a declaration, we simply add
 635             # the current line onto the end of it.
 636             if skip_block == 0:
 637                 decl += line
 638             else:
 639                 # Remove all nested pairs of curly braces.
 640                 brace_remover = r'{[^{]*}'
 641                 bm = re.search(brace_remover, line)
 642                 while bm:
 643                     line = re.sub(brace_remover, '', line)
 644                     bm = re.search(brace_remover, line)
 645                 # Then hope at most one remains in the line...
 646                 bm = re.search(r'(.*?){', line)
 647                 if bm:
 648                     if skip_block == 1:
 649                         decl += bm.group(1)
 650                     skip_block += 1
 651                 elif '}' in line:
 652                     skip_block -= 1
 653                     if skip_block == 1:
 654                         # this is a hack to detect the end of declaration
 655                         decl += ';'
 656                         skip_block = 0
 657                         logging.info('2: ---')
 658
 659                 else:
 660                     if skip_block == 1:
 661                         decl += line
 662
 663         if in_declaration == "g-declare":
 664             dm = re.search(r'\s*(\w+)\s*,\s*(\w+)\s*,\s*(\w+)\s*,\s*(\w+)\s*,\s*(\w+)\s*\).*$', decl)
 665             # FIXME the original code does s// stuff here and we don't. Is it necessary?
 666             if dm:
 667                 ModuleObjName = dm.group(1)
 668                 module_obj_name = dm.group(2)
 669                 if options.rebuild_types:
 670                     get_types.append(module_obj_name + '_get_type')
 671                 forward_decls[ModuleObjName] = '<STRUCT>\n<NAME>%s</NAME>\n%s</STRUCT>\n' % (ModuleObjName, deprecated)
 672                 if symbol.startswith('G_DECLARE_DERIVABLE'):
 673                     forward_decls[ModuleObjName + 'Class'] = '<STRUCT>\n<NAME>%sClass</NAME>\n%s</STRUCT>\n' % (
 674                         ModuleObjName, deprecated)
 675                 if symbol.startswith('G_DECLARE_INTERFACE'):
 676                     forward_decls[ModuleObjName + 'Interface'] = '<STRUCT>\n<NAME>%sInterface</NAME>\n%s</STRUCT>\n' % (
 677                         ModuleObjName, deprecated)
 678                 in_declaration = ''
 679
 680         # Note that sometimes functions end in ') G_GNUC_PRINTF (2, 3);' or
 681         # ') __attribute__ (...);'.
 682         if in_declaration == 'function':
 683             regex = r'\)\s*(G_GNUC_.*|.*DEPRECATED.*%s\s*|__attribute__\s*\(.*\)\s*)*;.*$' % ignore_decorators
 684             pm = re.search(regex, decl, flags=re.MULTILINE)
 685             if pm:
 686                 logging.info('scrubbing:[%s]', decl.strip())
 687                 decl = re.sub(regex, '', decl, flags=re.MULTILINE)
 688                 logging.info('scrubbed:[%s]', decl.strip())
 689                 if internal == 0:
 690                     decl = re.sub(r'/\*.*?\*/', '', decl, flags=re.MULTILINE)   # remove comments.
 691                     decl = re.sub(r'\s*\n\s*(?!$)', ' ', decl, flags=re.MULTILINE)
 692                     # consolidate whitespace at start/end of lines.
 693                     decl = decl.strip()
 694                     ret_type = re.sub(r'/\*.*?\*/', '', ret_type)               # remove comments in ret type.
 695                     if AddSymbolToList(slist, symbol):
 696                         decl_list.append('<FUNCTION>\n<NAME>%s</NAME>\n%s<RETURNS>%s</RETURNS>\n%s\n</FUNCTION>\n' %
 697                                          (symbol, deprecated, ret_type, decl))
 698                         if options.rebuild_types:
 699                             # check if this looks like a get_type function and if so remember
 700                             if symbol.endswith('_get_type') and 'GType' in ret_type and re.search(r'^(void|)$', decl):
 701                                 logging.info(
 702                                     "Adding get-type: [%s] [%s] [%s]\tfrom %s", ret_type, symbol, decl, input_file)
 703                                 get_types.append(symbol)
 704                 else:
 705                     internal = 0
 706                 deprecated_conditional_nest = int(deprecated_conditional_nest)
 707                 in_declaration = ''
 708                 skip_block = 0
 709
 710         if in_declaration == 'user_function':
 711             if re.search(r'\).*$', decl):
 712                 decl = re.sub(r'\).*$', '', decl)
 713                 if AddSymbolToList(slist, symbol):
 714                     decl_list.append('<USER_FUNCTION>\n<NAME>%s</NAME>\n%s<RETURNS>%s</RETURNS>\n%s</USER_FUNCTION>\n' %
 715                                      (symbol, deprecated, ret_type, decl))
 716                 deprecated_conditional_nest = int(deprecated_conditional_nest)
 717                 in_declaration = ''
 718
 719         if in_declaration == 'macro':
 720             if not re.search(r'\\\s*$', decl):
 721                 if internal == 0:
 722                     if AddSymbolToList(slist, symbol):
 723                         decl_list.append('<MACRO>\n<NAME>%s</NAME>\n%s%s</MACRO>\n' % (symbol, deprecated, decl))
 724                 else:
 725                     internal = 0
 726                 deprecated_conditional_nest = int(deprecated_conditional_nest)
 727                 in_declaration = ''
 728
 729         if in_declaration == 'enum':
 730             em = re.search(r'\}\s*(\w+)?;\s*$', decl)
 731             if em:
 732                 if symbol == '':
 733                     symbol = em.group(1)
 734                 if AddSymbolToList(slist, symbol):
 735                     decl_list.append('<ENUM>\n<NAME>%s</NAME>\n%s%s</ENUM>\n' % (symbol, deprecated, decl))
 736                 deprecated_conditional_nest = int(deprecated_conditional_nest)
 737                 in_declaration = ''
 738
 739         # We try to handle nested stucts/unions, but unmatched brackets in
 740         # comments will cause problems.
 741         if in_declaration == 'struct' or in_declaration == 'union':
 742             sm = re.search(r'\n\}\s*(\w*);\s*$', decl)
 743             if level <= 1 and sm:
 744                 if symbol == '':
 745                     symbol = sm.group(1)
 746
 747                 bm = re.search(r'^(\S+)(Class|Iface|Interface)\b', symbol)
 748                 if bm:
 749                     objectname = bm.group(1)
 750                     logging.info('Found object: "%s"', objectname)
 751                     title = '<TITLE>%s</TITLE>' % objectname
 752
 753                 logging.info('Store struct: "%s"', symbol)
 754                 if AddSymbolToList(slist, symbol):
 755                     structsym = in_declaration.upper()
 756                     decl_list.append('<%s>\n<NAME>%s</NAME>\n%s%s</%s>\n' %
 757                                      (structsym, symbol, deprecated, decl, structsym))
 758                     if symbol in forward_decls:
 759                         del forward_decls[symbol]
 760                 deprecated_conditional_nest = int(deprecated_conditional_nest)
 761                 in_declaration = ''
 762             else:
 763                 # We use tr to count the brackets in the line, and adjust
 764                 # $level accordingly.
 765                 level += line.count('{')
 766                 level -= line.count('}')
 767                 logging.info('struct/union level : %d', level)
 768
 769         pre_previous_line = previous_line
 770         previous_line = line
 771
 772     # print remaining forward declarations
 773     for symbol in sorted(forward_decls.keys()):
 774         if forward_decls[symbol]:
 775             AddSymbolToList(slist, symbol)
 776             decl_list.append(forward_decls[symbol])
 777
 778     # add title
 779     slist = [title] + slist
 780
 781     logging.info("Scanning %s done", input_file)
 782
 783     # Try to separate the standard macros and functions, placing them at the
 784     # end of the current section, in a subsection named 'Standard'.
 785     # do this in a loop to catch object, enums and flags
 786     klass = lclass = prefix = lprefix = None
 787     standard_decl = []
 788     liststr = '\n'.join(s for s in slist if s) + '\n'
 789     while True:
 790         m = re.search(r'^(\S+)_IS_(\S*)_CLASS\n', liststr, flags=re.MULTILINE)
 791         m2 = re.search(r'^(\S+)_IS_(\S*)\n', liststr, flags=re.MULTILINE)
 792         m3 = re.search(r'^(\S+?)_(\S*)_get_type\n', liststr, flags=re.MULTILINE)
 793         if m:
 794             prefix = m.group(1)
 795             lprefix = prefix.lower()
 796             klass = m.group(2)
 797             lclass = klass.lower()
 798             logging.info("Found gobject type '%s_%s' from is_class macro", prefix, klass)
 799         elif m2:
 800             prefix = m2.group(1)
 801             lprefix = prefix.lower()
 802             klass = m2.group(2)
 803             lclass = klass.lower()
 804             logging.info("Found gobject type '%s_%s' from is_ macro", prefix, klass)
 805         elif m3:
 806             lprefix = m3.group(1)
 807             prefix = lprefix.upper()
 808             lclass = m3.group(2)
 809             klass = lclass.upper()
 810             logging.info("Found gobject type '%s_%s' from get_type function", prefix, klass)
 811         else:
 812             break
 813
 814         cclass = lclass
 815         cclass = cclass.replace('_', '')
 816         mtype = lprefix + cclass
 817
 818         liststr, standard_decl = replace_once(liststr, standard_decl, r'^%sPrivate\n' % mtype)
 819
 820         # We only leave XxYy* in the normal section if they have docs
 821         if mtype not in doc_comments:
 822             logging.info("  Hide instance docs for %s", mtype)
 823             liststr, standard_decl = replace_once(liststr, standard_decl, r'^%s\n' % mtype)
 824
 825         if mtype + 'class' not in doc_comments:
 826             logging.info("  Hide class docs for %s", mtype)
 827             liststr, standard_decl = replace_once(liststr, standard_decl, r'^%sClass\n' % mtype)
 828
 829         if mtype + 'interface' not in doc_comments:
 830             logging.info("  Hide iface docs for %s", mtype)
 831             liststr, standard_decl = replace_once(liststr, standard_decl, r'%sInterface\n' % mtype)
 832
 833         if mtype + 'iface' not in doc_comments:
 834             logging.info("  Hide iface docs for " + mtype)
 835             liststr, standard_decl = replace_once(liststr, standard_decl, r'%sIface\n' % mtype)
 836
 837         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_IS_%s\n' % klass)
 838         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_TYPE_%s\n' % klass)
 839         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_%s_get_type\n' % lclass)
 840         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_%s_CLASS\n' % klass)
 841         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_IS_%s_CLASS\n' % klass)
 842         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_%s_GET_CLASS\n' % klass)
 843         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_%s_GET_IFACE\n' % klass)
 844         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_%s_GET_INTERFACE\n' % klass)
 845         # We do this one last, otherwise it tends to be caught by the IS_$class macro
 846         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_%s\n' % klass)
 847
 848     logging.info('Decl:%s---', liststr)
 849     logging.info('Std :%s---', ''.join(sorted(standard_decl)))
 850     if len(standard_decl):
 851         # sort the symbols
 852         liststr += '<SUBSECTION Standard>\n' + ''.join(sorted(standard_decl))
 853
 854     if liststr != '':
 855         if file_basename not in section_list:
 856             section_list[file_basename] = ''
 857         section_list[file_basename] += "<SECTION>\n<FILE>%s</FILE>\n%s</SECTION>\n\n" % (file_basename, liststr)
 858
 859
 860 def replace_once(liststr, standard_decl, regex):
 861     mre = re.search(regex, liststr,  flags=re.IGNORECASE | re.MULTILINE)
 862     if mre:
 863         standard_decl.append(mre.group(0))
 864         liststr = re.sub(regex, '', liststr, flags=re.IGNORECASE | re.MULTILINE)
 865     return liststr, standard_decl
 866
 867
 868 def replace_all(liststr, standard_decl, regex):
 869     mre = re.search(regex, liststr, flags=re.MULTILINE)
 870     while mre:
 871         standard_decl.append(mre.group(0))
 872         liststr = re.sub(regex, '', liststr, flags=re.MULTILINE)
 873         mre = re.search(regex, liststr, flags=re.MULTILINE)
 874     return liststr, standard_decl
 875
 876
 877 def AddSymbolToList(slist, symbol):
 878     """ Adds symbol to list of declaration if not already present.
 879
 880     Args:
 881         slist: The list of symbols.
 882         symbol: The symbol to add to the list.
 883     """
 884     if symbol in slist:
 885         # logging.info('Symbol %s already in list. skipping', symbol)
 886         # we return False to skip outputting another entry to -decl.txt
 887         # this is to avoid redeclarations (e.g. in conditional sections).
 888         return False
 889     slist.append(symbol)
 890     return True