gtkdoc/scan.py

   1 # -*- python -*-
   2 #
   3 # gtk-doc - GTK DocBook documentation generator.
   4 # Copyright (C) 1998  Damon Chaplin
   5 #               2007-2016  Stefan Sauer
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  20 #
  21
  22 """
  23 Extracts declarations of functions, macros, enums, structs and unions from
  24 header files.
  25
  26 It is called with a module name, an optional source directory, an optional
  27 output directory, and the header files to scan.
  28
  29 It outputs all declarations found to a file named '$MODULE-decl.txt', and the
  30 list of decarations to another file '$MODULE-decl-list.txt'.
  31
  32 This second list file is typically copied to '$MODULE-sections.txt' and
  33 organized into sections ready to output the XML pages.
  34 """
  35
  36 import logging
  37 import os
  38 import re
  39
  40 from . import common
  41
  42 # do not read files twice; checking it here permits to give both srcdir and
  43 # builddir as --source-dir without fear of duplicities
  44 seen_headers = {}
  45
  46
  47 def Run(options):
  48     # logging.basicConfig(level=logging.INFO)
  49
  50     if not os.path.isdir(options.output_dir):
  51         os.mkdir(options.output_dir)
  52
  53     base_filename = os.path.join(options.output_dir, options.module)
  54     old_decl_list = base_filename + '-decl-list.txt'
  55     new_decl_list = base_filename + '-decl-list.new'
  56     old_decl = base_filename + '-decl.txt'
  57     new_decl = base_filename + '-decl.new'
  58     old_types = base_filename + '.types'
  59     new_types = base_filename + '.types.new'
  60     sections_file = base_filename + '-sections.txt'
  61
  62     # If this is the very first run then we create the .types file automatically.
  63     if not os.path.exists(sections_file) and not os.path.exists(old_types):
  64         options.rebuild_types = True
  65
  66     section_list = {}
  67     decl_list = []
  68     get_types = []
  69
  70     for file in options.headers:
  71         ScanHeader(file, section_list, decl_list, get_types, options)
  72
  73     for dir in options.source_dir:
  74         ScanHeaders(dir, section_list, decl_list, get_types, options)
  75
  76     with open(new_decl_list, 'w') as f:
  77         for section in sorted(section_list.keys()):
  78             f.write(section_list[section])
  79
  80     with open(new_decl, 'w') as f:
  81         for decl in decl_list:
  82             f.write(decl)
  83
  84     if options.rebuild_types:
  85         with open(new_types, 'w') as f:
  86             for func in sorted(get_types):
  87                 f.write(func + '\n')
  88
  89         # remove the file if empty
  90         if len(get_types) == 0:
  91             os.unlink(new_types)
  92             if os.path.exists(old_types):
  93                 os.rename(old_types, old_types + '.bak')
  94         else:
  95             common.UpdateFileIfChanged(old_types, new_types, True)
  96
  97     common.UpdateFileIfChanged(old_decl_list, new_decl_list, True)
  98     common.UpdateFileIfChanged(old_decl, new_decl, True)
  99
 100     # If there is no MODULE-sections.txt file yet or we are asked to rebuild it,
 101     # we copy the MODULE-decl-list.txt file into its place. The user can tweak it
 102     # later if they want.
 103     if options.rebuild_sections or not os.path.exists(sections_file):
 104         common.UpdateFileIfChanged(sections_file, old_decl_list, False)
 105
 106     # If there is no MODULE-overrides.txt file we create an empty one
 107     # because EXTRA_DIST in gtk-doc.make requires it.
 108     overrides_file = base_filename + '-overrides.txt'
 109     if not os.path.exists(overrides_file):
 110         open(overrides_file, 'w').close()
 111
 112
 113 #
 114 # Function    : ScanHeaders
 115 # Description : This scans a directory tree looking for header files.
 116 #
 117 # Arguments   : $source_dir - the directory to scan.
 118 #               $section_list - a reference to the hashmap of sections.
 119 #
 120
 121 def ScanHeaders(source_dir, section_list, decl_list, get_types, options):
 122     logging.info('Scanning source directory: %s', source_dir)
 123
 124     # This array holds any subdirectories found.
 125     subdirs = []
 126
 127     for file in os.listdir(source_dir):
 128         if file.startswith('.'):
 129             continue
 130         fullname = os.path.join(source_dir, file)
 131         if os.path.isdir(fullname):
 132             subdirs.append(file)
 133         elif file.endswith('.h'):
 134             ScanHeader(fullname, section_list, decl_list, get_types, options)
 135
 136     # Now recursively scan the subdirectories.
 137     for dir in subdirs:
 138         matchstr = r'(\s|^)' + re.escape(dir) + r'(\s|$)'
 139         if re.search(matchstr, options.ignore_headers):
 140             continue
 141         ScanHeaders(os.path.join(source_dir, dir), section_list, decl_list,
 142                     get_types, options)
 143
 144
 145 #
 146 # Function    : ScanHeader
 147 # Description : This scans a header file, looking for declarations of
 148 #                functions, macros, typedefs, structs and unions, which it
 149 #                outputs to the decl_list.
 150 # Arguments   : $input_file - the header file to scan.
 151 #               $section_list - a map of sections.
 152 #               $decl_list - a list of declarations
 153 # Returns     : it adds declarations to the appropriate list.
 154 #
 155
 156 def ScanHeader(input_file, section_list, decl_list, get_types, options):
 157     global seen_headers
 158     slist = []                  # Holds the resulting list of declarations.
 159     title = ''                 # Holds the title of the section
 160     in_comment = 0                  # True if we are in a comment.
 161     in_declaration = ''          # The type of declaration we are in, e.g.
 162                               #   'function' or 'macro'.
 163     skip_block = 0                  # True if we should skip a block.
 164     symbol = None                  # The current symbol being declared.
 165     decl = ''                          # Holds the declaration of the current symbol.
 166     ret_type = None                  # For functions and function typedefs this
 167                               #   holds the function's return type.
 168     pre_previous_line = ''   # The pre-previous line read in - some Gnome
 169                               #   functions have the return type on one
 170                               #   line, the function name on the next,
 171                               #   and the rest of the declaration after.
 172     previous_line = ''          # The previous line read in - some Gnome
 173                               #   functions have the return type on one line
 174                               #   and the rest of the declaration after.
 175     first_macro = 1          # Used to try to skip the standard #ifdef XXX
 176                               # define XXX at the start of headers.
 177     level = None                          # Used to handle structs/unions which contain
 178                               #   nested structs or unions.
 179     internal = 0             # Set to 1 for internal symbols, we need to
 180                                 #   fully parse, but don't add them to docs
 181     forward_decls = {}         # hashtable of forward declarations, we skip
 182                                 #   them if we find the real declaration
 183                                 #   later.
 184     doc_comments = {}          # hastable of doc-comment we found. We can
 185                                 # use that to put element to the right
 186                                 # sction in the generated section-file
 187
 188     file_basename = None
 189
 190     deprecated_conditional_nest = 0
 191     ignore_conditional_nest = 0
 192
 193     deprecated = ''
 194     doc_comment = ''
 195
 196     # Don't scan headers twice
 197     canonical_input_file = os.path.realpath(input_file)
 198     if canonical_input_file in seen_headers:
 199         logging.info('File already scanned: %s', input_file)
 200         return
 201
 202     seen_headers[canonical_input_file] = 1
 203
 204     file_basename = os.path.split(input_file)[1][:-2]  # filename ends in .h
 205
 206     # Check if the basename is in the list of headers to ignore.
 207     matchstr = r'(\s|^)' + re.escape(file_basename) + r'\.h(\s|$)'
 208     if re.search(matchstr, options.ignore_headers):
 209         logging.info('File ignored: %s', input_file)
 210         return
 211
 212     # Check if the full name is in the list of headers to ignore.
 213     matchstr = r'(\s|^)' + re.escape(input_file) + r'(\s|$)'
 214     if re.search(matchstr, options.ignore_headers):
 215         logging.info('File ignored: %s', input_file)
 216         return
 217
 218     if not os.path.exists(input_file):
 219         logging.warning('File does not exist: %s', input_file)
 220         return
 221
 222     logging.info('Scanning %s', input_file)
 223
 224     for line in open(input_file):
 225         # If this is a private header, skip it.
 226         if re.search(r'%^\s*/\*\s*<\s*private_header\s*>\s*\*/', line):
 227             return
 228
 229         # Skip to the end of the current comment.
 230         if in_comment:
 231             logging.info('Comment: %s', line)
 232             doc_comment += line
 233             if re.search(r'\*/', line):
 234                 m = re.search(r'\* ([a-zA-Z][a-zA-Z0-9_]+):/', doc_comment)
 235                 if m:
 236                     doc_comments[m.group(1).lower()] = 1
 237                 in_comment = 0
 238                 doc_comment = ''
 239             continue
 240
 241         # Keep a count of #if, #ifdef, #ifndef nesting,
 242         # and if we enter a deprecation-symbol-bracketed
 243         # zone, take note.
 244         m = re.search(r'^\s*#\s*if(?:n?def\b|\s+!?\s*defined\s*\()\s*(\w+)', line)
 245         if m:
 246             define_name = m.group(1)
 247             if deprecated_conditional_nest < 1 and re.search(options.deprecated_guards, define_name):
 248                 deprecated_conditional_nest = 1
 249             elif deprecated_conditional_nest >= 1:
 250                 deprecated_conditional_nest += 1
 251             if ignore_conditional_nest == 0 and '__GTK_DOC_IGNORE__' in define_name:
 252                 ignore_conditional_nest = 1
 253             elif ignore_conditional_nest > 0:
 254                 ignore_conditional_nest = 1
 255
 256         elif re.search(r'^\s*#\sif', line):
 257             if deprecated_conditional_nest >= 1:
 258                 deprecated_conditional_nest += 1
 259
 260             if ignore_conditional_nest > 0:
 261                 ignore_conditional_nest += 1
 262         elif re.search(r'^\s*#endif', line):
 263             if deprecated_conditional_nest >= 1:
 264                 deprecated_conditional_nest -= 1
 265
 266             if ignore_conditional_nest > 0:
 267                 ignore_conditional_nest -= 1
 268
 269         # If we find a line containing _DEPRECATED, we hope that this is
 270         # attribute based deprecation and also treat this as a deprecation
 271         # guard, unless it's a macro definition.
 272         if deprecated_conditional_nest == 0 and '_DEPRECATED' in line:
 273             m = re.search(r'^\s*#\s*(if*|define)', line)
 274             if not (m or in_declaration == 'enum'):
 275                 logging.info('Found deprecation annotation (decl: "%s"): "%s"', in_declaration, line)
 276                 deprecated_conditional_nest += 0.1
 277
 278         # set flag that is used later when we do AddSymbolToList
 279         if deprecated_conditional_nest > 0:
 280             deprecated = '<DEPRECATED/>\n'
 281         else:
 282             deprecated = ''
 283
 284         if ignore_conditional_nest:
 285             continue
 286
 287         if not in_declaration:
 288             # Skip top-level comments.
 289             m = re.search(r'^\s*/\*', line)
 290             if m:
 291                 re.sub(r'^\s*/\*', '', line)
 292                 if re.search(r'\*/', line):
 293                     logging.info('Found one-line comment: %s', line)
 294                 else:
 295                     in_comment = 1
 296                     doc_comment = line
 297                     logging.info('Found start of comment: %s', line)
 298                 continue
 299
 300             logging.info('no decl: %s', line.strip())
 301
 302             # avoid generating regex with |'' (matching no string)
 303             ignore_decorators = ''
 304             if options.ignore_decorators:
 305                 ignore_decorators = '|' + options.ignore_decorators
 306
 307             m = re.search(r'^\s*#\s*define\s+(\w+)', line)
 308             #                   $1                                $3            $4             $5
 309             m2 = re.search(
 310                 r'^\s*typedef\s+((const\s+|G_CONST_RETURN\s+)?\w+)(\s+const)?\s*(\**)\s*\(\*\s*(\w+)\)\s*\(', line)
 311             #                    $1                                $3            $4             $5
 312             m3 = re.search(r'^\s*((const\s+|G_CONST_RETURN\s+)?\w+)(\s+const)?\s*(\**)\s*\(\*\s*(\w+)\)\s*\(', line)
 313             #                    $1            $2
 314             m4 = re.search(r'^\s*(\**)\s*\(\*\s*(\w+)\)\s*\(', line)
 315             #                              $1                                $3
 316             m5 = re.search(r'^\s*typedef\s*((const\s+|G_CONST_RETURN\s+)?\w+)(\s+const)?\s*', previous_line)
 317             #                                              $1                                $3            $4             $5
 318             m6 = re.search(
 319                 r'^\s*(?:\b(?:extern|G_INLINE_FUNC%s)\s*)*((const\s+|G_CONST_RETURN\s+)?\w+)(\s+const)?\s*(\**)\s*\(\*\s*(\w+)\)\s*\(' % ignore_decorators, line)
 320             m7 = re.search(r'^\s*enum\s+_?(\w+)\s+\{', line)
 321             m8 = re.search(r'^\s*typedef\s+enum', line)
 322             m9 = re.search(r'^\s*typedef\s+(struct|union)\s+_(\w+)\s+\2\s*;', line)
 323             m10 = re.search(r'^\s*(struct|union)\s+(\w+)\s*;', line)
 324             m11 = re.search(r'^\s*typedef\s+(struct|union)\s*\w*\s*{', line)
 325             m12 = re.search(r'^\s*typedef\s+(?:struct|union)\s+\w+[\s\*]+(\w+)\s*;', line)
 326             m13 = re.search(r'^\s*(G_GNUC_EXTENSION\s+)?typedef\s+(.+[\s\*])(\w+)(\s*\[[^\]]+\])*\s*;', line)
 327             m14 = re.search(
 328                 r'^\s*(extern|[A-Za-z_]+VAR%s)\s+((const\s+|signed\s+|unsigned\s+|long\s+|short\s+)*\w+)(\s+\*+|\*+|\s)\s*(const\s+)*([A-Za-z]\w*)\s*;' % ignore_decorators, line)
 329             m15 = re.search(
 330                 r'^\s*((const\s+|signed\s+|unsigned\s+|long\s+|short\s+)*\w+)(\s+\*+|\*+|\s)\s*(const\s+)*([A-Za-z]\w*)\s*\=', line)
 331             m16 = re.search(r'.*G_DECLARE_(FINAL_TYPE|DERIVABLE_TYPE|INTERFACE)\s*\(', line)
 332             #                                             $1                                                                                                    $2                                                     $3
 333             m17 = re.search(
 334                 r'^\s*(?:\b(?:extern|G_INLINE_FUNC%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|long\s+|short\s+|struct\s+|union\s+|enum\s+)*\w+)([\s*]+(?:\s*(?:\*+|\bconst\b|\bG_CONST_RETURN\b))*)\s*(_[A-Za-z]\w*)\s*\(' % ignore_decorators, line)
 335             #                                             $1                                                                                                    $2                                                     $3
 336             m18 = re.search(
 337                 r'^\s*(?:\b(?:extern|G_INLINE_FUNC%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|long\s+|short\s+|struct\s+|union\s+|enum\s+)*\w+)([\s*]+(?:\s*(?:\*+|\bconst\b|\bG_CONST_RETURN\b))*)\s*([A-Za-z]\w*)\s*\(' % ignore_decorators, line)
 338             m19 = re.search(r'^\s*([A-Za-z]\w*)\s*\(', line)
 339             m20 = re.search(r'^\s*\(', line)
 340             m21 = re.search(r'^\s*struct\s+_?(\w+)', line)
 341             m22 = re.search(r'^\s*union\s+_(\w+)', line)
 342
 343             # MACROS
 344
 345             if m:
 346                 symbol = m.group(1)
 347                 decl = line
 348                 # We assume all macros which start with '_' are private.
 349                 # We also try to skip the first macro if it looks like the
 350                 # standard #ifndef HEADER_FILE #define HEADER_FILE etc.
 351                 # And we only want TRUE & FALSE defined in GLib.
 352                 if not symbol.startswith('_') \
 353                     and (not re.search(r'#ifndef\s+' + symbol, previous_line)
 354                          or first_macro == 0) \
 355                     and ((symbol != 'TRUE' and symbol != 'FALSE')
 356                          or options.module == 'glib'):
 357                     in_declaration = 'macro'
 358                     logging.info('Macro: "%s"', symbol)
 359                 else:
 360                     logging.info('skipping Macro: "%s"', symbol)
 361                     in_declaration = 'macro'
 362                     internal = 1
 363                 first_macro = 0
 364
 365             # TYPEDEF'D FUNCTIONS (i.e. user functions)
 366             elif m2:
 367                 p3 = m2.group(3) or ''
 368                 ret_type = "%s%s %s" % (m2.group(1), p3, m2.group(4))
 369                 symbol = m2.group(5)
 370                 decl = line[m2.end():]
 371                 in_declaration = 'user_function'
 372                 logging.info('user function (1): "%s", Returns: "%s"', symbol, ret_type)
 373
 374             elif re.search(r'^\s*typedef\s*', previous_line) and m3:
 375                 p3 = m3.group(3) or ''
 376                 ret_type = '%s%s %s' % (m3.group(1), p3, m3.group(4))
 377                 symbol = m3.group(5)
 378                 decl = line[m3.end():]
 379                 in_declaration = 'user_function'
 380                 logging.info('user function (2): "%s", Returns: "%s"', symbol, ret_type)
 381
 382             elif re.search(r'^\s*typedef\s*', previous_line) and m4:
 383                 ret_type = m4.group(1)
 384                 symbol = m4.group(2)
 385                 decl = line[m4.end():]
 386                 if m5:
 387                     p3 = m5.group(3) or ''
 388                     ret_type = "%s%s %s" % (m5.group(1), p3, ret_type)
 389                     in_declaration = 'user_function'
 390                     logging.info('user function (3): "%s", Returns: "%s"', symbol, ret_type)
 391
 392             # FUNCTION POINTER VARIABLES
 393             elif m6:
 394                 p3 = m6.group(3) or ''
 395                 ret_type = '%s%s %s' % (m6.group(1), p3, m6.group(4))
 396                 symbol = m6.group(5)
 397                 decl = line[m6.end():]
 398                 in_declaration = 'user_function'
 399                 logging.info('function pointer variable: "%s", Returns: "%s"', symbol, ret_type)
 400
 401             # ENUMS
 402
 403             elif m7:
 404                 re.sub(r'^\s*enum\s+_?(\w+)\s+\{', r'enum \1 {', line)
 405                 # We assume that 'enum _<enum_name> {' is really the
 406                 # declaration of enum <enum_name>.
 407                 symbol = m7.group(1)
 408                 decl = line
 409                 in_declaration = 'enum'
 410                 logging.info('plain enum: "%s"', symbol)
 411
 412             elif re.search(r'^\s*typedef\s+enum\s+_?(\w+)\s+\1\s*;', line):
 413                 # We skip 'typedef enum <enum_name> _<enum_name>;' as the enum will
 414                 # be declared elsewhere.
 415                 logging.info('skipping enum typedef: "%s"', line)
 416             elif m8:
 417                 symbol = ''
 418                 decl = line
 419                 in_declaration = 'enum'
 420                 logging.info('typedef enum: -')
 421
 422             # STRUCTS AND UNIONS
 423
 424             elif m9:
 425                 # We've found a 'typedef struct _<name> <name>;'
 426                 # This could be an opaque data structure, so we output an
 427                 # empty declaration. If the structure is actually found that
 428                 # will override this.
 429                 structsym = m9.group(1).upper()
 430                 logging.info('%s typedef: "%s"', structsym, m9.group(2))
 431                 forward_decls[m9.group(2)] = '<%s>\n<NAME>%s</NAME>\n%s</%s>\n' % (
 432                     structsym, m9.group(2), deprecated, structsym)
 433
 434             elif re.search(r'^\s*(?:struct|union)\s+_(\w+)\s*;', line):
 435                 # Skip private structs/unions.
 436                 logging.info('private struct/union')
 437
 438             elif m10:
 439                 # Do a similar thing for normal structs as for typedefs above.
 440                 # But we output the declaration as well in this case, so we
 441                 # can differentiate it from a typedef.
 442                 structsym = m10.group(1).upper()
 443                 logging.info('%s:%s', structsym, m10.group(2))
 444                 forward_decls[m10.group(2)] = '<%s>\n<NAME>%s</NAME>\n%s%s</%s>\n' % (
 445                     structsym, m10.group(2), line, deprecated, structsym)
 446
 447             elif m11:
 448                 symbol = ''
 449                 decl = line
 450                 level = 0
 451                 in_declaration = m11.group(1)
 452                 logging.info('typedef struct/union "%s"', in_declaration)
 453
 454             # OTHER TYPEDEFS
 455
 456             elif m12:
 457                 logging.info('Found struct/union(*) typedef "%s": "%s"', m12.group(1), line)
 458                 if AddSymbolToList(slist, m12.group(1)):
 459                     decl_list.append('<TYPEDEF>\n<NAME>%s</NAME>\n%s%s</TYPEDEF>\n' % (m12.group(1), deprecated, line))
 460
 461             elif m13:
 462                 if m13.group(2).split()[0] not in ('struct', 'union'):
 463                     logging.info('Found typedef: "%s"', line)
 464                     if AddSymbolToList(slist, m13.group(3)):
 465                         decl_list.append(
 466                             '<TYPEDEF>\n<NAME>%s</NAME>\n%s%s</TYPEDEF>\n' % (m13.group(3), deprecated, line))
 467             elif re.search(r'^\s*typedef\s+', line):
 468                 logging.info('Skipping typedef: "%s"', line)
 469
 470             # VARIABLES (extern'ed variables)
 471
 472             elif m14:
 473                 symbol = m14.group(6)
 474                 line = re.sub(r'^\s*([A-Za-z_]+VAR)\b', r'extern', line)
 475                 decl = line
 476                 logging.info('Possible extern var "%s": "%s"', symbol, decl)
 477                 if AddSymbolToList(slist, symbol):
 478                     decl_list.append('<VARIABLE>\n<NAME>%s</NAME>\n%s%s</VARIABLE>\n' % (symbol, deprecated, decl))
 479
 480             # VARIABLES
 481
 482             elif m15:
 483                 symbol = m15.group(5)
 484                 decl = line
 485                 logging.info('Possible global var" %s": "%s"', symbol, decl)
 486                 if AddSymbolToList(slist, symbol):
 487                     decl_list.append('<VARIABLE>\n<NAME>%s</NAME>\n%s%s</VARIABLE>\n' % (symbol, deprecated, decl))
 488
 489             # G_DECLARE_*
 490
 491             elif m16:
 492                 in_declaration = 'g-declare'
 493                 symbol = 'G_DECLARE_' + m16.group(1)
 494                 decl = line[m16.end():]
 495
 496             # FUNCTIONS
 497
 498             # We assume that functions which start with '_' are private, so
 499             # we skip them.
 500             elif m17:
 501                 ret_type = m17.group(1)
 502                 if m17.group(2):
 503                     ret_type += ' ' + m17.group(2)
 504                 symbol = m17.group(3)
 505                 decl = line[m17.end():]
 506                 logging.info('internal Function: "%s", Returns: "%s""%s"', symbol, m17.group(1), m17.group(2))
 507                 in_declaration = 'function'
 508                 internal = 1
 509                 if line.strip().startswith('G_INLINE_FUNC'):
 510                     logging.info('skip block after inline function')
 511                     # now we we need to skip a whole { } block
 512                     skip_block = 1
 513
 514             elif m18:
 515                 ret_type = m18.group(1)
 516                 if m18.group(2):
 517                     ret_type += ' ' + m18.group(2)
 518                 symbol = m18.group(3)
 519                 decl = line[m18.end():]
 520                 logging.info('Function (1): "%s", Returns: "%s""%s"', symbol, m18.group(1), m18.group(2))
 521                 in_declaration = 'function'
 522                 if line.strip().startswith('G_INLINE_FUNC'):
 523                     logging.info('skip block after inline function')
 524                     # now we we need to skip a whole { } block
 525                     skip_block = 1
 526
 527             # Try to catch function declarations which have the return type on
 528             # the previous line. But we don't want to catch complete functions
 529             # which have been declared G_INLINE_FUNC, e.g. g_bit_nth_lsf in
 530             # glib, or 'static inline' functions.
 531             elif m19:
 532                 symbol = m19.group(1)
 533                 decl = line[m19.end():]
 534
 535                 previous_line_words = previous_line.strip().split()
 536
 537                 if not previous_line.strip().startswith('G_INLINE_FUNC'):
 538                     if not previous_line_words or previous_line_words[0] != 'static':
 539                         #                                           $1                                                                                                   $2
 540                         pm = re.search(r'^\s*(?:\b(?:extern%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|long\s+|short\s+|struct\s+|union\s+|enum\s+)*\w+)((?:\s*(?:\*+|\bconst\b|\bG_CONST_RETURN\b))*)\s*$' %
 541                                        ignore_decorators, previous_line)
 542                         if pm:
 543                             ret_type = pm.group(1)
 544                             if pm.group(2):
 545                                 ret_type += ' ' + pm.group(2)
 546                             logging.info('Function  (2): "%s", Returns: "%s"', symbol, ret_type)
 547                             in_declaration = 'function'
 548                     else:
 549                         logging.info('skip block after inline function')
 550                         # now we we need to skip a whole { } block
 551                         skip_block = 1
 552                         #                                                        $1                                                                                                    $2
 553                         pm = re.search(r'^\s*(?:\b(?:extern|static|inline%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|long\s+|short\s+|struct\s+|union\s+|enum\s+)*\w+)((?:\s*(?:\*+|\bconst\b|\bG_CONST_RETURN\b))*)\s*$' %
 554                                        ignore_decorators, previous_line)
 555                         if pm:
 556                             ret_type = pm.group(1)
 557                             if pm.group(2):
 558                                 ret_type += ' ' + pm.group(2)
 559                             logging.info('Function  (3): "%s", Returns: "%s"', symbol, ret_type)
 560                             in_declaration = 'function'
 561                 else:
 562                     if not previous_line_words or previous_line_words[0] != 'static':
 563                         logging.info('skip block after inline function')
 564                         # now we we need to skip a whole { } block
 565                         skip_block = 1
 566                         #                                                         $1                                                                                                    $2
 567                         pm = re.search(r'^\s*(?:\b(?:extern|G_INLINE_FUNC%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|long\s+|short\s+|struct\s+|union\s+|enum\s+)*\w+)((?:\s*(?:\*+|\bconst\b|\bG_CONST_RETURN\b))*)\s*$' %
 568                                        ignore_decorators, previous_line)
 569                         if pm:
 570                             ret_type = pm.group(1)
 571                             if pm.group(2):
 572                                 ret_type += ' ' + pm.group(2)
 573                             logging.info('Function (4): "%s", Returns: "%s"', symbol, ret_type)
 574                             in_declaration = 'function'
 575
 576             # Try to catch function declarations with the return type and name
 577             # on the previous line(s), and the start of the parameters on this.
 578             elif m20:
 579                 decl = line[m20.end():]
 580                 pm = re.search(
 581                     r'^\s*(?:\b(?:extern|G_INLINE_FUNC%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|enum\s+)*\w+)(\s+\*+|\*+|\s)\s*([A-Za-z]\w*)\s*$' % ignore_decorators, previous_line)
 582                 ppm = re.search(r'^\s*(?:\b(?:extern|G_INLINE_FUNC%s)\s*)*((?:const\s+|G_CONST_RETURN\s+|signed\s+|unsigned\s+|struct\s+|union\s+|enum\s+)*\w+(?:\**\s+\**(?:const|G_CONST_RETURN))?(?:\s+|\s*\*+))\s*$' %
 583                                 ignore_decorators, pre_previous_line)
 584                 if pm:
 585                     ret_type = pm.group(1) + ' ' + pm.group(2)
 586                     symbol = pm.group(3)
 587                     in_declaration = 'function'
 588                     logging.info('Function (5): "%s", Returns: "%s"', symbol, ret_type)
 589
 590                 elif re.search(r'^\s*\w+\s*$', previous_line) and ppm:
 591                     ret_type = ppm.group(1)
 592                     ret_type = re.sub(r'\s*\n', '', ret_type, flags=re.MULTILINE)
 593                     in_declaration = 'function'
 594
 595                     symbol = previous_line
 596                     symbol = re.sub(r'^\s+', '', symbol)
 597                     symbol = re.sub(r'\s*\n', '', symbol, flags=re.MULTILINE)
 598                     logging.info('Function (6): "%s", Returns: "%s"', symbol, ret_type)
 599
 600             #} elsif (m/^extern\s+/) {
 601                 # print "DEBUG: Skipping extern: $_"
 602
 603             # STRUCTS
 604             elif re.search(r'^\s*struct\s+_?(\w+)\s*\*', line):
 605                 # Skip 'struct _<struct_name> *', since it could be a
 606                 # return type on its own line.
 607                 pass
 608             elif m21:
 609                 # We assume that 'struct _<struct_name>' is really the
 610                 # declaration of struct <struct_name>.
 611                 symbol = m21.group(1)
 612                 decl = line
 613                 # we will find the correct level as below we do $level += tr/{//
 614                 level = 0
 615                 in_declaration = 'struct'
 616                 logging.info('Struct(_): "%s"', symbol)
 617
 618             # UNIONS
 619             elif re.search(r'^\s*union\s+_(\w+)\s*\*', line):
 620                 # Skip 'union _<union_name> *' (see above)
 621                 pass
 622             elif m22:
 623                 symbol = m22.group(1)
 624                 decl = line
 625                 level = 0
 626                 in_declaration = 'union'
 627                 logging.info('Union(_): "%s"', symbol)
 628         else:
 629             logging.info('in decl: skip=%s %s', skip_block, line.strip())
 630             # If we were already in the middle of a declaration, we simply add
 631             # the current line onto the end of it.
 632             if skip_block == 0:
 633                 decl += line
 634             else:
 635                 # Remove all nested pairs of curly braces.
 636                 brace_remover = r'{[^{]*}'
 637                 bm = re.search(brace_remover, line)
 638                 while bm:
 639                     line = re.sub(brace_remover, '', line)
 640                     bm = re.search(brace_remover, line)
 641                 # Then hope at most one remains in the line...
 642                 bm = re.search(r'(.*?){', line)
 643                 if bm:
 644                     if skip_block == 1:
 645                         decl += bm.group(1)
 646                     skip_block += 1
 647                 elif '}' in line:
 648                     skip_block -= 1
 649                     if skip_block == 1:
 650                         # this is a hack to detect the end of declaration
 651                         decl += ';'
 652                         skip_block = 0
 653                         logging.info('2: ---')
 654
 655                 else:
 656                     if skip_block == 1:
 657                         decl += line
 658
 659         if in_declaration == "g-declare":
 660             dm = re.search(r'\s*(\w+)\s*,\s*(\w+)\s*,\s*(\w+)\s*,\s*(\w+)\s*,\s*(\w+)\s*\).*$', decl)
 661             # FIXME the original code does s// stuff here and we don't. Is it necessary?
 662             if dm:
 663                 ModuleObjName = dm.group(1)
 664                 module_obj_name = dm.group(2)
 665                 if options.rebuild_types:
 666                     get_types.append(module_obj_name + '_get_type')
 667                 forward_decls[ModuleObjName] = '<STRUCT>\n<NAME>%s</NAME>\n%s</STRUCT>\n' % (ModuleObjName, deprecated)
 668                 if symbol.startswith('G_DECLARE_DERIVABLE'):
 669                     forward_decls[ModuleObjName + 'Class'] = '<STRUCT>\n<NAME>%sClass</NAME>\n%s</STRUCT>\n' % (
 670                         ModuleObjName, deprecated)
 671                 if symbol.startswith('G_DECLARE_INTERFACE'):
 672                     forward_decls[ModuleObjName + 'Interface'] = '<STRUCT>\n<NAME>%sInterface</NAME>\n%s</STRUCT>\n' % (
 673                         ModuleObjName, deprecated)
 674                 in_declaration = ''
 675
 676         # Note that sometimes functions end in ') G_GNUC_PRINTF (2, 3);' or
 677         # ') __attribute__ (...);'.
 678         if in_declaration == 'function':
 679             regex = r'\)\s*(G_GNUC_.*|.*DEPRECATED.*%s\s*|__attribute__\s*\(.*\)\s*)*;.*$' % ignore_decorators
 680             pm = re.search(regex, decl, flags=re.MULTILINE)
 681             if pm:
 682                 logging.info('scrubbing:[%s]', decl)
 683                 decl = re.sub(regex, '', decl, flags=re.MULTILINE)
 684                 logging.info('scrubbed:[%s]', decl)
 685                 if internal == 0:
 686                     decl = re.sub(r'/\*.*?\*/', '', decl, flags=re.MULTILINE)   # remove comments.
 687                     decl = re.sub(r'\s*\n\s*(?!$)', ' ', decl, flags=re.MULTILINE)
 688                                   # consolidate whitespace at start/end of lines.
 689                     decl = decl.strip()
 690                     ret_type = re.sub(r'/\*.*?\*/', '', ret_type)               # remove comments in ret type.
 691                     if AddSymbolToList(slist, symbol):
 692                         decl_list.append('<FUNCTION>\n<NAME>%s</NAME>\n%s<RETURNS>%s</RETURNS>\n%s\n</FUNCTION>\n' %
 693                                          (symbol, deprecated, ret_type, decl))
 694                         if options.rebuild_types:
 695                             # check if this looks like a get_type function and if so remember
 696                             if symbol.endswith('_get_type') and 'GType' in ret_type and re.search(r'^(void|)$', decl):
 697                                 logging.info(
 698                                     "Adding get-type: [%s] [%s] [%s]\tfrom %s", ret_type, symbol, decl, input_file)
 699                                 get_types.append(symbol)
 700                 else:
 701                     internal = 0
 702                 deprecated_conditional_nest = int(deprecated_conditional_nest)
 703                 in_declaration = ''
 704                 skip_block = 0
 705
 706         if in_declaration == 'user_function':
 707             if re.search(r'\).*$', decl):
 708                 decl = re.sub(r'\).*$', '', decl)
 709                 if AddSymbolToList(slist, symbol):
 710                     decl_list.append('<USER_FUNCTION>\n<NAME>%s</NAME>\n%s<RETURNS>%s</RETURNS>\n%s</USER_FUNCTION>\n' %
 711                                      (symbol, deprecated, ret_type, decl))
 712                 deprecated_conditional_nest = int(deprecated_conditional_nest)
 713                 in_declaration = ''
 714
 715         if in_declaration == 'macro':
 716             if not re.search(r'\\\s*$', decl):
 717                 if internal == 0:
 718                     if AddSymbolToList(slist, symbol):
 719                         decl_list.append('<MACRO>\n<NAME>%s</NAME>\n%s%s</MACRO>\n' % (symbol, deprecated, decl))
 720                 else:
 721                     internal = 0
 722                 deprecated_conditional_nest = int(deprecated_conditional_nest)
 723                 in_declaration = ''
 724
 725         if in_declaration == 'enum':
 726             em = re.search(r'\}\s*(\w+)?;\s*$', decl)
 727             if em:
 728                 if symbol == '':
 729                     symbol = em.group(1)
 730                 if AddSymbolToList(slist, symbol):
 731                     decl_list.append('<ENUM>\n<NAME>%s</NAME>\n%s%s</ENUM>\n' % (symbol, deprecated, decl))
 732                 deprecated_conditional_nest = int(deprecated_conditional_nest)
 733                 in_declaration = ''
 734
 735         # We try to handle nested stucts/unions, but unmatched brackets in
 736         # comments will cause problems.
 737         if in_declaration == 'struct' or in_declaration == 'union':
 738             sm = re.search(r'\n\}\s*(\w*);\s*$', decl)
 739             if level <= 1 and sm:
 740                 if symbol == '':
 741                     symbol = sm.group(1)
 742
 743                 bm = re.search(r'^(\S+)(Class|Iface|Interface)\b', symbol)
 744                 if bm:
 745                     objectname = bm.group(1)
 746                     logging.info('Found object: "%s"', objectname)
 747                     title = '<TITLE>%s</TITLE>' % objectname
 748
 749                 logging.info('Store struct: "%s"', symbol)
 750                 if AddSymbolToList(slist, symbol):
 751                     structsym = in_declaration.upper()
 752                     decl_list.append('<%s>\n<NAME>%s</NAME>\n%s%s</%s>\n' %
 753                                      (structsym, symbol, deprecated, decl, structsym))
 754                     if symbol in forward_decls:
 755                         del forward_decls[symbol]
 756                 deprecated_conditional_nest = int(deprecated_conditional_nest)
 757                 in_declaration = ''
 758             else:
 759                 # We use tr to count the brackets in the line, and adjust
 760                 # $level accordingly.
 761                 level += line.count('{')
 762                 level -= line.count('}')
 763                 logging.info('struct/union level : %d', level)
 764
 765         pre_previous_line = previous_line
 766         previous_line = line
 767
 768     # print remaining forward declarations
 769     for symbol in sorted(forward_decls.keys()):
 770         if forward_decls[symbol]:
 771             AddSymbolToList(slist, symbol)
 772             decl_list.append(forward_decls[symbol])
 773
 774     # add title
 775     slist = [title] + slist
 776
 777     logging.info("Scanning %s done", input_file)
 778
 779     # Try to separate the standard macros and functions, placing them at the
 780     # end of the current section, in a subsection named 'Standard'.
 781     # do this in a loop to catch object, enums and flags
 782     klass = lclass = prefix = lprefix = None
 783     standard_decl = []
 784     liststr = '\n'.join(s for s in slist if s) + '\n'
 785     while True:
 786         m = re.search(r'^(\S+)_IS_(\S*)_CLASS\n', liststr, flags=re.MULTILINE)
 787         m2 = re.search(r'^(\S+)_IS_(\S*)\n', liststr, flags=re.MULTILINE)
 788         m3 = re.search(r'^(\S+?)_(\S*)_get_type\n', liststr, flags=re.MULTILINE)
 789         if m:
 790             prefix = m.group(1)
 791             lprefix = prefix.lower()
 792             klass = m.group(2)
 793             lclass = klass.lower()
 794             logging.info("Found gobject type '%s_%s' from is_class macro", prefix, klass)
 795         elif m2:
 796             prefix = m2.group(1)
 797             lprefix = prefix.lower()
 798             klass = m2.group(2)
 799             lclass = klass.lower()
 800             logging.info("Found gobject type '%s_%s' from is_ macro", prefix, klass)
 801         elif m3:
 802             lprefix = m3.group(1)
 803             prefix = lprefix.upper()
 804             lclass = m3.group(2)
 805             klass = lclass.upper()
 806             logging.info("Found gobject type '%s_%s' from get_type function", prefix, klass)
 807         else:
 808             break
 809
 810         cclass = lclass
 811         cclass = cclass.replace('_', '')
 812         mtype = lprefix + cclass
 813
 814         liststr, standard_decl = replace_once(liststr, standard_decl, r'^%sPrivate\n' % mtype)
 815
 816         # We only leave XxYy* in the normal section if they have docs
 817         if mtype not in doc_comments:
 818             logging.info("  Hide instance docs for %s", mtype)
 819             liststr, standard_decl = replace_once(liststr, standard_decl, r'^%s\n' % mtype)
 820
 821         if mtype + 'class' not in doc_comments:
 822             logging.info("  Hide class docs for %s", mtype)
 823             liststr, standard_decl = replace_once(liststr, standard_decl, r'^%sClass\n' % mtype)
 824
 825         if mtype + 'interface' not in doc_comments:
 826             logging.info("  Hide iface docs for %s", mtype)
 827             liststr, standard_decl = replace_once(liststr, standard_decl, r'%sInterface\n' % mtype)
 828
 829         if mtype + 'iface' not in doc_comments:
 830             logging.info("  Hide iface docs for " + mtype)
 831             liststr, standard_decl = replace_once(liststr, standard_decl, r'%sIface\n' % mtype)
 832
 833         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_IS_%s\n' % klass)
 834         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_TYPE_%s\n' % klass)
 835         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_%s_get_type\n' % lclass)
 836         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_%s_CLASS\n' % klass)
 837         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_IS_%s_CLASS\n' % klass)
 838         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_%s_GET_CLASS\n' % klass)
 839         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_%s_GET_IFACE\n' % klass)
 840         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_%s_GET_INTERFACE\n' % klass)
 841         # We do this one last, otherwise it tends to be caught by the IS_$class macro
 842         liststr, standard_decl = replace_all(liststr, standard_decl, r'^\S+_%s\n' % klass)
 843
 844     logging.info('Decl:%s---', liststr)
 845     logging.info('Std :%s---', ''.join(sorted(standard_decl)))
 846     if len(standard_decl):
 847         # sort the symbols
 848         liststr += '<SUBSECTION Standard>\n' + ''.join(sorted(standard_decl))
 849
 850     if liststr != '':
 851         if file_basename not in section_list:
 852             section_list[file_basename] = ''
 853         section_list[file_basename] += "<SECTION>\n<FILE>%s</FILE>\n%s</SECTION>\n\n" % (file_basename, liststr)
 854
 855
 856 def replace_once(liststr, standard_decl, regex):
 857     mre = re.search(regex, liststr,  flags=re.IGNORECASE | re.MULTILINE)
 858     if mre:
 859         standard_decl.append(mre.group(0))
 860         liststr = re.sub(regex, '', liststr, flags=re.IGNORECASE | re.MULTILINE)
 861     return liststr, standard_decl
 862
 863
 864 def replace_all(liststr, standard_decl, regex):
 865     mre = re.search(regex, liststr, flags=re.MULTILINE)
 866     while mre:
 867         standard_decl.append(mre.group(0))
 868         liststr = re.sub(regex, '', liststr, flags=re.MULTILINE)
 869         mre = re.search(regex, liststr, flags=re.MULTILINE)
 870     return liststr, standard_decl
 871
 872
 873 def AddSymbolToList(slist, symbol):
 874     """ Adds symbol to list of declaration if not already present.
 875
 876     Args:
 877         slist: The list of symbols.
 878         symbol: The symbol to add to the list.
 879     """
 880     if symbol in slist:
 881          # logging.info('Symbol %s already in list. skipping', symbol)
 882          # we return False to skip outputting another entry to -decl.txt
 883          # this is to avoid redeclarations (e.g. in conditional sections).
 884         return False
 885     slist.append(symbol)
 886     return True