bin/update_pch

   1 #! /usr/bin/env python
   2 # -*- Mode: python; tab-width: 4; indent-tabs-mode: t -*-
   3 #
   4 # This file is part of the LibreOffice project.
   5 #
   6 # This Source Code Form is subject to the terms of the Mozilla Public
   7 # License, v. 2.0. If a copy of the MPL was not distributed with this
   8 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   9 #
  10
  11 """
  12 This script generates precompiled headers for a given
  13 module and library.
  14
  15 Given a gmake makefile that belongs to some LO module:
  16 1) Process the makefile to find source files (process_makefile).
  17 2) For every source file, find all includes (process_source).
  18 3) Uncommon and rare includes are filtered (remove_rare).
  19 4) Conflicting headers are excluded (filter_ignore).
  20 5) Local files to the source are excluded (Filter_Local).
  21 6) Fixup missing headers that sources expect (fixup).
  22 7) The resulting includes are sorted by category (sort_by_category).
  23 8) The pch file is generated (generate).
  24 """
  25
  26 import sys
  27 import re
  28 import os
  29 import unittest
  30
  31 CUTOFF = 1
  32 EXCLUDE_MODULE = False
  33 EXCLUDE_LOCAL = False
  34 EXCLUDE_SYSTEM = True
  35 SILENT = False
  36
  37 # System includes: oox, sal, sd, svl, vcl
  38
  39 INCLUDE = False
  40 EXCLUDE = True
  41 DEFAULTS = \
  42 {
  43 #    module.library : (min, system, module, local), best time
  44     'accessibility.acc'                 : ( 4, EXCLUDE, INCLUDE, INCLUDE), #   7.8
  45     'basctl.basctl'                     : ( 3, EXCLUDE, INCLUDE, EXCLUDE), #  11.9
  46     'basegfx.basegfx'                   : ( 3, EXCLUDE, EXCLUDE, INCLUDE), #   3.8
  47     'basic.sb'                          : ( 2, EXCLUDE, EXCLUDE, INCLUDE), #  10.7
  48     'chart2.chartcontroller'            : ( 6, EXCLUDE, INCLUDE, INCLUDE), #  18.4
  49     'chart2.chartcore'                  : ( 3, EXCLUDE, EXCLUDE, INCLUDE), #  22.5
  50     'chart2.chartopengl'                : (12, EXCLUDE, EXCLUDE, EXCLUDE), #   5.3
  51     'comphelper.comphelper'             : ( 4, EXCLUDE, INCLUDE, INCLUDE), #   7.6
  52     'configmgr.configmgr'               : ( 6, EXCLUDE, INCLUDE, INCLUDE), #   6.0
  53     'connectivity.ado'                  : ( 2, EXCLUDE, EXCLUDE, EXCLUDE), #   6.4
  54     'connectivity.calc'                 : ( 2, EXCLUDE, EXCLUDE, EXCLUDE), #   4.6
  55     'connectivity.dbase'                : ( 2, EXCLUDE, INCLUDE, INCLUDE), #   5.2
  56     'connectivity.dbpool2'              : ( 5, EXCLUDE, INCLUDE, EXCLUDE), #   3.0
  57     'connectivity.dbtools'              : ( 2, EXCLUDE, EXCLUDE, INCLUDE), #   0.8
  58     'connectivity.file'                 : ( 2, EXCLUDE, INCLUDE, EXCLUDE), #   5.1
  59     'connectivity.firebird_sdbc'        : ( 2, EXCLUDE, EXCLUDE, EXCLUDE), #   5.1
  60     'connectivity.flat'                 : ( 2, EXCLUDE, INCLUDE, INCLUDE), #   4.6
  61     'connectivity.mysql'                : ( 4, EXCLUDE, INCLUDE, EXCLUDE), #   3.4
  62     'connectivity.odbc'                 : ( 2, EXCLUDE, EXCLUDE, INCLUDE), #   5.0
  63     'connectivity.postgresql-sdbc-impl' : ( 3, EXCLUDE, EXCLUDE, EXCLUDE), #   6.7
  64     'cppcanvas.cppcanvas'               : (11, EXCLUDE, INCLUDE, INCLUDE), #   4.8
  65     'cppuhelper.cppuhelper'             : ( 3, EXCLUDE, EXCLUDE, EXCLUDE), #   4.6
  66     'cui.cui'                           : ( 8, EXCLUDE, INCLUDE, EXCLUDE), #  19.7
  67     'dbaccess.dba'                      : ( 6, EXCLUDE, INCLUDE, INCLUDE), #  13.8
  68     'dbaccess.dbaxml'                   : ( 2, EXCLUDE, EXCLUDE, EXCLUDE), #   6.5
  69     'dbaccess.dbmm'                     : (10, EXCLUDE, INCLUDE, EXCLUDE), #   4.3
  70     'dbaccess.dbu'                      : (12, EXCLUDE, EXCLUDE, EXCLUDE), #  23.6
  71     'dbaccess.sdbt'                     : ( 1, EXCLUDE, INCLUDE, EXCLUDE), #   2.9
  72     'desktop.deployment'                : ( 3, EXCLUDE, EXCLUDE, EXCLUDE), #   6.1
  73     'desktop.deploymentgui'             : ( 3, EXCLUDE, EXCLUDE, EXCLUDE), #   5.7
  74     'desktop.deploymentmisc'            : ( 3, EXCLUDE, EXCLUDE, EXCLUDE), #   3.4
  75     'desktop.sofficeapp'                : ( 6, EXCLUDE, INCLUDE, INCLUDE), #   6.5
  76     'drawinglayer.drawinglayer'         : ( 4, EXCLUDE, EXCLUDE, EXCLUDE), #   7.4
  77     'editeng.editeng'                   : ( 5, EXCLUDE, INCLUDE, EXCLUDE), #  13.0
  78     'forms.frm'                         : ( 2, EXCLUDE, EXCLUDE, EXCLUDE), #  14.2
  79     'framework.fwe'                     : (10, EXCLUDE, INCLUDE, EXCLUDE), #   5.5
  80     'framework.fwi'                     : ( 9, EXCLUDE, INCLUDE, EXCLUDE), #   3.4
  81     'framework.fwk'                     : ( 7, EXCLUDE, INCLUDE, INCLUDE), #  14.8
  82     'framework.fwl'                     : ( 5, EXCLUDE, INCLUDE, INCLUDE), #   5.1
  83     'hwpfilter.hwp'                     : ( 3, EXCLUDE, INCLUDE, INCLUDE), #   6.0
  84     'lotuswordpro.lwpft'                : ( 2, EXCLUDE, EXCLUDE, EXCLUDE), #  11.6
  85     'oox.oox'                           : ( 6, EXCLUDE, EXCLUDE, INCLUDE), #  28.2
  86     'package.package2'                  : ( 3, EXCLUDE, INCLUDE, INCLUDE), #   4.5
  87     'package.xstor'                     : ( 2, EXCLUDE, INCLUDE, EXCLUDE), #   3.8
  88     'reportdesign.rpt'                  : ( 9, EXCLUDE, INCLUDE, INCLUDE), #   9.4
  89     'reportdesign.rptui'                : ( 4, EXCLUDE, INCLUDE, INCLUDE), #  13.1
  90     'reportdesign.rptxml'               : ( 2, EXCLUDE, EXCLUDE, INCLUDE), #   7.6
  91     'sal.sal'                           : ( 2, EXCLUDE, EXCLUDE, INCLUDE), #   4.2
  92     'sc.sc'                             : (12, EXCLUDE, INCLUDE, INCLUDE), #  92.6
  93     'sc.scfilt'                         : ( 4, EXCLUDE, EXCLUDE, INCLUDE), #  39.9
  94     'sc.scui'                           : ( 1, EXCLUDE, EXCLUDE, INCLUDE), #  15.0
  95     'sc.vbaobj'                         : ( 1, EXCLUDE, EXCLUDE, INCLUDE), #  17.3
  96     'sd.sd'                             : ( 4, EXCLUDE, EXCLUDE, INCLUDE), #  47.4
  97     'sd.sdui'                           : ( 4, EXCLUDE, INCLUDE, INCLUDE), #   9.4
  98     'sdext.PresentationMinimizer'       : ( 2, EXCLUDE, INCLUDE, INCLUDE), #   4.1
  99     'sdext.PresenterScreen'             : ( 2, EXCLUDE, INCLUDE, EXCLUDE), #   7.1
 100     'sfx2.sfx'                          : ( 3, EXCLUDE, EXCLUDE, EXCLUDE), #  27.4
 101     'slideshow.slideshow'               : ( 4, EXCLUDE, INCLUDE, EXCLUDE), #  10.8
 102     'sot.sot'                           : ( 5, EXCLUDE, EXCLUDE, INCLUDE), #   3.1
 103     'starmath.sm'                       : ( 5, EXCLUDE, EXCLUDE, INCLUDE), #  10.9
 104     'svgio.svgio'                       : ( 8, EXCLUDE, EXCLUDE, INCLUDE), #   4.3
 105     'svl.svl'                           : ( 6, EXCLUDE, EXCLUDE, EXCLUDE), #   7.6
 106     'svtools.svt'                       : ( 4, EXCLUDE, INCLUDE, EXCLUDE), #  17.6
 107     'svx.svx'                           : ( 3, EXCLUDE, EXCLUDE, INCLUDE), #  20.7
 108     'svx.svxcore'                       : ( 7, EXCLUDE, INCLUDE, EXCLUDE), #  37.0
 109     'sw.msword'                         : ( 4, EXCLUDE, INCLUDE, INCLUDE), #  22.4
 110     'sw.sw'                             : ( 7, EXCLUDE, EXCLUDE, INCLUDE), # 129.6
 111     'sw.swui'                           : ( 3, EXCLUDE, INCLUDE, INCLUDE), #  26.1
 112     'sw.vbaswobj'                       : ( 4, EXCLUDE, INCLUDE, INCLUDE), #  13.1
 113     'tools.tl'                          : ( 5, EXCLUDE, EXCLUDE, EXCLUDE), #   4.2
 114     'unotools.utl'                      : ( 3, EXCLUDE, EXCLUDE, INCLUDE), #   7.0
 115     'unoxml.unoxml'                     : ( 1, EXCLUDE, EXCLUDE, EXCLUDE), #   4.6
 116     'uui.uui'                           : ( 4, EXCLUDE, EXCLUDE, EXCLUDE), #   4.9
 117     'vbahelper.msforms'                 : ( 3, EXCLUDE, INCLUDE, INCLUDE), #   5.2
 118     'vbahelper.vbahelper'               : ( 3, EXCLUDE, EXCLUDE, INCLUDE), #   7.0
 119     'vcl.vcl'                           : ( 6, EXCLUDE, INCLUDE, INCLUDE), #  35.7
 120     'writerfilter.writerfilter'         : ( 5, EXCLUDE, EXCLUDE, EXCLUDE), #  19.7/27.3
 121     'xmloff.xo'                         : ( 7, EXCLUDE, INCLUDE, INCLUDE), #  22.1
 122     'xmloff.xof'                        : ( 1, EXCLUDE, EXCLUDE, INCLUDE), #   4.4
 123     'xmlscript.xmlscript'               : ( 4, EXCLUDE, EXCLUDE, INCLUDE), #   3.6
 124     'xmlsecurity.xmlsecurity'           : ( 6, EXCLUDE, INCLUDE, INCLUDE), #   5.1
 125     'xmlsecurity.xsec_fw'               : ( 2, EXCLUDE, INCLUDE, EXCLUDE), #   2.7
 126     'xmlsecurity.xsec_xmlsec'           : ( 2, EXCLUDE, INCLUDE, INCLUDE), #   4.4
 127     'xmlsecurity.xsec_gpg'              : ( 2, EXCLUDE, INCLUDE, INCLUDE), #   ?
 128 }
 129
 130 def remove_rare(raw, min_use=-1):
 131     """ Remove headers not commonly included.
 132         The minimum threshold is min_use.
 133     """
 134     # The minimum number of times a header
 135     # must be included to be in the PCH.
 136     min_use = min_use if min_use >= 0 else CUTOFF
 137
 138     out = []
 139     if not raw or not len(raw):
 140         return out
 141
 142     inc = sorted(raw)
 143     last = inc[0]
 144     count = 1
 145     for x in range(1, len(inc)):
 146         i = inc[x]
 147         if i == last:
 148             count += 1
 149         else:
 150             if count >= min_use:
 151                 out.append(last)
 152             last = i
 153             count = 1
 154
 155     # Last group.
 156     if count >= min_use:
 157         out.append(last)
 158
 159     return out
 160
 161 def process_list(list, callable):
 162     """ Given a list and callable
 163         we pass each entry through
 164         the callable and only add to
 165         the output if not blank.
 166     """
 167     out = []
 168     for i in list:
 169         line = callable(i)
 170         if line and len(line):
 171             out.append(line)
 172     return out
 173
 174 def find_files(path, recurse=True):
 175     list = []
 176     for root, dir, files in os.walk(path):
 177         list += map(lambda x: os.path.join(root, x), files)
 178     return list
 179
 180 def get_filename(line):
 181     """ Strips the line from the
 182         '#include' and angled brakets
 183         and return the filename only.
 184     """
 185     if not len(line) or line[0] != '#':
 186         return line
 187     return re.sub(r'(.*#include\s*)<(.*)>(.*)', r'\2', line)
 188
 189 def is_c_runtime(inc):
 190     """ Heuristic-based detection of C/C++
 191         runtime headers.
 192         They are all-lowercase, with .h or
 193         no extension, filename only.
 194     """
 195     inc = get_filename(inc)
 196
 197     if inc.endswith('.hxx') or inc.endswith('.hpp'):
 198         return False
 199
 200     for c in inc:
 201         if c == '/':
 202             return False
 203         if c == '.':
 204             return inc.endswith('.h')
 205         if c.isupper():
 206             return False
 207
 208     return True
 209
 210 def sanitize(raw):
 211     """ There are two forms of includes,
 212         those with <> and "".
 213         Technically, the difference is that
 214         the compiler can use an internal
 215         representation for an angled include,
 216         such that it doesn't have to be a file.
 217         For our purposes, there is no difference.
 218         Here, we convert everything to angled.
 219     """
 220     if not raw or not len(raw):
 221         return ''
 222     raw = raw.strip()
 223     if not len(raw):
 224         return ''
 225     return re.sub(r'(.*#include\s*)\"(.*)\"(.*)', r'#include <\2>', raw)
 226
 227 class Filter_Local(object):
 228     """ Filter headers local to a module.
 229         allow_public: allows include/module/file.hxx
 230                       #include <module/file.hxx>
 231         allow_module: allows module/inc/file.hxx
 232                       #include <file.hxx>
 233         allow_locals: allows module/source/file.hxx and
 234                              module/source/inc/file.hxx
 235                       #include <file.hxx>
 236     """
 237     def __init__(self, root, module, allow_public=True, allow_module=True, allow_locals=True):
 238         self.root = root
 239         self.module = module
 240         self.allow_public = allow_public
 241         self.allow_module = allow_module
 242         self.allow_locals = allow_locals
 243         self.public_prefix = '<' + self.module + '/'
 244
 245         all = find_files(os.path.join(root, module))
 246         self.module = []
 247         self.locals = []
 248         mod_prefix = module + '/inc/'
 249         for i in all:
 250             if mod_prefix in i:
 251                 self.module.append(i)
 252             else:
 253                 self.locals.append(i)
 254
 255     def is_public(self, line):
 256         return self.public_prefix in line
 257
 258     def is_module(self, line):
 259         """ Returns True if in module/inc/... """
 260         filename = get_filename(line)
 261         for i in self.module:
 262             if i.endswith(filename):
 263                 return True
 264         return False
 265
 266     def is_local(self, line):
 267         """ Returns True if in module/source/... """
 268         filename = get_filename(line)
 269         for i in self.locals:
 270             if i.endswith(filename):
 271                 return True
 272         return False
 273
 274     def is_external(self, line):
 275         return is_c_runtime(line) and \
 276                not self.is_public(line) and \
 277                not self.is_module(line) and \
 278                not self.is_local(line)
 279
 280     def find_local_file(self, line):
 281         """ Finds the header file in the module dir,
 282             but doesn't validate.
 283         """
 284         filename = get_filename(line)
 285         for i in self.locals:
 286             if i.endswith(filename):
 287                 return i
 288         for i in self.module:
 289             if i.endswith(filename):
 290                 return i
 291         return None
 292
 293     def proc(self, line):
 294         assert line and len(line)
 295         assert line[0] != '<' and line[0] != '#'
 296
 297         filename = get_filename(line)
 298
 299         # Local with relative path.
 300         if filename.startswith('..'):
 301             # Exclude for now as we don't have cxx path.
 302             return ''
 303
 304         # Locals are included first (by the compiler).
 305         if self.is_local(filename):
 306             return line if self.allow_locals and '/inc/' in filename else ''
 307
 308         # Module headers are next.
 309         if self.is_module(filename):
 310             return line if self.allow_module else ''
 311
 312         # Public headers are last.
 313         if self.is_public(line):
 314             return line if self.allow_public else ''
 315
 316         # Leave out potentially unrelated files local
 317         # to some other module we can't include directly.
 318         if '/' not in filename and not self.is_external(filename):
 319             return ''
 320
 321         # Unfiltered.
 322         return line
 323
 324 def filter_ignore(line, module):
 325     """ Filters includes from known
 326         problematic ones.
 327         Expects sanitized input.
 328     """
 329     assert line and len(line)
 330
 331     # Always include files without extension.
 332     if '.' not in line:
 333         return line
 334
 335     # Extract filenames for ease of comparison.
 336     line = get_filename(line)
 337
 338     # Filter out all files that are not normal headers.
 339     if not line.endswith('.h') and \
 340        not line.endswith('.hxx') and \
 341        not line.endswith('.hpp') and \
 342        not line.endswith('.hdl'):
 343        return ''
 344
 345     ignore_list = [
 346             'LibreOfficeKit/LibreOfficeKitEnums.h', # Needs special directives
 347             'LibreOfficeKit/LibreOfficeKitTypes.h', # Needs special directives
 348             'jerror.h',     # c++ unfriendly
 349             'jpeglib.h',    # c++ unfriendly
 350             'boost/spirit/include/classic_core.hpp', # depends on BOOST_SPIRIT_DEBUG
 351             'svtools/editimplementation.hxx' # no direct include
 352         ]
 353
 354     if module == 'accessibility':
 355         ignore_list += [
 356             # STR_SVT_ACC_LISTENTRY_SELCTED_STATE redefined from svtools.hrc
 357             'accessibility/extended/textwindowaccessibility.hxx',
 358             ]
 359     if module == 'basic':
 360         ignore_list += [
 361             'basic/vbahelper.hxx',
 362             ]
 363     if module == 'connectivity':
 364         ignore_list += [
 365             'com/sun/star/beans/PropertyAttribute.hpp', # OPTIONAL defined via objbase.h
 366             'com/sun/star/sdbcx/Privilege.hpp', # DELETE defined via objbase.h
 367             ]
 368     if module == 'reportdesign':
 369         ignore_list += [
 370             'editeng/eeitemid.hxx', # macro redefined in ui/misc/UITools.cxx
 371             ]
 372     if module == 'sc':
 373         ignore_list += [
 374             'progress.hxx', # special directives
 375             'scslots.hxx',  # special directives
 376            ]
 377     if module == 'sd':
 378         ignore_list += [
 379             'sdgslots.hxx', # special directives
 380             'sdslots.hxx',  # special directives
 381            ]
 382     if module == 'sfx2':
 383         ignore_list += [
 384             'sfx2/recentdocsview.hxx', # Redefines ApplicationType defined in objidl.h
 385             'sfx2/sidebar/Sidebar.hxx',
 386             'sfx2/sidebar/UnoSidebar.hxx',
 387             'sfxslots.hxx', # externally defined types
 388             ]
 389     if module == 'sot':
 390         ignore_list += [
 391             'sysformats.hxx',   # Windows headers
 392             ]
 393     if module == 'vcl':
 394         ignore_list += [
 395             'accmgr.hxx',   # redefines ImplAccelList
 396             'image.h',
 397             'jobset.h',
 398             'opengl/gdiimpl.hxx',
 399             'opengl/salbmp.hxx',
 400             'openglgdiimpl',   # ReplaceTextA
 401             'printdlg.hxx',
 402             'salinst.hxx',  # GetDefaultPrinterA
 403             'salprn.hxx',   # SetPrinterDataA
 404             'vcl/jobset.hxx',
 405             'vcl/oldprintadaptor.hxx',
 406             'vcl/opengl/OpenGLContext.hxx',
 407             'vcl/opengl/OpenGLHelper.hxx',  # Conflicts with X header on *ix
 408             'vcl/print.hxx',
 409             'vcl/prntypes.hxx', # redefines Orientation from filter/jpeg/Exif.hxx
 410             'vcl/sysdata.hxx',
 411             ]
 412     if module == 'xmloff':
 413         ignore_list += [
 414             'SchXMLExport.hxx', # SchXMLAutoStylePoolP.hxx not found
 415             'SchXMLImport.hxx', # enums redefined in draw\sdxmlimp_impl.hxx
 416             'XMLEventImportHelper.hxx', # NameMap redefined in XMLEventExport.hxx
 417             'xmloff/XMLEventExport.hxx', # enums redefined
 418             ]
 419     if module == 'xmlsecurity':
 420         ignore_list += [
 421             'xmlsec/*',
 422             'xmlsecurity/xmlsec-wrapper.h',
 423             ]
 424
 425     for i in ignore_list:
 426         if line.startswith(i):
 427             return ''
 428         if i[0] == '*' and line.endswith(i[1:]):
 429             return ''
 430         if i[-1] == '*' and line.startswith(i[:-1]):
 431             return ''
 432
 433     return line
 434
 435 def fixup(includes, module):
 436     """ Here we add any headers
 437         necessary in the pch.
 438         These could be known to be very
 439         common but for technical reasons
 440         left out of the pch by this generator.
 441         Or, they could be missing from the
 442         source files where they are used
 443         (probably because they had been
 444         in the old pch, they were missed).
 445         Also, these could be headers
 446         that make the build faster but
 447         aren't added automatically.
 448     """
 449     fixes = []
 450     def append(inc):
 451         # Add a space to exclude from
 452         # ignore bisecting.
 453         line = ' #include <{}>'.format(inc)
 454         try:
 455             i = fixes.index(inc)
 456             fixes[i] = inc
 457         except:
 458             fixes.append(inc)
 459
 460     if module == 'basctl':
 461         if 'basslots.hxx' in includes:
 462             append('sfx2/msg.hxx')
 463
 464     #if module == 'sc':
 465     #    if 'scslots.hxx' in includes:
 466     #        append('sfx2/msg.hxx')
 467     return fixes
 468
 469 def sort_by_category(list, module, filter_local):
 470     """ Move all 'system' headers first.
 471         Core files of osl, rtl, sal, next.
 472         Everything non-module-specific third.
 473         Last, module-specific headers.
 474     """
 475     sys = []
 476     boo = []
 477     cor = []
 478     rst = []
 479     mod = []
 480
 481     prefix = '<' + module + '/'
 482     for i in list:
 483         if is_c_runtime(i):
 484             sys.append(i)
 485         elif '<boost/' in i:
 486             boo.append(i)
 487         elif '<osl' in i or '<rtl' in i or '<sal' in i or '<vcl' in i:
 488             cor.append(i)
 489         elif prefix in i:
 490             mod.append(i)
 491         else:
 492             rst.append(i)
 493
 494     out = []
 495     out += sorted(sys)
 496     out += sorted(boo)
 497     out += sorted(cor)
 498     out += sorted(rst)
 499     out += sorted(mod)
 500     return out
 501
 502 def parse_makefile(groups, lines, lineno, lastif, ifstack):
 503
 504     inobjects = False
 505     inelse = False
 506     os_cond_re = re.compile('(ifeq|ifneq)\s*\(\$\(OS\)\,(\w*)\)')
 507
 508     line = lines[lineno]
 509     if line.startswith('if'):
 510         lastif = line
 511         if ifstack == 0:
 512             # Correction if first line is an if.
 513             lineno = parse_makefile(groups, lines, lineno, line, ifstack+1)
 514     else:
 515         lineno -= 1
 516
 517     while lineno + 1 < len(lines):
 518         lineno += 1
 519         line = lines[lineno].strip()
 520         line = line.rstrip('\\').strip()
 521         #print('line #{}: {}'.format(lineno, line))
 522         if len(line) == 0:
 523             continue
 524
 525         if line == '))':
 526             inobjects = False
 527         elif 'add_exception_objects' in line or \
 528              'add_cxxobject' in line:
 529              inobjects = True
 530              #print('inobjects')
 531              #if ifstack and not SILENT:
 532                 #sys.stderr.write('Sources in a conditional, ignoring for now.\n')
 533         elif line.startswith('if'):
 534             lineno = parse_makefile(groups, lines, lineno, line, ifstack+1)
 535             continue
 536         elif line.startswith('endif'):
 537             if ifstack:
 538                 return lineno
 539             continue
 540         elif line.startswith('else'):
 541             inelse = True
 542         elif inobjects:
 543             if EXCLUDE_SYSTEM and ifstack:
 544                 continue
 545             file = line + '.cxx'
 546             if ',' in line or '(' in line or ')' in line:
 547                 #print('passing: ' + line)
 548                 pass # $if() probably, or something similar
 549             else:
 550                 osname = ''
 551                 if lastif:
 552                     if 'filter' in lastif:
 553                         # We can't grok filter, yet.
 554                         continue
 555                     match = os_cond_re.match(lastif)
 556                     if not match:
 557                         # We only support OS conditionals.
 558                         continue
 559                     in_out = match.group(1)
 560                     osname = match.group(2) if match else ''
 561                     if (in_out == 'ifneq' and not inelse) or \
 562                        (in_out == 'ifeq' and inelse):
 563                         osname = '!' + osname
 564
 565                 if osname not in groups:
 566                     groups[osname] = []
 567                 groups[osname].append(file)
 568
 569     return groups
 570
 571 def process_makefile(root, module, makefile):
 572     """ Parse a gmake makefile and extract
 573         source filenames from it.
 574     """
 575
 576     filename = os.path.join(os.path.join(root, module), makefile)
 577     if not os.path.isfile(filename):
 578         sys.stderr.write('Error: Module {} has no makefile at {}.'.format(module, filename))
 579
 580     groups = {'':[], 'ANDROID':[], 'IOS':[], 'WNT':[], 'LINUX':[], 'MACOSX':[]}
 581
 582     with open(filename, 'r') as f:
 583         lines = f.readlines()
 584         groups = parse_makefile(groups, lines, lineno=0, lastif=None, ifstack=0)
 585
 586     return groups
 587
 588 def process_source(root, module, filename, maxdepth=0):
 589     """ Process a source file to extract
 590         included headers.
 591         For now, skip on compiler directives.
 592         maxdepth is used when processing headers
 593         which typically have protecting ifndef.
 594     """
 595
 596     ifdepth = 0
 597     lastif = ''
 598     raw_includes = []
 599     with open(filename, 'r') as f:
 600         for line in f:
 601             line = line.strip()
 602             if line.startswith('#if'):
 603                 ifdepth += 1
 604                 lastif = line
 605             elif line.startswith('#endif'):
 606                 ifdepth -= 1
 607                 lastif = '#if'
 608             elif line.startswith('#include'):
 609                 if ifdepth <= maxdepth:
 610                     line = sanitize(line)
 611                     if line:
 612                         line = get_filename(line)
 613                     if line and len(line):
 614                         raw_includes.append(line)
 615                 elif not SILENT:
 616                     sys.stderr.write('#include in {} : {}\n'.format(lastif, line))
 617
 618     return raw_includes
 619
 620 def explode(root, module, includes, tree, filter_local, recurse):
 621     incpath = os.path.join(root, 'include')
 622
 623     for inc in includes:
 624         filename = get_filename(inc)
 625         if filename in tree or len(filter_local.proc(filename)) == 0:
 626             continue
 627
 628         try:
 629             # Module or Local header.
 630             filepath = filter_local.find_local_file(inc)
 631             if filepath:
 632                 #print('trying loc: ' + filepath)
 633                 incs = process_source(root, module, filepath, maxdepth=1)
 634                 incs = map(get_filename, incs)
 635                 incs = process_list(incs, lambda x: filter_ignore(x, module))
 636                 incs = process_list(incs, filter_local.proc)
 637                 tree[filename] = incs
 638                 if recurse:
 639                     tree = explode(root, module, incs, tree, filter_local, recurse)
 640                 #print('{} => {}'.format(filepath, tree[filename]))
 641                 continue
 642         except:
 643             pass
 644
 645         try:
 646             # Public header.
 647             filepath = os.path.join(incpath, filename)
 648             #print('trying pub: ' + filepath)
 649             incs = process_source(root, module, filepath, maxdepth=1)
 650             incs = map(get_filename, incs)
 651             incs = process_list(incs, lambda x: filter_ignore(x, module))
 652             incs = process_list(incs, filter_local.proc)
 653             tree[filename] = incs
 654             if recurse:
 655                 tree = explode(root, module, incs, tree, filter_local, recurse)
 656             #print('{} => {}'.format(filepath, tree[filename]))
 657             continue
 658         except:
 659             pass
 660
 661         # Failed, but remember to avoid searching again.
 662         tree[filename] = []
 663
 664     return tree
 665
 666 def make_command_line():
 667     args = sys.argv[:]
 668     # Remove command line flags and
 669     # use internal flags.
 670     for i in xrange(len(args)-1, 0, -1):
 671         if args[i].startswith('--'):
 672             args.pop(i)
 673
 674     args.append('--cutoff=' + str(CUTOFF))
 675     if EXCLUDE_SYSTEM:
 676         args.append('--exclude:system')
 677     else:
 678         args.append('--include:system')
 679     if EXCLUDE_MODULE:
 680         args.append('--exclude:module')
 681     else:
 682         args.append('--include:module')
 683     if EXCLUDE_LOCAL:
 684         args.append('--exclude:local')
 685     else:
 686         args.append('--include:local')
 687
 688     return ' '.join(args)
 689
 690 def generate_includes(includes):
 691     """Generates the include lines of the pch.
 692     """
 693     lines = []
 694     for osname, group in includes.iteritems():
 695         if not len(group):
 696             continue
 697
 698         if len(osname):
 699             not_eq = ''
 700             if osname[0] == '!':
 701                 not_eq = '!'
 702                 osname = osname[1:]
 703             lines.append('')
 704             lines.append('#if {}defined({})'.format(not_eq, osname))
 705
 706         for i in group:
 707             lines.append(i)
 708
 709         if len(osname):
 710             lines.append('#endif')
 711
 712     return lines
 713
 714 def generate(includes, libname, filename, module):
 715     header = \
 716 """/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
 717 /*
 718  * This file is part of the LibreOffice project.
 719  *
 720  * This Source Code Form is subject to the terms of the Mozilla Public
 721  * License, v. 2.0. If a copy of the MPL was not distributed with this
 722  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 723  */
 724
 725 /*
 726  This file has been autogenerated by update_pch.sh. It is possible to edit it
 727  manually (such as when an include file has been moved/renamed/removed). All such
 728  manual changes will be rewritten by the next run of update_pch.sh (which presumably
 729  also fixes all possible problems, so it's usually better to use it).
 730 """
 731
 732     footer = \
 733 """
 734 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */
 735 """
 736     import datetime
 737
 738     with open(filename, 'w') as f:
 739         f.write(header)
 740         f.write('\n Generated on {} using:\n {}\n'.format(
 741                 datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
 742                 make_command_line()))
 743         f.write('\n If after updating build fails, use the following command to locate conflicting headers:\n ./bin/update_pch_bisect {} "make {}.build" --find-conflicts\n*/\n'.format(
 744                 filename, module))
 745
 746         # svx needs this (sendreportw32.cxx)
 747         if module == 'svx' and libname == 'svx':
 748             svx_define = """
 749 #ifdef _WIN32
 750 #   define UNICODE
 751 #   define _UNICODE
 752 #endif
 753 """
 754             f.write(svx_define)
 755
 756         # Dump the headers.
 757         f.write('\n')
 758         for i in includes:
 759             f.write(i + '\n')
 760
 761         # Some libraries pull windows headers that aren't self contained.
 762         if (module == 'connectivity' and libname == 'ado') or \
 763            (module == 'xmlsecurity' and libname == 'xsec_xmlsec'):
 764             ado_define = """
 765 // Cleanup windows header macro pollution.
 766 #if defined(_WIN32) && defined(WINAPI)
 767 #   include <postwin.h>
 768 #   undef RGB
 769 #endif
 770 """
 771             f.write(ado_define)
 772
 773         f.write(footer)
 774
 775 def remove_from_tree(filename, tree):
 776     # Remove this file, if top-level.
 777     incs = tree.pop(filename, [])
 778     for i in incs:
 779         tree = remove_from_tree(i, tree)
 780
 781     # Also remove if included from another.
 782     for (k, v) in tree.iteritems():
 783         if filename in v:
 784             v.remove(filename)
 785
 786     return tree
 787
 788 def tree_to_list(includes, filename, tree):
 789     if filename in includes:
 790         return includes
 791     includes.append(filename)
 792     #incs = tree.pop(filename, [])
 793     incs = tree[filename] if filename in tree else []
 794     for i in incs:
 795         tree_to_list(includes, i, tree)
 796
 797     return includes
 798
 799 def promote(includes):
 800     """ Common library headers are heavily
 801         referenced, even if they are included
 802         from a few places.
 803         Here we separate them to promote
 804         their inclusion in the final pch.
 805     """
 806     promo = []
 807     for inc in includes:
 808         if inc.startswith('boost') or \
 809            inc.startswith('sal') or \
 810            inc.startswith('osl') or \
 811            inc.startswith('rtl'):
 812             promo.append(inc)
 813     return promo
 814
 815 def make_pch_filename(root, module, libname):
 816     """ PCH files are stored here:
 817         <root>/<module>/inc/pch/precompiled_<libname>.hxx
 818     """
 819
 820     path = os.path.join(root, module)
 821     path = os.path.join(path, 'inc')
 822     path = os.path.join(path, 'pch')
 823     path = os.path.join(path, 'precompiled_' + libname + '.hxx')
 824     return path
 825
 826 def main():
 827
 828     global CUTOFF
 829     global EXCLUDE_MODULE
 830     global EXCLUDE_LOCAL
 831     global EXCLUDE_SYSTEM
 832     global SILENT
 833
 834     root = '.'
 835     module = sys.argv[1]
 836     libname = sys.argv[2]
 837     header = make_pch_filename(root, module, libname)
 838
 839     if not os.path.exists(os.path.join(root, module)):
 840         raise Exception('Error: module [{}] not found.'.format(module))
 841
 842     key = '{}.{}'.format(module, libname)
 843     if key in DEFAULTS:
 844         # Load the module-specific defaults.
 845         CUTOFF = DEFAULTS[key][0]
 846         EXCLUDE_SYSTEM = DEFAULTS[key][1]
 847         EXCLUDE_MODULE = DEFAULTS[key][2]
 848         EXCLUDE_LOCAL = DEFAULTS[key][3]
 849
 850     force_update = False
 851     for x in xrange(3, len(sys.argv)):
 852         i = sys.argv[x]
 853         if i.startswith('--cutoff='):
 854             CUTOFF = int(i.split('=')[1])
 855         elif i.startswith('--exclude:'):
 856             cat = i.split(':')[1]
 857             if cat == 'module':
 858                 EXCLUDE_MODULE = True
 859             elif cat == 'local':
 860                 EXCLUDE_LOCAL = True
 861             elif cat == 'system':
 862                 EXCLUDE_SYSTEM = True
 863         elif i.startswith('--include:'):
 864             cat = i.split(':')[1]
 865             if cat == 'module':
 866                 EXCLUDE_MODULE = False
 867             elif cat == 'local':
 868                 EXCLUDE_LOCAL = False
 869             elif cat == 'system':
 870                 EXCLUDE_SYSTEM = False
 871         elif i == '--silent':
 872             SILENT = True
 873         elif i == '--force':
 874             force_update = True
 875         else:
 876             sys.stderr.write('Unknown option [{}].'.format(i))
 877             return 1
 878
 879     filter_local = Filter_Local(root, module, \
 880                                 not EXCLUDE_MODULE, \
 881                                 not EXCLUDE_LOCAL)
 882
 883     # Read input.
 884     makefile = 'Library_{}.mk'.format(libname)
 885     groups = process_makefile(root, module, makefile)
 886
 887     generic = []
 888     for osname, group in groups.iteritems():
 889         if not len(group):
 890             continue
 891
 892         includes = []
 893         for filename in group:
 894             includes += process_source(root, module, filename)
 895
 896         # Save unique top-level includes.
 897         unique = set(includes)
 898         promoted = promote(unique)
 899
 900         # Process includes.
 901         includes = remove_rare(includes)
 902         includes = process_list(includes, lambda x: filter_ignore(x, module))
 903         includes = process_list(includes, filter_local.proc)
 904
 905         # Remove the already included ones.
 906         for inc in includes:
 907             unique.discard(inc)
 908
 909         # Explode the excluded ones.
 910         tree = {i:[] for i in includes}
 911         tree = explode(root, module, unique, tree, filter_local, not EXCLUDE_MODULE)
 912
 913         # Remove the already included ones from the tree.
 914         for inc in includes:
 915             filename = get_filename(inc)
 916             tree = remove_from_tree(filename, tree)
 917
 918         extra = []
 919         for (k, v) in tree.iteritems():
 920             extra += tree_to_list([], k, tree)
 921
 922         promoted += promote(extra)
 923         promoted = process_list(promoted, lambda x: filter_ignore(x, module))
 924         promoted = process_list(promoted, filter_local.proc)
 925         promoted = set(promoted)
 926         # If a promoted header includes others, remove the rest.
 927         for (k, v) in tree.iteritems():
 928             if k in promoted:
 929                 for i in v:
 930                     promoted.discard(i)
 931         includes += [x for x in promoted]
 932
 933         extra = remove_rare(extra)
 934         extra = process_list(extra, lambda x: filter_ignore(x, module))
 935         extra = process_list(extra, filter_local.proc)
 936         includes += extra
 937
 938         includes = [x for x in set(includes)]
 939         fixes = fixup(includes, module)
 940         fixes = map(lambda x: '#include <' + x + '>', fixes)
 941
 942         includes = map(lambda x: '#include <' + x + '>', includes)
 943         sorted = sort_by_category(includes, module, filter_local)
 944         includes = fixes + sorted
 945
 946         if len(osname):
 947             for i in generic:
 948                 if i in includes:
 949                     includes.remove(i)
 950
 951         groups[osname] = includes
 952         if not len(osname):
 953             generic = includes
 954
 955     # Open the old pch and compare its contents
 956     # with new includes.
 957     # Clobber only if they are different.
 958     with open(header, 'r') as f:
 959         old_pch_lines = [x.strip() for x in f.readlines()]
 960         new_lines = generate_includes(groups)
 961         # Find the first include in the old pch.
 962         start = -1
 963         for i in xrange(len(old_pch_lines)):
 964             if old_pch_lines[i].startswith('#include'):
 965                 start = i
 966                 break
 967         # Clobber if there is a mismatch.
 968         if force_update or start < 0 or (len(old_pch_lines) - start < len(new_lines)):
 969             generate(new_lines, libname, header, module)
 970             return 0
 971         else:
 972             for i in xrange(len(new_lines)):
 973                 if new_lines[i] != old_pch_lines[start + i]:
 974                     generate(new_lines, libname, header, module)
 975                     return 0
 976             else:
 977                 # Identical, but see if new pch removed anything.
 978                 for i in xrange(start + len(new_lines), len(old_pch_lines)):
 979                     if '#include' in old_pch_lines[i]:
 980                         generate(new_lines, libname, header, module)
 981                         return 0
 982
 983     # Didn't update.
 984     return 1
 985
 986 if __name__ == '__main__':
 987     """ Process all the includes in a Module
 988         to make into a PCH file.
 989         Run without arguments for unittests,
 990         and to see usage.
 991     """
 992
 993     if len(sys.argv) >= 3:
 994         status = main()
 995         sys.exit(status)
 996
 997     print('Usage: {} <Module name> <Library name> [options]'.format(sys.argv[0]))
 998     print('    Always run from the root of LO repository.\n')
 999     print('    Options:')
1000     print('    --cutoff=<count> - Threshold to excluding headers.')
1001     print('    --exclude:<category> - Exclude category-specific headers.')
1002     print('    --include:<category> - Include category-specific headers.')
1003     print('    --force - Force updating the pch even when nothing changes.')
1004     print('    Categories:')
1005     print('         module - Headers in /inc directory of a module.')
1006     print('         local  - Headers local to a source file.')
1007     print('         system - Platform-specific headers.')
1008     print('    --silent - print only errors.')
1009     print('\nRunning unit-tests...')
1010
1011
1012 class TestMethods(unittest.TestCase):
1013
1014     def test_sanitize(self):
1015         self.assertEqual(sanitize('#include "blah/file.cxx"'),
1016                                 '#include <blah/file.cxx>')
1017         self.assertEqual(sanitize('  #include\t"blah/file.cxx" '),
1018                                 '#include <blah/file.cxx>')
1019         self.assertEqual(sanitize('  '),
1020                                 '')
1021
1022     def test_filter_ignore(self):
1023         self.assertEqual(filter_ignore('blah/file.cxx', 'mod'),
1024                                      '')
1025         self.assertEqual(filter_ignore('vector', 'mod'),
1026                                      'vector')
1027         self.assertEqual(filter_ignore('file.cxx', 'mod'),
1028                                      '')
1029
1030     def test_remove_rare(self):
1031         self.assertEqual(remove_rare([]),
1032                                 [])
1033
1034 class TestMakefileParser(unittest.TestCase):
1035
1036     def setUp(self):
1037         global EXCLUDE_SYSTEM
1038         EXCLUDE_SYSTEM = False
1039
1040     def test_parse_singleline_eval(self):
1041         source = "$(eval $(call gb_Library_Library,sal))"
1042         lines = source.split('\n')
1043         groups = {'':[]}
1044         groups = parse_makefile(groups, lines, 0, None, 0)
1045         self.assertEqual(len(groups), 1)
1046         self.assertEqual(len(groups['']), 0)
1047
1048     def test_parse_multiline_eval(self):
1049         source = """$(eval $(call gb_Library_set_include,sal,\\
1050         $$(INCLUDE) \\
1051         -I$(SRCDIR)/sal/inc \\
1052 ))
1053 """
1054         lines = source.split('\n')
1055         groups = {'':[]}
1056         groups = parse_makefile(groups, lines, 0, None, 0)
1057         self.assertEqual(len(groups), 1)
1058         self.assertEqual(len(groups['']), 0)
1059
1060     def test_parse_multiline_eval_with_if(self):
1061         source = """$(eval $(call gb_Library_add_defs,sal,\\
1062         $(if $(filter $(OS),IOS), \\
1063                 -DNO_CHILD_PROCESSES \\
1064         ) \\
1065 ))
1066 """
1067         lines = source.split('\n')
1068         groups = {'':[]}
1069         groups = parse_makefile(groups, lines, 0, None, 0)
1070         self.assertEqual(len(groups), 1)
1071         self.assertEqual(len(groups['']), 0)
1072
1073     def test_parse_multiline_add_with_if(self):
1074         source = """$(eval $(call gb_Library_add_exception_objects,sal,\\
1075         sal/osl/unx/time \\
1076         $(if $(filter DESKTOP,$(BUILD_TYPE)), sal/osl/unx/salinit) \\
1077 ))
1078 """
1079         lines = source.split('\n')
1080         groups = {'':[]}
1081         groups = parse_makefile(groups, lines, 0, None, 0)
1082         self.assertEqual(len(groups), 1)
1083         self.assertEqual(len(groups['']), 1)
1084         self.assertEqual(groups[''][0], 'sal/osl/unx/time.cxx')
1085
1086     def test_parse_if_else(self):
1087         source = """ifeq ($(OS),MACOSX)
1088 $(eval $(call gb_Library_add_exception_objects,sal,\\
1089         sal/osl/mac/mac \\
1090 ))
1091 else
1092 $(eval $(call gb_Library_add_exception_objects,sal,\\
1093         sal/osl/unx/uunxapi \\
1094 ))
1095 endif
1096 """
1097         lines = source.split('\n')
1098         groups = {'':[]}
1099         groups = parse_makefile(groups, lines, 0, None, 0)
1100         self.assertEqual(len(groups), 3)
1101         self.assertEqual(len(groups['']), 0)
1102         self.assertEqual(len(groups['MACOSX']), 1)
1103         self.assertEqual(len(groups['!MACOSX']), 1)
1104         self.assertEqual(groups['MACOSX'][0], 'sal/osl/mac/mac.cxx')
1105         self.assertEqual(groups['!MACOSX'][0], 'sal/osl/unx/uunxapi.cxx')
1106
1107     def test_parse_nested_if(self):
1108         source = """ifeq ($(OS),MACOSX)
1109 $(eval $(call gb_Library_add_exception_objects,sal,\\
1110         sal/osl/mac/mac \\
1111 ))
1112 else
1113 $(eval $(call gb_Library_add_exception_objects,sal,\\
1114         sal/osl/unx/uunxapi \\
1115 ))
1116
1117 ifeq ($(OS),LINUX)
1118 $(eval $(call gb_Library_add_exception_objects,sal,\\
1119         sal/textenc/context \\
1120 ))
1121 endif
1122 endif
1123 """
1124         lines = source.split('\n')
1125         groups = {'':[]}
1126         groups = parse_makefile(groups, lines, 0, None, 0)
1127         self.assertEqual(len(groups), 4)
1128         self.assertEqual(len(groups['']), 0)
1129         self.assertEqual(len(groups['MACOSX']), 1)
1130         self.assertEqual(len(groups['!MACOSX']), 1)
1131         self.assertEqual(len(groups['LINUX']), 1)
1132         self.assertEqual(groups['MACOSX'][0], 'sal/osl/mac/mac.cxx')
1133         self.assertEqual(groups['!MACOSX'][0], 'sal/osl/unx/uunxapi.cxx')
1134         self.assertEqual(groups['LINUX'][0], 'sal/textenc/context.cxx')
1135
1136     def test_parse_exclude_system(self):
1137         source = """ifeq ($(OS),MACOSX)
1138 $(eval $(call gb_Library_add_exception_objects,sal,\\
1139         sal/osl/mac/mac \\
1140 ))
1141 else
1142 $(eval $(call gb_Library_add_exception_objects,sal,\\
1143         sal/osl/unx/uunxapi \\
1144 ))
1145
1146 ifeq ($(OS),LINUX)
1147 $(eval $(call gb_Library_add_exception_objects,sal,\\
1148         sal/textenc/context \\
1149 ))
1150 endif
1151 endif
1152 """
1153         global EXCLUDE_SYSTEM
1154         EXCLUDE_SYSTEM = True
1155
1156         lines = source.split('\n')
1157         groups = {'':[]}
1158         groups = parse_makefile(groups, lines, 0, None, 0)
1159         self.assertEqual(len(groups), 1)
1160         self.assertEqual(len(groups['']), 0)
1161
1162     def test_parse_filter(self):
1163         source = """ifneq ($(filter $(OS),MACOSX IOS),)
1164 $(eval $(call gb_Library_add_exception_objects,sal,\\
1165         sal/osl/unx/osxlocale \\
1166 ))
1167 endif
1168 """
1169         # Filter is still unsupported.
1170         lines = source.split('\n')
1171         groups = {'':[]}
1172         groups = parse_makefile(groups, lines, 0, None, 0)
1173         self.assertEqual(len(groups), 1)
1174         self.assertEqual(len(groups['']), 0)
1175
1176 unittest.main()
1177
1178 # vim: set et sw=4 ts=4 expandtab: