jhbuild/config.py

   1 # jhbuild - a build script for GNOME 1.x and 2.x
   2 # Copyright (C) 2001-2006  James Henstridge
   3 # Copyright (C) 2007-2008  Frederic Peters
   4 #
   5 #   config.py: configuration file parser
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  20
  21 import os
  22 import sys
  23 import traceback
  24 import types
  25 import logging
  26 import __builtin__
  27
  28 from jhbuild.errors import UsageError, FatalError, CommandError
  29 from jhbuild.utils.cmds import get_output
  30
  31 if sys.platform.startswith('win'):
  32     # For munging paths for MSYS's benefit
  33     import jhbuild.utils.subprocess_win32
  34
  35 __all__ = [ 'Config' ]
  36
  37 _defaults_file = os.path.join(os.path.dirname(__file__), 'defaults.jhbuildrc')
  38 _default_jhbuildrc = os.path.join(os.environ['HOME'], '.jhbuildrc')
  39
  40 _known_keys = [ 'moduleset', 'modules', 'skip', 'tags', 'prefix',
  41                 'checkoutroot', 'buildroot', 'autogenargs', 'makeargs',
  42                 'installprog', 'repos', 'branches', 'noxvfb', 'xvfbargs',
  43                 'builddir_pattern', 'module_autogenargs', 'module_makeargs',
  44                 'interact', 'buildscript', 'nonetwork',
  45                 'alwaysautogen', 'nobuild', 'makeclean', 'makecheck', 'module_makecheck',
  46                 'use_lib64', 'tinderbox_outputdir', 'sticky_date',
  47                 'tarballdir', 'pretty_print', 'svn_program', 'makedist',
  48                 'makedistcheck', 'nonotify', 'notrayicon', 'cvs_program',
  49                 'checkout_mode', 'copy_dir', 'module_checkout_mode',
  50                 'build_policy', 'trycheckout', 'min_time',
  51                 'nopoison', 'module_nopoison', 'forcecheck',
  52                 'makecheck_advisory', 'quiet_mode', 'progress_bar',
  53                 'module_extra_env', 'jhbuildbot_master', 'jhbuildbot_slavename',
  54                 'jhbuildbot_password', 'jhbuildbot_svn_commits_box',
  55                 'jhbuildbot_slaves_dir', 'jhbuildbot_dir',
  56                 'jhbuildbot_mastercfg', 'use_local_modulesets',
  57                 'ignore_suggests', 'modulesets_dir', 'mirror_policy',
  58                 'module_mirror_policy', 'dvcs_mirror_dir', 'build_targets' ]
  59
  60 env_prepends = {}
  61 def prependpath(envvar, path):
  62     env_prepends.setdefault(envvar, []).append(path)
  63
  64 def addpath(envvar, path):
  65     '''Adds a path to an environment variable.'''
  66     # special case ACLOCAL_FLAGS
  67     if envvar in [ 'ACLOCAL_FLAGS' ]:
  68         if sys.platform.startswith('win'):
  69             path = jhbuild.utils.subprocess_win32.fix_path_for_msys(path)
  70
  71         envval = os.environ.get(envvar, '-I %s' % path)
  72         parts = ['-I', path] + envval.split()
  73         i = 2
  74         while i < len(parts)-1:
  75             if parts[i] == '-I':
  76                 # check if "-I parts[i]" comes earlier
  77                 for j in range(0, i-1):
  78                     if parts[j] == '-I' and parts[j+1] == parts[i+1]:
  79                         del parts[i:i+2]
  80                         break
  81                 else:
  82                     i += 2
  83             else:
  84                 i += 1
  85         envval = ' '.join(parts)
  86     elif envvar in [ 'LDFLAGS', 'CFLAGS', 'CXXFLAGS' ]:
  87         if sys.platform.startswith('win'):
  88             path = jhbuild.utils.subprocess_win32.fix_path_for_msys(path)
  89
  90         envval = os.environ.get(envvar)
  91         if envval:
  92             envval = path + ' ' + envval
  93         else:
  94             envval = path
  95     else:
  96         if envvar == 'PATH':
  97             # PATH is special cased on Windows to allow execution without
  98             # sh.exe. The other env vars (like LD_LIBRARY_PATH) don't mean
  99             # anything to native Windows so they stay in UNIX format, but
 100             # PATH is kept in Windows format (; separated, c:/ or c:\ format
 101             # paths) so native Popen works.
 102             pathsep = os.pathsep
 103         else:
 104             pathsep = ':'
 105             if sys.platform.startswith('win'):
 106                 path = jhbuild.utils.subprocess_win32.fix_path_for_msys(path)
 107
 108             if sys.platform.startswith('win') and path[1]==':':
 109                 # Windows: Don't allow c:/ style paths in :-separated env vars
 110                 # for obvious reasons. /c/ style paths are valid - if a var is
 111                 # separated by : it will only be of interest to programs inside
 112                 # MSYS anyway.
 113                 path='/'+path[0]+path[2:]
 114
 115         envval = os.environ.get(envvar, path)
 116         parts = envval.split(pathsep)
 117         parts.insert(0, path)
 118         # remove duplicate entries:
 119         i = 1
 120         while i < len(parts):
 121             if parts[i] in parts[:i]:
 122                 del parts[i]
 123             elif envvar == 'PYTHONPATH' and parts[i] == "":
 124                 del parts[i]
 125             else:
 126                 i += 1
 127         envval = pathsep.join(parts)
 128
 129     os.environ[envvar] = envval
 130
 131 def parse_relative_time(s):
 132     m = re.match(r'(\d+) *([smhdw])', s.lower())
 133     if m:
 134         coeffs = {'s': 1, 'm': 60, 'h': 3600, 'd': 86400, 'w':7*86400}
 135         return float(m.group(1)) * coeffs[m.group(2)]
 136     else:
 137         raise ValueError(_('unable to parse \'%s\' as relative time.') % s)
 138
 139
 140 class Config:
 141     _orig_environ = None
 142
 143     def __init__(self, filename=_default_jhbuildrc):
 144         self._config = {
 145             '__file__': _defaults_file,
 146             'addpath':  addpath,
 147             'prependpath':  prependpath,
 148             'include': self.include,
 149             }
 150
 151         if not self._orig_environ:
 152             self.__dict__['_orig_environ'] = os.environ.copy()
 153         os.environ['UNMANGLED_PATH'] = os.environ.get('PATH', '')
 154
 155         try:
 156             SRCDIR
 157         except NameError:
 158             # this happens when an old jhbuild script is called
 159             if os.path.realpath(sys.argv[0]) == os.path.expanduser('~/bin/jhbuild'):
 160                 # if it was installed in ~/bin/, it may be because the new one
 161                 # is installed in ~/.local/bin/jhbuild
 162                 if os.path.exists(os.path.expanduser('~/.local/bin/jhbuild')):
 163                     logging.warning(
 164                             _('JHBuild start script has been installed in '
 165                               '~/.local/bin/jhbuild, you should remove the '
 166                               'old version that is still in ~/bin/ (or make '
 167                               'it a symlink to ~/.local/bin/jhbuild)'))
 168             if os.path.exists(os.path.join(sys.path[0], 'jhbuild')):
 169                 # the old start script inserted its source directory in
 170                 # sys.path, use it now to set new variables
 171                 __builtin__.__dict__['SRCDIR'] = sys.path[0]
 172                 __builtin__.__dict__['PKGDATADIR'] = None
 173                 __builtin__.__dict__['DATADIR'] = None
 174             else:
 175                 raise FatalError(
 176                     _('Obsolete JHBuild start script, make sure it is removed '
 177                       'then do run \'make install\''))
 178
 179         env_prepends.clear()
 180         try:
 181             execfile(_defaults_file, self._config)
 182         except:
 183             traceback.print_exc()
 184             raise FatalError(_('could not load config defaults'))
 185         self._config['__file__'] = filename
 186         self.filename = filename
 187         if not os.path.exists(filename):
 188             raise FatalError(_('could not load config file, %s is missing') % filename)
 189
 190         self.load()
 191         self.setup_env()
 192
 193     def reload(self):
 194         os.environ = self._orig_environ.copy()
 195         self.__init__(filename=self._config.get('__file__'))
 196         self.set_from_cmdline_options(options=None)
 197
 198     def include(self, filename):
 199         '''Read configuration variables from a file.'''
 200         try:
 201             execfile(filename, self._config)
 202         except:
 203             traceback.print_exc()
 204             raise FatalError(_('Could not include config file (%s)') % filename)
 205
 206     def load(self):
 207         config = self._config
 208         try:
 209             execfile(self.filename, config)
 210         except Exception, e:
 211             if isinstance(e, FatalError):
 212                 # raise FatalErrors back, as it means an error in include()
 213                 # and it will print a traceback, and provide a meaningful
 214                 # message.
 215                 raise e
 216             traceback.print_exc()
 217             raise FatalError(_('could not load config file'))
 218
 219         if not config.get('quiet_mode'):
 220             unknown_keys = []
 221             for k in config.keys():
 222                 if k in _known_keys + ['cvsroots', 'svnroots', 'cflags']:
 223                     continue
 224                 if k[0] == '_':
 225                     continue
 226                 if type(config[k]) in (types.ModuleType, types.FunctionType, types.MethodType):
 227                     continue
 228                 unknown_keys.append(k)
 229             if unknown_keys:
 230                 logging.info(
 231                         _('unknown keys defined in configuration file: %s') % \
 232                         ', '.join(unknown_keys))
 233
 234         # backward compatibility, from the days when jhbuild only
 235         # supported Gnome.org CVS.
 236         if config.get('cvsroot'):
 237             logging.warning(
 238                     _('the "%s" configuration variable is deprecated, '
 239                       'you should use "repos[\'gnome.org\']".') % 'cvsroot')
 240             config['repos'].update({'gnome.org': config['cvsroot']})
 241         if config.get('cvsroots'):
 242             logging.warning(
 243                     _('the "%s" configuration variable is deprecated, '
 244                       'you should use "repos".') % 'cvsroots')
 245             config['repos'].update(config['cvsroots'])
 246         if config.get('svnroots'):
 247             logging.warning(
 248                     _('the "%s" configuration variable is deprecated, '
 249                       'you should use "repos".') % 'svnroots')
 250             config['repos'].update(config['svnroots'])
 251
 252         # environment variables
 253         if config.has_key('cflags') and config['cflags']:
 254             os.environ['CFLAGS'] = config['cflags']
 255         if config.get('installprog') and os.path.exists(config['installprog']):
 256             os.environ['INSTALL'] = config['installprog']
 257
 258         # copy known config keys to attributes on the instance
 259         for name in _known_keys:
 260             setattr(self, name, config[name])
 261
 262         # default tarballdir to checkoutroot
 263         if not self.tarballdir: self.tarballdir = self.checkoutroot
 264
 265         # check possible checkout_mode values
 266         seen_copy_mode = (self.checkout_mode == 'copy')
 267         possible_checkout_modes = ('update', 'clobber', 'export', 'copy')
 268         if self.checkout_mode not in possible_checkout_modes:
 269             raise FatalError(_('invalid checkout mode'))
 270         for module, checkout_mode in self.module_checkout_mode.items():
 271             seen_copy_mode = seen_copy_mode or (checkout_mode == 'copy')
 272             if checkout_mode not in possible_checkout_modes:
 273                 raise FatalError(_('invalid checkout mode (module: %s)') % module)
 274         if seen_copy_mode and not self.copy_dir:
 275             raise FatalError(_('copy mode requires copy_dir to be set'))
 276
 277         if not os.path.exists(self.modulesets_dir):
 278             if self.use_local_modulesets:
 279                 logging.warning(
 280                         _('modulesets directory (%s) not found, '
 281                           'disabling use_local_modulesets') % self.modulesets_dir)
 282                 self.use_local_modulesets = False
 283             self.modulesets_dir = None
 284
 285     def setup_env(self):
 286         '''set environment variables for using prefix'''
 287
 288         if not os.path.exists(self.prefix):
 289             try:
 290                 os.makedirs(self.prefix)
 291             except:
 292                 raise FatalError(_('install prefix (%s) can not be created') % self.prefix)
 293
 294         os.environ['UNMANGLED_LD_LIBRARY_PATH'] = os.environ.get('LD_LIBRARY_PATH', '')
 295
 296         # LD_LIBRARY_PATH
 297         if self.use_lib64:
 298             libdir = os.path.join(self.prefix, 'lib64')
 299         else:
 300             libdir = os.path.join(self.prefix, 'lib')
 301         addpath('LD_LIBRARY_PATH', libdir)
 302
 303         # LDFLAGS and C_INCLUDE_PATH are required for autoconf configure
 304         # scripts to find modules that do not use pkg-config (such as guile
 305         # looking for gmp, or wireless-tools for NetworkManager)
 306         # (see bug #377724 and bug #545018)
 307
 308         # This path doesn't always get passed to addpath so we fix it here
 309         if sys.platform.startswith('win'):
 310             libdir = jhbuild.utils.subprocess_win32.fix_path_for_msys(libdir)
 311         os.environ['LDFLAGS'] = ('-L%s ' % libdir) + os.environ.get('LDFLAGS', '')
 312
 313         includedir = os.path.join(self.prefix, 'include')
 314         addpath('C_INCLUDE_PATH', includedir)
 315         addpath('CPLUS_INCLUDE_PATH', includedir)
 316
 317         # On Mac OS X, we use DYLD_FALLBACK_LIBRARY_PATH
 318         addpath('DYLD_FALLBACK_LIBRARY_PATH', libdir)
 319
 320         # PATH
 321         bindir = os.path.join(self.prefix, 'bin')
 322         addpath('PATH', bindir)
 323
 324         # MANPATH
 325         manpathdir = os.path.join(self.prefix, 'share', 'man')
 326         addpath('MANPATH', '')
 327         addpath('MANPATH', manpathdir)
 328
 329         # PKG_CONFIG_PATH
 330         if os.environ.get('PKG_CONFIG_PATH') is None:
 331             # add system pkgconfig lookup-directories by default, as pkg-config
 332             # usage spread and is now used by libraries that are out of jhbuild
 333             # realm; this also helps when building a single module with
 334             # jhbuild.  It is possible to avoid this by setting PKG_CONFIG_PATH
 335             # to the empty string.
 336             for dirname in ('share', 'lib', 'lib64'):
 337                 full_name = '/usr/%s/pkgconfig' % dirname
 338                 if os.path.exists(full_name):
 339                     addpath('PKG_CONFIG_PATH', full_name)
 340         pkgconfigdatadir = os.path.join(self.prefix, 'share', 'pkgconfig')
 341         pkgconfigdir = os.path.join(libdir, 'pkgconfig')
 342         addpath('PKG_CONFIG_PATH', pkgconfigdatadir)
 343         addpath('PKG_CONFIG_PATH', pkgconfigdir)
 344
 345         # XDG_DATA_DIRS
 346         xdgdatadir = os.path.join(self.prefix, 'share')
 347         addpath('XDG_DATA_DIRS', xdgdatadir)
 348
 349         # XDG_CONFIG_DIRS
 350         xdgconfigdir = os.path.join(self.prefix, 'etc', 'xdg')
 351         addpath('XDG_CONFIG_DIRS', xdgconfigdir)
 352
 353         # XCURSOR_PATH
 354         xcursordir = os.path.join(self.prefix, 'share', 'icons')
 355         addpath('XCURSOR_PATH', xcursordir)
 356
 357         # ACLOCAL_FLAGS
 358         aclocaldir = os.path.join(self.prefix, 'share', 'aclocal')
 359         if not os.path.exists(aclocaldir):
 360             try:
 361                 os.makedirs(aclocaldir)
 362             except:
 363                 raise FatalError(_("Can't create %s directory") % aclocaldir)
 364         if os.path.exists('/usr/share/aclocal'):
 365             addpath('ACLOCAL_FLAGS', '/usr/share/aclocal')
 366         if os.path.exists('/usr/local/share/aclocal'):
 367             addpath('ACLOCAL_FLAGS', '/usr/local/share/aclocal')
 368         addpath('ACLOCAL_FLAGS', aclocaldir)
 369
 370         # PERL5LIB
 371         perl5lib = os.path.join(self.prefix, 'lib', 'perl5')
 372         addpath('PERL5LIB', perl5lib)
 373
 374         os.environ['CERTIFIED_GNOMIE'] = 'yes'
 375
 376         # PYTHONPATH
 377         # Python inside jhbuild may be different than Python executing jhbuild,
 378         # so it is executed to get its version number (fallback to local
 379         # version number should never happen)
 380         python_bin = os.environ.get('PYTHON', 'python')
 381         try:
 382             pythonversion = 'python' + get_output([python_bin, '-c',
 383                 'import sys; print ".".join([str(x) for x in sys.version_info[:2]])'],
 384                 get_stderr = False).strip()
 385         except CommandError:
 386             pythonversion = 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1])
 387
 388         # In Python 2.6, site-packages got replaced by dist-packages, get the
 389         # actual value by asking distutils
 390         # <http://bugzilla.gnome.org/show_bug.cgi?id=575426>
 391         try:
 392             python_packages_dir = get_output([python_bin, '-c',
 393                 'import os, distutils.sysconfig; '\
 394                 'print distutils.sysconfig.get_python_lib(prefix="%s").split(os.path.sep)[-1]' % self.prefix],
 395                 get_stderr=False).strip()
 396         except CommandError:
 397             python_packages_dir = 'site-packages'
 398
 399         if self.use_lib64:
 400             pythonpath = os.path.join(self.prefix, 'lib64', pythonversion, python_packages_dir)
 401             addpath('PYTHONPATH', pythonpath)
 402             if not os.path.exists(pythonpath):
 403                 os.makedirs(pythonpath)
 404
 405         pythonpath = os.path.join(self.prefix, 'lib', pythonversion, python_packages_dir)
 406         addpath('PYTHONPATH', pythonpath)
 407         if not os.path.exists(pythonpath):
 408             os.makedirs(pythonpath)
 409
 410         # if there is a Python installed in JHBuild prefix, set it in PYTHON
 411         # environment variable, so it gets picked up by configure scripts
 412         # <http://bugzilla.gnome.org/show_bug.cgi?id=560872>
 413         if os.path.exists(os.path.join(self.prefix, 'bin', 'python')):
 414             os.environ['PYTHON'] = os.path.join(self.prefix, 'bin', 'python')
 415
 416         # Mono Prefixes
 417         os.environ['MONO_PREFIX'] = self.prefix
 418         os.environ['MONO_GAC_PREFIX'] = self.prefix
 419
 420         # GConf:
 421         # Create a GConf source path file that tells GConf to use the data in
 422         # the jhbuild prefix (in addition to the data in the system prefix),
 423         # and point to it with GCONF_DEFAULT_SOURCE_PATH so modules will be read
 424         # the right data (assuming a new enough libgconf).
 425         gconfdir = os.path.join(self.prefix, 'etc', 'gconf')
 426         gconfpathdir = os.path.join(gconfdir, '2')
 427         if not os.path.exists(gconfpathdir):
 428             os.makedirs(gconfpathdir)
 429         gconfpath = os.path.join(gconfpathdir, 'path.jhbuild')
 430         if not os.path.exists(gconfpath) and os.path.exists('/etc/gconf/2/path'):
 431             try:
 432                 inp = open('/etc/gconf/2/path')
 433                 out = open(gconfpath, 'w')
 434                 for line in inp.readlines():
 435                     if '/etc/gconf' in line:
 436                         out.write(line.replace('/etc/gconf', gconfdir))
 437                     out.write(line)
 438                 out.close()
 439                 inp.close()
 440             except:
 441                 traceback.print_exc()
 442                 raise FatalError(_('Could not create GConf config (%s)') % gconfpath)
 443         os.environ['GCONF_DEFAULT_SOURCE_PATH'] = gconfpath
 444
 445         # Set GCONF_SCHEMA_INSTALL_SOURCE to point into the jhbuild prefix so
 446         # modules will install their schemas there (rather than failing to
 447         # install them into /etc).
 448         os.environ['GCONF_SCHEMA_INSTALL_SOURCE'] = 'xml:merged:' + os.path.join(
 449                 gconfdir, 'gconf.xml.defaults')
 450
 451         # handle environment prepends ...
 452         for envvar in env_prepends.keys():
 453             for path in env_prepends[envvar]:
 454                 addpath(envvar, path)
 455
 456
 457         # get rid of gdkxft from the env -- it will cause problems.
 458         if os.environ.has_key('LD_PRELOAD'):
 459             valarr = os.environ['LD_PRELOAD'].split(' ')
 460             for x in valarr[:]:
 461                 if x.find('libgdkxft.so') >= 0:
 462                     valarr.remove(x)
 463             os.environ['LD_PRELOAD'] = ' '.join(valarr)
 464
 465         self.update_build_targets()
 466
 467     def update_build_targets(self):
 468         # update build targets according to old flags
 469         if self.makecheck and not 'check' in self.build_targets:
 470             self.build_targets.insert(0, 'check')
 471         if self.makeclean and not 'clean' in self.build_targets:
 472             self.build_targets.insert(0, 'clean')
 473         if self.nobuild:
 474             # nobuild actually means "checkout"
 475             for phase in ('configure', 'build', 'check', 'clean', 'install'):
 476                 if phase in self.build_targets:
 477                     self.build_targets.remove(phase)
 478             self.build_targets.append('checkout')
 479         if self.makedist and not 'dist' in self.build_targets:
 480             self.build_targets.append('dist')
 481         if self.makedistcheck and not 'distcheck' in self.build_targets:
 482             self.build_targets.append('distcheck')
 483
 484     def set_from_cmdline_options(self, options=None):
 485         if options is None:
 486             options = self.cmdline_options
 487         else:
 488             self.cmdline_options = options
 489         if hasattr(options, 'autogen') and options.autogen:
 490             self.alwaysautogen = True
 491         if hasattr(options, 'clean') and (
 492                 options.clean and not 'clean' in self.build_targets):
 493             self.build_targets.insert(0, 'clean')
 494         if hasattr(options, 'check') and (
 495                 options.check and not 'check' in self.build_targets):
 496             self.build_targets.insert(0, 'check')
 497         if hasattr(options, 'dist') and (
 498                 options.dist and not 'dist' in self.build_targets):
 499             self.build_targets.append('dist')
 500         if hasattr(options, 'distcheck') and (
 501                 options.distcheck and not 'distcheck' in self.build_targets):
 502             self.build_targets.append('distcheck')
 503         if hasattr(options, 'ignore_suggests') and options.ignore_suggests:
 504             self.ignore_suggests = True
 505         if hasattr(options, 'nonetwork') and options.nonetwork:
 506             self.nonetwork = True
 507         if hasattr(options, 'skip'):
 508             for item in options.skip:
 509                 self.skip += item.split(',')
 510         if hasattr(options, 'tags'):
 511             for item in options.tags:
 512                 self.tags += item.split(',')
 513         if hasattr(options, 'sticky_date') and options.sticky_date is not None:
 514                 self.sticky_date = options.sticky_date
 515         if hasattr(options, 'xvfb') and options.noxvfb is not None:
 516                 self.noxvfb = options.noxvfb
 517         if hasattr(options, 'trycheckout') and  options.trycheckout:
 518             self.trycheckout = True
 519         if hasattr(options, 'nopoison') and options.nopoison:
 520             self.nopoison = True
 521         if hasattr(options, 'quiet') and options.quiet:
 522             self.quiet_mode = True
 523         if hasattr(options, 'force_policy') and options.force_policy:
 524             self.build_policy = 'all'
 525         if hasattr(options, 'min_age') and options.min_age:
 526             try:
 527                 self.min_time = time.time() - parse_relative_time(options.min_age)
 528             except ValueError:
 529                 raise FatalError(_('Failed to parse relative time'))
 530
 531
 532     def __setattr__(self, k, v):
 533         '''Override __setattr__ for additional checks on some options.'''
 534         if k == 'quiet_mode' and v:
 535             try:
 536                 import curses
 537             except ImportError:
 538                 logging.warning(
 539                         _('quiet mode has been disabled because the Python curses module is missing.'))
 540                 v = False
 541
 542         self.__dict__[k] = v
 543