jhbuild/commands/bot.py

   1 # jhbuild - a build script for GNOME 1.x and 2.x
   2 # Copyright (C) 2001-2006  James Henstridge
   3 # Copyright (C) 2008 Frederic Peters
   4 #
   5 #   bot.py: buildbot control commands
   6 #
   7 # This program is free software; you can redistribute it and/or modify
   8 # it under the terms of the GNU General Public License as published by
   9 # the Free Software Foundation; either version 2 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU General Public License
  18 # along with this program; if not, write to the Free Software
  19 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  20 #
  21 #
  22 # Some methods are derived from Buildbot own methods (when it was not possible
  23 # to override just some parts of them).  Buildbot is also licensed under the
  24 # GNU General Public License.
  25
  26 import os
  27 import signal
  28 import sys
  29 import urllib
  30 from optparse import make_option
  31 import socket
  32 import __builtin__
  33 import csv
  34 import logging
  35
  36 try:
  37     import elementtree.ElementTree as ET
  38 except ImportError:
  39     import xml.etree.ElementTree as ET
  40
  41 import jhbuild.moduleset
  42 import jhbuild.frontends
  43 from jhbuild.commands import Command, register_command
  44 from jhbuild.commands.base import cmd_build
  45 from jhbuild.config import addpath
  46 from jhbuild.errors import UsageError, FatalError, CommandError
  47
  48 try:
  49     import buildbot
  50 except ImportError:
  51     buildbot = None
  52
  53 class cmd_bot(Command):
  54     doc = N_('Control buildbot')
  55
  56     name = 'bot'
  57     usage_args = N_('[ options ... ]')
  58
  59     def __init__(self):
  60         Command.__init__(self, [
  61             make_option('--setup',
  62                         action='store_true', dest='setup', default=False,
  63                         help=_('setup a buildbot environment')),
  64             make_option('--start',
  65                         action='store_true', dest='start', default=False,
  66                         help=_('start a buildbot slave server')),
  67             make_option('--stop',
  68                         action='store_true', dest='stop', default=False,
  69                         help=_('stop a buildbot slave server')),
  70             make_option('--start-server',
  71                         action='store_true', dest='start_server', default=False,
  72                         help=_('start a buildbot master server')),
  73             make_option('--reload-server-config',
  74                         action='store_true', dest='reload_server_config', default=False,
  75                         help=_('reload a buildbot master server configuration')),
  76             make_option('--stop-server',
  77                         action='store_true', dest='stop_server', default=False,
  78                         help=_('stop a buildbot master server')),
  79             make_option('--daemon',
  80                         action='store_true', dest='daemon', default=False,
  81                         help=_('start as daemon')),
  82             make_option('--pidfile', metavar='PIDFILE',
  83                         action='store', dest='pidfile', default=None,
  84                         help=_('pid file location')),
  85             make_option('--logfile', metavar='LOGFILE',
  86                         action='store', dest='logfile', default=None,
  87                         help=_('log file location')),
  88             make_option('--slaves-dir', metavar='SLAVESDIR',
  89                         action='store', dest='slaves_dir', default=None,
  90                         help=_('directory with slave files (only with --start-server)')),
  91             make_option('--buildbot-dir', metavar='BUILDBOTDIR',
  92                         action='store', dest='buildbot_dir', default=None,
  93                         help=_('directory with buildbot work files (only with --start-server)')),
  94             make_option('--mastercfg', metavar='CFGFILE',
  95                         action='store', dest='mastercfgfile', default=None,
  96                         help=_('master cfg file location (only with --start-server)')),
  97             make_option('--step',
  98                         action='store_true', dest='step', default=False,
  99                         help=_('exec a buildbot step (internal use only)')),
 100             ])
 101
 102     def run(self, config, options, args, help=None):
 103         if options.setup:
 104             return self.setup(config)
 105
 106         global buildbot
 107         if buildbot is None:
 108             import site
 109             pythonversion = 'python' + str(sys.version_info[0]) + '.' + str(sys.version_info[1])
 110             pythonpath = os.path.join(config.prefix, 'lib', pythonversion, 'site-packages')
 111             site.addsitedir(pythonpath)
 112             if config.use_lib64:
 113                 pythonpath = os.path.join(config.prefix, 'lib64', pythonversion, 'site-packages')
 114                 site.addsitedir(pythonpath)
 115             try:
 116                 import buildbot
 117             except ImportError:
 118                 raise FatalError(_('buildbot and twisted not found, run jhbuild bot --setup'))
 119
 120         # make jhbuild config file accessible to buildbot files
 121         # (master.cfg , steps.py, etc.)
 122         __builtin__.__dict__['jhbuild_config'] = config
 123
 124         daemonize = False
 125         pidfile = None
 126         logfile = None
 127         slaves_dir = config.jhbuildbot_slaves_dir
 128         mastercfgfile = config.jhbuildbot_mastercfg
 129         buildbot_dir = config.jhbuildbot_dir
 130
 131         if options.daemon:
 132             daemonize = True
 133         if options.pidfile:
 134             pidfile = options.pidfile
 135         if options.logfile:
 136             logfile = options.logfile
 137         if options.slaves_dir:
 138             slaves_dir = options.slaves_dir
 139         if options.mastercfgfile:
 140             mastercfgfile = options.mastercfgfile
 141         if options.buildbot_dir:
 142             buildbot_dir = os.path.abspath(options.buildbot_dir)
 143
 144         if options.start:
 145             return self.start(config, daemonize, pidfile, logfile)
 146
 147         if options.step:
 148             os.environ['JHBUILDRC'] = config.filename
 149             os.environ['LC_ALL'] = 'C'
 150             os.environ['LANGUAGE'] = 'C'
 151             os.environ['LANG'] = 'C'
 152             __builtin__.__dict__['_'] = lambda x: x
 153             config.interact = False
 154             config.nonetwork = True
 155             os.environ['TERM'] = 'dumb'
 156             if args[0] in ('update', 'build', 'check', 'clean'):
 157                 module_set = jhbuild.moduleset.load(config)
 158                 buildscript = jhbuild.frontends.get_buildscript(config,
 159                         [module_set.get_module(x, ignore_case=True) for x in args[1:]])
 160                 phases = None
 161                 if args[0] == 'update':
 162                     config.nonetwork = False
 163                     phases = ['checkout']
 164                 elif args[0] == 'build':
 165                     config.alwaysautogen = True
 166                     # make check will be run in another step
 167                     config.makecheck = False
 168                     config.build_targets = ['install']
 169                 elif args[0] == 'check':
 170                     config.makecheck = True
 171                     config.build_targets = ['check']
 172                     phases = ['check']
 173                 elif args[0] == 'clean':
 174                     phases = ['clean']
 175                 rc = buildscript.build(phases=phases)
 176             else:
 177                 command = args[0]
 178                 rc = jhbuild.commands.run(command, config, args[1:], help=None)
 179             sys.exit(rc)
 180
 181         if options.start_server:
 182             return self.start_server(config, daemonize, pidfile, logfile,
 183                     slaves_dir, mastercfgfile, buildbot_dir)
 184
 185         if options.stop or options.stop_server:
 186             return self.stop(config, pidfile)
 187
 188         if options.reload_server_config:
 189             return self.reload_server_config(config, pidfile)
 190
 191     def setup(self, config):
 192         module_set = jhbuild.moduleset.load(config, 'buildbot')
 193         module_list = module_set.get_module_list('all', config.skip)
 194         build = jhbuild.frontends.get_buildscript(config, module_list)
 195         return build.build()
 196
 197     def start(self, config, daemonize, pidfile, logfile):
 198         from twisted.application import service
 199         application = service.Application('buildslave')
 200         if ':' in config.jhbuildbot_master:
 201             master_host, master_port = config.jhbuildbot_master.split(':')
 202             master_port = int(master_port)
 203         else:
 204             master_host, master_port = config.jhbuildbot_master, 9070
 205
 206         slave_name = config.jhbuildbot_slavename or socket.gethostname()
 207
 208         keepalive = 600
 209         usepty = 0
 210         umask = None
 211         basedir = os.path.join(config.checkoutroot, 'jhbuildbot')
 212         if not os.path.exists(os.path.join(basedir, 'builddir')):
 213             os.makedirs(os.path.join(basedir, 'builddir'))
 214         os.chdir(basedir)
 215
 216         from buildbot.slave.bot import BuildSlave
 217         s = BuildSlave(master_host, master_port,
 218                 slave_name, config.jhbuildbot_password, basedir,
 219                 keepalive, usepty, umask=umask)
 220         s.setServiceParent(application)
 221
 222
 223         from twisted.scripts._twistd_unix import UnixApplicationRunner, ServerOptions
 224
 225         opts = ['--no_save']
 226         if not daemonize:
 227             opts.append('--nodaemon')
 228         if pidfile:
 229             opts.extend(['--pidfile', pidfile])
 230         if logfile:
 231             opts.extend(['--logfile', logfile])
 232         options = ServerOptions()
 233         options.parseOptions(opts)
 234
 235         class JhBuildbotApplicationRunner(UnixApplicationRunner):
 236             application = None
 237
 238             def createOrGetApplication(self):
 239                 return self.application
 240
 241         JhBuildbotApplicationRunner.application = application
 242         JhBuildbotApplicationRunner(options).run()
 243
 244     def start_server(self, config, daemonize, pidfile, logfile, slaves_dir,
 245             mastercfgfile, buildbot_dir):
 246
 247         from twisted.scripts._twistd_unix import UnixApplicationRunner, ServerOptions
 248
 249         opts = ['--no_save']
 250         if not daemonize:
 251             opts.append('--nodaemon')
 252         if pidfile:
 253             opts.extend(['--pidfile', pidfile])
 254         if pidfile:
 255             opts.extend(['--logfile', logfile])
 256         options = ServerOptions()
 257         options.parseOptions(opts)
 258
 259         class JhBuildbotApplicationRunner(UnixApplicationRunner):
 260             application = None
 261
 262             def createOrGetApplication(self):
 263                 return self.application
 264
 265         from twisted.application import service, strports
 266         from buildbot.master import BuildMaster
 267         application = service.Application('buildmaster')
 268         from buildbot.buildslave import BuildSlave
 269
 270         from twisted.python import log
 271         from twisted.internet import defer
 272         from buildbot import interfaces
 273         from buildbot.process.properties import Properties
 274
 275         class JhBuildSlave(BuildSlave):
 276             contact_name = None
 277             contact_email = None
 278             url = None
 279             distribution = None
 280             architecture = None
 281             version = None
 282
 283             max_builds = 2
 284             scheduler = None
 285
 286             run_checks = True
 287             run_coverage_report = False
 288             run_clean_afterwards = False
 289
 290             def load_extra_configuration(self, slaves_dir):
 291                 from twisted.python import log
 292                 slave_xml_file = os.path.join(slaves_dir, self.slavename + '.xml')
 293                 if not os.path.exists(slave_xml_file):
 294                     log.msg(_('No description for slave %s.') % self.slavename)
 295                     return
 296                 try:
 297                     cfg = ET.parse(slave_xml_file)
 298                 except: # parse error
 299                     log.msg(_('Failed to parse slave config for %s.') % self.slavename)
 300                     return
 301
 302                 for attribute in ('config/max_builds', 'config/missing_timeout',
 303                             'config/run_checks', 'config/run_coverage_report',
 304                             'config/run_clean_afterwards',
 305                             'config/scheduler',
 306                             'nightly_scheduler/minute',
 307                             'nightly_scheduler/hour',
 308                             'nightly_scheduler/dayOfMonth',
 309                             'nightly_scheduler/month',
 310                             'nightly_scheduler/dayOfWeek',
 311                             'info/contact_name', 'info/contact_email',
 312                             'info/url', 'info/distribution', 'info/architecture',
 313                             'info/version'):
 314                     attr_name = attribute.split('/')[-1]
 315                     try:
 316                         value = cfg.find(attribute).text
 317                     except AttributeError:
 318                         continue
 319
 320                     if attr_name in ('max_builds', 'missing_timeout'): # int value
 321                         try:
 322                             value = int(value)
 323                         except ValueError:
 324                             continue
 325
 326                     if attr_name in ('run_checks', 'run_coverage_report', 'run_clean_afterwards'):
 327                         value = (value == 'yes')
 328
 329                     if attr_name in ('minute', 'hour', 'dayOfMonth', 'month', 'dayOfWeek'):
 330                         try:
 331                             value = int(value)
 332                         except ValueError:
 333                             value = '*'
 334
 335                     setattr(self, attr_name, value)
 336
 337                 if self.scheduler == 'nightly':
 338                     self.nightly_kwargs = {}
 339                     for attr_name in ('minute', 'hour', 'dayOfMonth', 'month', 'dayOfWeek'):
 340                         if hasattr(self, attr_name):
 341                             self.nightly_kwargs[attr_name] = getattr(self, attr_name)
 342
 343         class JhBuildMaster(BuildMaster):
 344             jhbuild_config = config
 345             def loadConfig(self, f):
 346                 # modified from parent method to get slaves, projects, change
 347                 # sources, schedulers, builders and web status ouf of
 348                 # master.cfg [it would have been cleaner if jhbuild didn't
 349                 # have to copy all that code.]
 350                 localDict = {'basedir': os.path.expanduser(self.basedir)}
 351                 try:
 352                     exec f in localDict
 353                 except:
 354                     log.msg("error while parsing config file")
 355                     raise
 356
 357                 jhbuild_config.load()
 358
 359                 try:
 360                     config = localDict['BuildmasterConfig']
 361                 except KeyError:
 362                     log.err("missing config dictionary")
 363                     log.err("config file must define BuildmasterConfig")
 364                     raise
 365
 366                 known_keys = ("bots", "slaves",
 367                               "sources", "change_source",
 368                               "schedulers", "builders", "mergeRequests",
 369                               "slavePortnum", "debugPassword", "logCompressionLimit",
 370                               "manhole", "status", "projectName", "projectURL",
 371                               "buildbotURL", "properties", "prioritizeBuilders",
 372                               "eventHorizon", "buildCacheSize", "logHorizon", "buildHorizon",
 373                               "changeHorizon", "logMaxSize", "logMaxTailSize",
 374                               "logCompressionMethod",
 375                               )
 376                 for k in config.keys():
 377                     if k not in known_keys:
 378                         log.msg("unknown key '%s' defined in config dictionary" % k)
 379
 380                 # the 'slaves' list is read from the 'slaves.csv' file in the
 381                 # current directory (unless instructed different from command line)
 382                 # it is a CSV file structured like this:
 383                 #   slavename,password
 384                 config['slaves'] = []
 385                 slaves_csv_file = os.path.join(slaves_dir, 'slaves.csv')
 386                 if os.path.exists(slaves_csv_file):
 387                     for x in csv.reader(file(slaves_csv_file)):
 388                         if not x or x[0].startswith('#'):
 389                             continue
 390                         kw = {}
 391                         build_slave = JhBuildSlave(x[0], x[1])
 392                         build_slave.load_extra_configuration(slaves_dir)
 393                         config['slaves'].append(build_slave)
 394
 395                 if len(config['slaves']) == 0:
 396                     log.msg('you must fill slaves.csv with slaves')
 397
 398                 module_set = jhbuild.moduleset.load(self.jhbuild_config)
 399                 module_list = module_set.get_module_list(
 400                         self.jhbuild_config.modules,
 401                         self.jhbuild_config.skip,
 402                         include_optional_modules=True)
 403                 config['projects'] = [x.name for x in module_list \
 404                                       if not x.name.startswith('meta-')]
 405
 406                 if self.jhbuild_config.jhbuildbot_svn_commits_box:
 407                     # trigger builds from mails to svn-commit-list
 408                     # (note Maildir must be correct, or everything will fail)
 409                     from jhbuild.buildbot.changes import GnomeMaildirSource
 410                     config['change_source'] = GnomeMaildirSource(
 411                             self.jhbuild_config.jhbuildbot_svn_commits_box,
 412                             prefix=None)
 413                 else:
 414                     # support injection (use 'buildbot sendchange')
 415                     from buildbot.changes.pb import PBChangeSource
 416                     config['change_source'] = PBChangeSource()
 417
 418                 # Schedulers
 419                 from jhbuild.buildbot.scheduler import SerialScheduler, NightlySerialScheduler, OnCommitScheduler
 420                 config['schedulers'] = []
 421                 for slave in config['slaves']:
 422                     s = None
 423                     for project in config['projects']:
 424                         buildername = str('%s-%s' % (project, slave.slavename))
 425                         scheduler_kwargs = {}
 426                         if slave.scheduler == 'nightly':
 427                             scheduler_class = NightlySerialScheduler
 428                             scheduler_kwargs = slave.nightly_kwargs
 429                         else:
 430                             scheduler_class = SerialScheduler
 431                         s = scheduler_class(buildername, project, upstream=s,
 432                                             builderNames=[buildername],
 433                                             **scheduler_kwargs)
 434                         config['schedulers'].append(s)
 435                         if self.jhbuild_config.jhbuildbot_svn_commits_box:
 436                             # schedulers that will launch job when receiving
 437                             # change notifications
 438                             s2 = OnCommitScheduler('oc-' + buildername,
 439                                     project, builderNames=[buildername])
 440                             config['schedulers'].append(s2)
 441
 442                 # Builders
 443                 from jhbuild.buildbot.factory import JHBuildFactory
 444                 config['builders'] = []
 445                 for project in config['projects']:
 446                     for slave in config['slaves']:
 447                         f = JHBuildFactory(project, slave)
 448                         config['builders'].append({
 449                             'name' : "%s-%s" % (project, slave.slavename),
 450                             'slavename' : slave.slavename,
 451                             'builddir' : 'builddir/%s.%s' % (project, slave.slavename),
 452                             'factory' : f,
 453                             'category' : project
 454                         })
 455
 456                 # Status targets
 457                 if not config.has_key('status'):
 458                     # let it be possible to define additional status in
 459                     # master.cfg
 460                     config['status'] = []
 461
 462                 from jhbuild.buildbot.status.web import JHBuildWebStatus
 463                 config['status'].append(
 464                     JHBuildWebStatus(
 465                         self.jhbuild_config.moduleset,
 466                         config['projects'],
 467                         [x.slavename for x in config['slaves']],
 468                         http_port=8080, allowForce=True)
 469                 )
 470
 471                 # remaining of the method is a straight copy from buildbot
 472                 # ...
 473                 try:
 474                     # required
 475                     schedulers = config['schedulers']
 476                     builders = config['builders']
 477                     slavePortnum = config['slavePortnum']
 478                     #slaves = config['slaves']
 479                     #change_source = config['change_source']
 480
 481                     # optional
 482                     debugPassword = config.get('debugPassword')
 483                     manhole = config.get('manhole')
 484                     status = config.get('status', [])
 485                     projectName = config.get('projectName')
 486                     projectURL = config.get('projectURL')
 487                     buildbotURL = config.get('buildbotURL')
 488                     properties = config.get('properties', {})
 489                     buildCacheSize = config.get('buildCacheSize', None)
 490                     eventHorizon = config.get('eventHorizon', None)
 491                     logHorizon = config.get('logHorizon', None)
 492                     buildHorizon = config.get('buildHorizon', None)
 493                     logCompressionLimit = config.get('logCompressionLimit', 4*1024)
 494                     if logCompressionLimit is not None and not \
 495                             isinstance(logCompressionLimit, int):
 496                         raise ValueError("logCompressionLimit needs to be bool or int")
 497                     logCompressionMethod = config.get('logCompressionMethod', "bz2")
 498                     if logCompressionMethod not in ('bz2', 'gz'):
 499                         raise ValueError("logCompressionMethod needs to be 'bz2', or 'gz'")
 500                     logMaxSize = config.get('logMaxSize')
 501                     if logMaxSize is not None and not \
 502                             isinstance(logMaxSize, int):
 503                         raise ValueError("logMaxSize needs to be None or int")
 504                     logMaxTailSize = config.get('logMaxTailSize')
 505                     if logMaxTailSize is not None and not \
 506                             isinstance(logMaxTailSize, int):
 507                         raise ValueError("logMaxTailSize needs to be None or int")
 508                     mergeRequests = config.get('mergeRequests')
 509                     if mergeRequests is not None and not callable(mergeRequests):
 510                         raise ValueError("mergeRequests must be a callable")
 511                     prioritizeBuilders = config.get('prioritizeBuilders')
 512                     if prioritizeBuilders is not None and not callable(prioritizeBuilders):
 513                         raise ValueError("prioritizeBuilders must be callable")
 514                     changeHorizon = config.get("changeHorizon")
 515                     if changeHorizon is not None and not isinstance(changeHorizon, int):
 516                         raise ValueError("changeHorizon needs to be an int")
 517
 518                 except KeyError, e:
 519                     log.msg("config dictionary is missing a required parameter")
 520                     log.msg("leaving old configuration in place")
 521                     raise
 522
 523                 #if "bots" in config:
 524                 #    raise KeyError("c['bots'] is no longer accepted")
 525
 526                 slaves = config.get('slaves', [])
 527                 if "bots" in config:
 528                     m = ("c['bots'] is deprecated as of 0.7.6 and will be "
 529                          "removed by 0.8.0 . Please use c['slaves'] instead.")
 530                     log.msg(m)
 531                     warnings.warn(m, DeprecationWarning)
 532                     for name, passwd in config['bots']:
 533                         slaves.append(JhBuildSlave(name, passwd))
 534
 535                 if "bots" not in config and "slaves" not in config:
 536                     log.msg("config dictionary must have either 'bots' or 'slaves'")
 537                     log.msg("leaving old configuration in place")
 538                     raise KeyError("must have either 'bots' or 'slaves'")
 539
 540                 #if "sources" in config:
 541                 #    raise KeyError("c['sources'] is no longer accepted")
 542
 543                 if changeHorizon is not None:
 544                     self.change_svc.changeHorizon = changeHorizon
 545
 546                 change_source = config.get('change_source', [])
 547                 if isinstance(change_source, (list, tuple)):
 548                     change_sources = change_source
 549                 else:
 550                     change_sources = [change_source]
 551                 if "sources" in config:
 552                     m = ("c['sources'] is deprecated as of 0.7.6 and will be "
 553                          "removed by 0.8.0 . Please use c['change_source'] instead.")
 554                     log.msg(m)
 555                     warnings.warn(m, DeprecationWarning)
 556                     for s in config['sources']:
 557                         change_sources.append(s)
 558
 559                 # do some validation first
 560                 for s in slaves:
 561                     assert interfaces.IBuildSlave.providedBy(s)
 562                     if s.slavename in ("debug", "change", "status"):
 563                         raise KeyError(
 564                             "reserved name '%s' used for a bot" % s.slavename)
 565                 if config.has_key('interlocks'):
 566                     raise KeyError("c['interlocks'] is no longer accepted")
 567
 568                 assert isinstance(change_sources, (list, tuple))
 569                 for s in change_sources:
 570                     assert interfaces.IChangeSource(s, None)
 571                 # this assertion catches c['schedulers'] = Scheduler(), since
 572                 # Schedulers are service.MultiServices and thus iterable.
 573                 errmsg = "c['schedulers'] must be a list of Scheduler instances"
 574                 assert isinstance(schedulers, (list, tuple)), errmsg
 575                 for s in schedulers:
 576                     assert interfaces.IScheduler(s, None), errmsg
 577                 assert isinstance(status, (list, tuple))
 578                 for s in status:
 579                     assert interfaces.IStatusReceiver(s, None)
 580
 581                 slavenames = [s.slavename for s in slaves]
 582                 buildernames = []
 583                 dirnames = []
 584
 585                 # convert builders from objects to config dictionaries
 586                 builders_dicts = []
 587                 for b in builders:
 588                     if isinstance(b, buildbot.config.BuilderConfig):
 589                         builders_dicts.append(b.getConfigDict())
 590                     elif type(b) is dict:
 591                         builders_dicts.append(b)
 592                     else:
 593                         raise ValueError("builder %s is not a BuilderConfig object (or a dict)" % b)
 594                 builders = builders_dicts
 595
 596                 for b in builders:
 597                     if b.has_key('slavename') and b['slavename'] not in slavenames:
 598                         raise ValueError("builder %s uses undefined slave %s" \
 599                                          % (b['name'], b['slavename']))
 600                     for n in b.get('slavenames', []):
 601                         if n not in slavenames:
 602                             raise ValueError("builder %s uses undefined slave %s" \
 603                                              % (b['name'], n))
 604                     if b['name'] in buildernames:
 605                         raise ValueError("duplicate builder name %s"
 606                                          % b['name'])
 607                     buildernames.append(b['name'])
 608
 609                     # sanity check name (BuilderConfig does this too)
 610                     if b['name'].startswith("_"):
 611                         errmsg = ("builder names must not start with an "
 612                                   "underscore: " + b['name'])
 613                         log.err(errmsg)
 614                         raise ValueError(errmsg)
 615
 616                     # Fix the dictionnary with default values, in case this wasn't
 617                     # specified with a BuilderConfig object (which sets the same defaults)
 618                     b.setdefault('builddir', buildbot.util.safeTranslate(b['name']))
 619                     b.setdefault('slavebuilddir', b['builddir'])
 620
 621                     if b['builddir'] in dirnames:
 622                         raise ValueError("builder %s reuses builddir %s"
 623                                          % (b['name'], b['builddir']))
 624                     dirnames.append(b['builddir'])
 625
 626                 unscheduled_buildernames = buildernames[:]
 627                 schedulernames = []
 628                 for s in schedulers:
 629                     for b in s.listBuilderNames():
 630                         assert b in buildernames, \
 631                                "%s uses unknown builder %s" % (s, b)
 632                         if b in unscheduled_buildernames:
 633                             unscheduled_buildernames.remove(b)
 634
 635                     if s.name in schedulernames:
 636                         # TODO: schedulers share a namespace with other Service
 637                         # children of the BuildMaster node, like status plugins, the
 638                         # Manhole, the ChangeMaster, and the BotMaster (although most
 639                         # of these don't have names)
 640                         msg = ("Schedulers must have unique names, but "
 641                                "'%s' was a duplicate" % (s.name,))
 642                         raise ValueError(msg)
 643                     schedulernames.append(s.name)
 644
 645                 if unscheduled_buildernames:
 646                     log.msg("Warning: some Builders have no Schedulers to drive them:"
 647                             " %s" % (unscheduled_buildernames,))
 648
 649                 # assert that all locks used by the Builds and their Steps are
 650                 # uniquely named.
 651                 lock_dict = {}
 652                 for b in builders:
 653                     for l in b.get('locks', []):
 654                         if isinstance(l, locks.LockAccess): # User specified access to the lock
 655                             l = l.lockid
 656                         if lock_dict.has_key(l.name):
 657                             if lock_dict[l.name] is not l:
 658                                 raise ValueError("Two different locks (%s and %s) "
 659                                                  "share the name %s"
 660                                                  % (l, lock_dict[l.name], l.name))
 661                         else:
 662                             lock_dict[l.name] = l
 663                     # TODO: this will break with any BuildFactory that doesn't use a
 664                     # .steps list, but I think the verification step is more
 665                     # important.
 666                     for s in b['factory'].steps:
 667                         for l in s[1].get('locks', []):
 668                             if isinstance(l, locks.LockAccess): # User specified access to the lock
 669                                 l = l.lockid
 670                             if lock_dict.has_key(l.name):
 671                                 if lock_dict[l.name] is not l:
 672                                     raise ValueError("Two different locks (%s and %s)"
 673                                                      " share the name %s"
 674                                                      % (l, lock_dict[l.name], l.name))
 675                             else:
 676                                 lock_dict[l.name] = l
 677
 678                 if not isinstance(properties, dict):
 679                     raise ValueError("c['properties'] must be a dictionary")
 680
 681                 # slavePortnum supposed to be a strports specification
 682                 if type(slavePortnum) is int:
 683                     slavePortnum = "tcp:%d" % slavePortnum
 684
 685                 # now we're committed to implementing the new configuration, so do
 686                 # it atomically
 687                 # TODO: actually, this is spread across a couple of Deferreds, so it
 688                 # really isn't atomic.
 689
 690                 d = defer.succeed(None)
 691
 692                 self.projectName = projectName
 693                 self.projectURL = projectURL
 694                 self.buildbotURL = buildbotURL
 695
 696                 self.properties = Properties()
 697                 self.properties.update(properties, self.configFileName)
 698
 699                 self.status.logCompressionLimit = logCompressionLimit
 700                 self.status.logCompressionMethod = logCompressionMethod
 701                 self.status.logMaxSize = logMaxSize
 702                 self.status.logMaxTailSize = logMaxTailSize
 703                 # Update any of our existing builders with the current log parameters.
 704                 # This is required so that the new value is picked up after a
 705                 # reconfig.
 706                 for builder in self.botmaster.builders.values():
 707                     builder.builder_status.setLogCompressionLimit(logCompressionLimit)
 708                     builder.builder_status.setLogCompressionMethod(logCompressionMethod)
 709                     builder.builder_status.setLogMaxSize(logMaxSize)
 710                     builder.builder_status.setLogMaxTailSize(logMaxTailSize)
 711
 712                 if mergeRequests is not None:
 713                     self.botmaster.mergeRequests = mergeRequests
 714                 if prioritizeBuilders is not None:
 715                     self.botmaster.prioritizeBuilders = prioritizeBuilders
 716
 717                 self.buildCacheSize = buildCacheSize
 718                 self.eventHorizon = eventHorizon
 719                 self.logHorizon = logHorizon
 720                 self.buildHorizon = buildHorizon
 721
 722                 # self.slaves: Disconnect any that were attached and removed from the
 723                 # list. Update self.checker with the new list of passwords, including
 724                 # debug/change/status.
 725                 d.addCallback(lambda res: self.loadConfig_Slaves(slaves))
 726
 727                 # self.debugPassword
 728                 if debugPassword:
 729                     self.checker.addUser("debug", debugPassword)
 730                     self.debugPassword = debugPassword
 731
 732                 # self.manhole
 733                 if manhole != self.manhole:
 734                     # changing
 735                     if self.manhole:
 736                         # disownServiceParent may return a Deferred
 737                         d.addCallback(lambda res: self.manhole.disownServiceParent())
 738                         def _remove(res):
 739                             self.manhole = None
 740                             return res
 741                         d.addCallback(_remove)
 742                     if manhole:
 743                         def _add(res):
 744                             self.manhole = manhole
 745                             manhole.setServiceParent(self)
 746                         d.addCallback(_add)
 747
 748                 # add/remove self.botmaster.builders to match builders. The
 749                 # botmaster will handle startup/shutdown issues.
 750                 d.addCallback(lambda res: self.loadConfig_Builders(builders))
 751
 752                 d.addCallback(lambda res: self.loadConfig_status(status))
 753
 754                 # Schedulers are added after Builders in case they start right away
 755                 d.addCallback(lambda res: self.loadConfig_Schedulers(schedulers))
 756                 # and Sources go after Schedulers for the same reason
 757                 d.addCallback(lambda res: self.loadConfig_Sources(change_sources))
 758
 759                 # self.slavePort
 760                 if self.slavePortnum != slavePortnum:
 761                     if self.slavePort:
 762                         def closeSlavePort(res):
 763                             d1 = self.slavePort.disownServiceParent()
 764                             self.slavePort = None
 765                             return d1
 766                         d.addCallback(closeSlavePort)
 767                     if slavePortnum is not None:
 768                         def openSlavePort(res):
 769                             self.slavePort = strports.service(slavePortnum,
 770                                                               self.slaveFactory)
 771                             self.slavePort.setServiceParent(self)
 772                         d.addCallback(openSlavePort)
 773                         log.msg("BuildMaster listening on port %s" % slavePortnum)
 774                     self.slavePortnum = slavePortnum
 775
 776                 log.msg("configuration update started")
 777                 def _done(res):
 778                     self.readConfig = True
 779                     log.msg("configuration update complete")
 780                 d.addCallback(_done)
 781                 d.addCallback(lambda res: self.botmaster.maybeStartAllBuilds())
 782                 return d
 783
 784         if buildbot_dir:
 785             basedir = buildbot_dir
 786         else:
 787             if PKGDATADIR:
 788                 basedir = os.path.join(PKGDATADIR, 'buildbot')
 789             else:
 790                 basedir = os.path.join(SRCDIR, 'buildbot')
 791         os.chdir(basedir)
 792         if not os.path.exists(os.path.join(basedir, 'builddir')):
 793             os.makedirs(os.path.join(basedir, 'builddir'))
 794         master_cfg_path = mastercfgfile
 795
 796         JhBuildMaster(basedir, master_cfg_path).setServiceParent(application)
 797
 798         JhBuildbotApplicationRunner.application = application
 799         JhBuildbotApplicationRunner(options).run()
 800
 801     def stop(self, config, pidfile):
 802         try:
 803             pid = int(file(pidfile).read())
 804         except:
 805             raise FatalError(_('failed to get buildbot PID'))
 806
 807         os.kill(pid, signal.SIGTERM)
 808
 809     def reload_server_config(self, config, pidfile):
 810         try:
 811             pid = int(file(pidfile).read())
 812         except:
 813             raise FatalError(_('failed to get buildbot PID'))
 814
 815         os.kill(pid, signal.SIGHUP)
 816
 817
 818 register_command(cmd_bot)
 819