buildbot/master.py

   1 # -*- test-case-name: buildbot.test.test_run -*-
   2
   3 import os
   4 signal = None
   5 try:
   6     import signal
   7 except ImportError:
   8     pass
   9 from cPickle import load
  10 import warnings
  11
  12 from zope.interface import implements
  13 from twisted.python import log, components
  14 from twisted.internet import defer, reactor
  15 from twisted.spread import pb
  16 from twisted.cred import portal, checkers
  17 from twisted.application import service, strports
  18 from twisted.persisted import styles
  19
  20 import buildbot
  21 # sibling imports
  22 from buildbot.util import now
  23 from buildbot.pbutil import NewCredPerspective
  24 from buildbot.process.builder import Builder, IDLE
  25 from buildbot.process.base import BuildRequest
  26 from buildbot.status.builder import Status
  27 from buildbot.changes.changes import Change, ChangeMaster
  28 from buildbot.sourcestamp import SourceStamp
  29 from buildbot.buildslave import BuildSlave
  30 from buildbot import interfaces, locks
  31 from buildbot.process.properties import Properties
  32
  33 ########################################
  34
  35 class BotMaster(service.MultiService):
  36
  37     """This is the master-side service which manages remote buildbot slaves.
  38     It provides them with BuildSlaves, and distributes file change
  39     notification messages to them.
  40     """
  41
  42     debug = 0
  43
  44     def __init__(self):
  45         service.MultiService.__init__(self)
  46         self.builders = {}
  47         self.builderNames = []
  48         # builders maps Builder names to instances of bb.p.builder.Builder,
  49         # which is the master-side object that defines and controls a build.
  50         # They are added by calling botmaster.addBuilder() from the startup
  51         # code.
  52
  53         # self.slaves contains a ready BuildSlave instance for each
  54         # potential buildslave, i.e. all the ones listed in the config file.
  55         # If the slave is connected, self.slaves[slavename].slave will
  56         # contain a RemoteReference to their Bot instance. If it is not
  57         # connected, that attribute will hold None.
  58         self.slaves = {} # maps slavename to BuildSlave
  59         self.statusClientService = None
  60         self.watchers = {}
  61
  62         # self.locks holds the real Lock instances
  63         self.locks = {}
  64
  65     # these four are convenience functions for testing
  66
  67     def waitUntilBuilderAttached(self, name):
  68         b = self.builders[name]
  69         #if b.slaves:
  70         #    return defer.succeed(None)
  71         d = defer.Deferred()
  72         b.watchers['attach'].append(d)
  73         return d
  74
  75     def waitUntilBuilderDetached(self, name):
  76         b = self.builders.get(name)
  77         if not b or not b.slaves:
  78             return defer.succeed(None)
  79         d = defer.Deferred()
  80         b.watchers['detach'].append(d)
  81         return d
  82
  83     def waitUntilBuilderFullyDetached(self, name):
  84         b = self.builders.get(name)
  85         # TODO: this looks too deeply inside the Builder object
  86         if not b or not b.slaves:
  87             return defer.succeed(None)
  88         d = defer.Deferred()
  89         b.watchers['detach_all'].append(d)
  90         return d
  91
  92     def waitUntilBuilderIdle(self, name):
  93         b = self.builders[name]
  94         # TODO: this looks way too deeply inside the Builder object
  95         for sb in b.slaves:
  96             if sb.state != IDLE:
  97                 d = defer.Deferred()
  98                 b.watchers['idle'].append(d)
  99                 return d
 100         return defer.succeed(None)
 101
 102     def loadConfig_Slaves(self, new_slaves):
 103         old_slaves = [c for c in list(self)
 104                       if interfaces.IBuildSlave.providedBy(c)]
 105
 106         # identify added/removed slaves. For each slave we construct a tuple
 107         # of (name, password, class), and we consider the slave to be already
 108         # present if the tuples match. (we include the class to make sure
 109         # that BuildSlave(name,pw) is different than
 110         # SubclassOfBuildSlave(name,pw) ). If the password or class has
 111         # changed, we will remove the old version of the slave and replace it
 112         # with a new one. If anything else has changed, we just update the
 113         # old BuildSlave instance in place. If the name has changed, of
 114         # course, it looks exactly the same as deleting one slave and adding
 115         # an unrelated one.
 116         old_t = {}
 117         for s in old_slaves:
 118             old_t[(s.slavename, s.password, s.__class__)] = s
 119         new_t = {}
 120         for s in new_slaves:
 121             new_t[(s.slavename, s.password, s.__class__)] = s
 122         removed = [old_t[t]
 123                    for t in old_t
 124                    if t not in new_t]
 125         added = [new_t[t]
 126                  for t in new_t
 127                  if t not in old_t]
 128         remaining_t = [t
 129                        for t in new_t
 130                        if t in old_t]
 131         # removeSlave will hang up on the old bot
 132         dl = []
 133         for s in removed:
 134             dl.append(self.removeSlave(s))
 135         d = defer.DeferredList(dl, fireOnOneErrback=True)
 136         def _add(res):
 137             for s in added:
 138                 self.addSlave(s)
 139             for t in remaining_t:
 140                 old_t[t].update(new_t[t])
 141         d.addCallback(_add)
 142         return d
 143
 144     def addSlave(self, s):
 145         s.setServiceParent(self)
 146         s.setBotmaster(self)
 147         self.slaves[s.slavename] = s
 148
 149     def removeSlave(self, s):
 150         # TODO: technically, disownServiceParent could return a Deferred
 151         s.disownServiceParent()
 152         d = self.slaves[s.slavename].disconnect()
 153         del self.slaves[s.slavename]
 154         return d
 155
 156     def slaveLost(self, bot):
 157         for name, b in self.builders.items():
 158             if bot.slavename in b.slavenames:
 159                 b.detached(bot)
 160
 161     def getBuildersForSlave(self, slavename):
 162         return [b
 163                 for b in self.builders.values()
 164                 if slavename in b.slavenames]
 165
 166     def getBuildernames(self):
 167         return self.builderNames
 168
 169     def getBuilders(self):
 170         allBuilders = [self.builders[name] for name in self.builderNames]
 171         return allBuilders
 172
 173     def setBuilders(self, builders):
 174         self.builders = {}
 175         self.builderNames = []
 176         for b in builders:
 177             for slavename in b.slavenames:
 178                 # this is actually validated earlier
 179                 assert slavename in self.slaves
 180             self.builders[b.name] = b
 181             self.builderNames.append(b.name)
 182             b.setBotmaster(self)
 183         d = self._updateAllSlaves()
 184         return d
 185
 186     def _updateAllSlaves(self):
 187         """Notify all buildslaves about changes in their Builders."""
 188         dl = [s.updateSlave() for s in self.slaves.values()]
 189         return defer.DeferredList(dl)
 190
 191     def maybeStartAllBuilds(self):
 192         builders = self.builders.values()
 193         def _sortfunc(b1, b2):
 194             t1 = b1.getOldestRequestTime()
 195             t2 = b2.getOldestRequestTime()
 196             # If t1 or t2 is None, then there are no build requests,
 197             # so sort it at the end
 198             if t1 is None:
 199                 return 1
 200             if t2 is None:
 201                 return -1
 202             return cmp(t1, t2)
 203         builders.sort(cmp=_sortfunc)
 204         for b in builders:
 205             b.maybeStartBuild()
 206
 207     def getPerspective(self, slavename):
 208         return self.slaves[slavename]
 209
 210     def shutdownSlaves(self):
 211         # TODO: make this into a bot method rather than a builder method
 212         for b in self.slaves.values():
 213             b.shutdownSlave()
 214
 215     def stopService(self):
 216         for b in self.builders.values():
 217             b.builder_status.addPointEvent(["master", "shutdown"])
 218             b.builder_status.saveYourself()
 219         return service.Service.stopService(self)
 220
 221     def getLockByID(self, lockid):
 222         """Convert a Lock identifier into an actual Lock instance.
 223         @param lockid: a locks.MasterLock or locks.SlaveLock instance
 224         @return: a locks.RealMasterLock or locks.RealSlaveLock instance
 225         """
 226         assert isinstance(lockid, (locks.MasterLock, locks.SlaveLock))
 227         if not lockid in self.locks:
 228             self.locks[lockid] = lockid.lockClass(lockid)
 229         # if the master.cfg file has changed maxCount= on the lock, the next
 230         # time a build is started, they'll get a new RealLock instance. Note
 231         # that this requires that MasterLock and SlaveLock (marker) instances
 232         # be hashable and that they should compare properly.
 233         return self.locks[lockid]
 234
 235 ########################################
 236
 237
 238
 239 class DebugPerspective(NewCredPerspective):
 240     def attached(self, mind):
 241         return self
 242     def detached(self, mind):
 243         pass
 244
 245     def perspective_requestBuild(self, buildername, reason, branch, revision, properties={}):
 246         c = interfaces.IControl(self.master)
 247         bc = c.getBuilder(buildername)
 248         ss = SourceStamp(branch, revision)
 249         bpr = Properties()
 250         bpr.update(properties, "remote requestBuild")
 251         br = BuildRequest(reason, ss, builderName=buildername, properties=bpr)
 252         bc.requestBuild(br)
 253
 254     def perspective_pingBuilder(self, buildername):
 255         c = interfaces.IControl(self.master)
 256         bc = c.getBuilder(buildername)
 257         bc.ping()
 258
 259     def perspective_fakeChange(self, file, revision=None, who="fakeUser",
 260                                branch=None):
 261         change = Change(who, [file], "some fake comments\n",
 262                         branch=branch, revision=revision)
 263         c = interfaces.IControl(self.master)
 264         c.addChange(change)
 265
 266     def perspective_setCurrentState(self, buildername, state):
 267         builder = self.botmaster.builders.get(buildername)
 268         if not builder: return
 269         if state == "offline":
 270             builder.statusbag.currentlyOffline()
 271         if state == "idle":
 272             builder.statusbag.currentlyIdle()
 273         if state == "waiting":
 274             builder.statusbag.currentlyWaiting(now()+10)
 275         if state == "building":
 276             builder.statusbag.currentlyBuilding(None)
 277     def perspective_reload(self):
 278         print "doing reload of the config file"
 279         self.master.loadTheConfigFile()
 280     def perspective_pokeIRC(self):
 281         print "saying something on IRC"
 282         from buildbot.status import words
 283         for s in self.master:
 284             if isinstance(s, words.IRC):
 285                 bot = s.f
 286                 for channel in bot.channels:
 287                     print " channel", channel
 288                     bot.p.msg(channel, "Ow, quit it")
 289
 290     def perspective_print(self, msg):
 291         print "debug", msg
 292
 293 class Dispatcher(styles.Versioned):
 294     implements(portal.IRealm)
 295     persistenceVersion = 2
 296
 297     def __init__(self):
 298         self.names = {}
 299
 300     def upgradeToVersion1(self):
 301         self.master = self.botmaster.parent
 302     def upgradeToVersion2(self):
 303         self.names = {}
 304
 305     def register(self, name, afactory):
 306         self.names[name] = afactory
 307     def unregister(self, name):
 308         del self.names[name]
 309
 310     def requestAvatar(self, avatarID, mind, interface):
 311         assert interface == pb.IPerspective
 312         afactory = self.names.get(avatarID)
 313         if afactory:
 314             p = afactory.getPerspective()
 315         elif avatarID == "debug":
 316             p = DebugPerspective()
 317             p.master = self.master
 318             p.botmaster = self.botmaster
 319         elif avatarID == "statusClient":
 320             p = self.statusClientService.getPerspective()
 321         else:
 322             # it must be one of the buildslaves: no other names will make it
 323             # past the checker
 324             p = self.botmaster.getPerspective(avatarID)
 325
 326         if not p:
 327             raise ValueError("no perspective for '%s'" % avatarID)
 328
 329         d = defer.maybeDeferred(p.attached, mind)
 330         d.addCallback(self._avatarAttached, mind)
 331         return d
 332
 333     def _avatarAttached(self, p, mind):
 334         return (pb.IPerspective, p, lambda p=p,mind=mind: p.detached(mind))
 335
 336 ########################################
 337
 338 # service hierarchy:
 339 #  BuildMaster
 340 #   BotMaster
 341 #   ChangeMaster
 342 #    all IChangeSource objects
 343 #   StatusClientService
 344 #   TCPClient(self.ircFactory)
 345 #   TCPServer(self.slaveFactory) -> dispatcher.requestAvatar
 346 #   TCPServer(self.site)
 347 #   UNIXServer(ResourcePublisher(self.site))
 348
 349
 350 class BuildMaster(service.MultiService, styles.Versioned):
 351     debug = 0
 352     persistenceVersion = 3
 353     manhole = None
 354     debugPassword = None
 355     projectName = "(unspecified)"
 356     projectURL = None
 357     buildbotURL = None
 358     change_svc = None
 359     properties = Properties()
 360
 361     def __init__(self, basedir, configFileName="master.cfg"):
 362         service.MultiService.__init__(self)
 363         self.setName("buildmaster")
 364         self.basedir = basedir
 365         self.configFileName = configFileName
 366
 367         # the dispatcher is the realm in which all inbound connections are
 368         # looked up: slave builders, change notifications, status clients, and
 369         # the debug port
 370         dispatcher = Dispatcher()
 371         dispatcher.master = self
 372         self.dispatcher = dispatcher
 373         self.checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
 374         # the checker starts with no user/passwd pairs: they are added later
 375         p = portal.Portal(dispatcher)
 376         p.registerChecker(self.checker)
 377         self.slaveFactory = pb.PBServerFactory(p)
 378         self.slaveFactory.unsafeTracebacks = True # let them see exceptions
 379
 380         self.slavePortnum = None
 381         self.slavePort = None
 382
 383         self.botmaster = BotMaster()
 384         self.botmaster.setName("botmaster")
 385         self.botmaster.setServiceParent(self)
 386         dispatcher.botmaster = self.botmaster
 387
 388         self.status = Status(self.botmaster, self.basedir)
 389
 390         self.statusTargets = []
 391
 392         # this ChangeMaster is a dummy, only used by tests. In the real
 393         # buildmaster, where the BuildMaster instance is activated
 394         # (startService is called) by twistd, this attribute is overwritten.
 395         self.useChanges(ChangeMaster())
 396
 397         self.readConfig = False
 398
 399     def upgradeToVersion1(self):
 400         self.dispatcher = self.slaveFactory.root.portal.realm
 401
 402     def upgradeToVersion2(self): # post-0.4.3
 403         self.webServer = self.webTCPPort
 404         del self.webTCPPort
 405         self.webDistribServer = self.webUNIXPort
 406         del self.webUNIXPort
 407         self.configFileName = "master.cfg"
 408
 409     def upgradeToVersion3(self):
 410         # post 0.6.3, solely to deal with the 0.6.3 breakage. Starting with
 411         # 0.6.5 I intend to do away with .tap files altogether
 412         self.services = []
 413         self.namedServices = {}
 414         del self.change_svc
 415
 416     def startService(self):
 417         service.MultiService.startService(self)
 418         self.loadChanges() # must be done before loading the config file
 419         if not self.readConfig:
 420             # TODO: consider catching exceptions during this call to
 421             # loadTheConfigFile and bailing (reactor.stop) if it fails,
 422             # since without a config file we can't do anything except reload
 423             # the config file, and it would be nice for the user to discover
 424             # this quickly.
 425             self.loadTheConfigFile()
 426         if signal and hasattr(signal, "SIGHUP"):
 427             signal.signal(signal.SIGHUP, self._handleSIGHUP)
 428         for b in self.botmaster.builders.values():
 429             b.builder_status.addPointEvent(["master", "started"])
 430             b.builder_status.saveYourself()
 431
 432     def useChanges(self, changes):
 433         if self.change_svc:
 434             # TODO: can return a Deferred
 435             self.change_svc.disownServiceParent()
 436         self.change_svc = changes
 437         self.change_svc.basedir = self.basedir
 438         self.change_svc.setName("changemaster")
 439         self.dispatcher.changemaster = self.change_svc
 440         self.change_svc.setServiceParent(self)
 441
 442     def loadChanges(self):
 443         filename = os.path.join(self.basedir, "changes.pck")
 444         try:
 445             changes = load(open(filename, "rb"))
 446             styles.doUpgrade()
 447         except IOError:
 448             log.msg("changes.pck missing, using new one")
 449             changes = ChangeMaster()
 450         except EOFError:
 451             log.msg("corrupted changes.pck, using new one")
 452             changes = ChangeMaster()
 453         self.useChanges(changes)
 454
 455     def _handleSIGHUP(self, *args):
 456         reactor.callLater(0, self.loadTheConfigFile)
 457
 458     def getStatus(self):
 459         """
 460         @rtype: L{buildbot.status.builder.Status}
 461         """
 462         return self.status
 463
 464     def loadTheConfigFile(self, configFile=None):
 465         if not configFile:
 466             configFile = os.path.join(self.basedir, self.configFileName)
 467
 468         log.msg("Creating BuildMaster -- buildbot.version: %s" % buildbot.version)
 469         log.msg("loading configuration from %s" % configFile)
 470         configFile = os.path.expanduser(configFile)
 471
 472         try:
 473             f = open(configFile, "r")
 474         except IOError, e:
 475             log.msg("unable to open config file '%s'" % configFile)
 476             log.msg("leaving old configuration in place")
 477             log.err(e)
 478             return
 479
 480         try:
 481             self.loadConfig(f)
 482         except:
 483             log.msg("error during loadConfig")
 484             log.err()
 485             log.msg("The new config file is unusable, so I'll ignore it.")
 486             log.msg("I will keep using the previous config file instead.")
 487         f.close()
 488
 489     def loadConfig(self, f):
 490         """Internal function to load a specific configuration file. Any
 491         errors in the file will be signalled by raising an exception.
 492
 493         @return: a Deferred that will fire (with None) when the configuration
 494         changes have been completed. This may involve a round-trip to each
 495         buildslave that was involved."""
 496
 497         localDict = {'basedir': os.path.expanduser(self.basedir)}
 498         try:
 499             exec f in localDict
 500         except:
 501             log.msg("error while parsing config file")
 502             raise
 503
 504         try:
 505             config = localDict['BuildmasterConfig']
 506         except KeyError:
 507             log.err("missing config dictionary")
 508             log.err("config file must define BuildmasterConfig")
 509             raise
 510
 511         known_keys = ("bots", "slaves",
 512                       "sources", "change_source",
 513                       "schedulers", "builders",
 514                       "slavePortnum", "debugPassword", "logCompressionLimit",
 515                       "manhole", "status", "projectName", "projectURL",
 516                       "buildbotURL", "properties"
 517                       )
 518         for k in config.keys():
 519             if k not in known_keys:
 520                 log.msg("unknown key '%s' defined in config dictionary" % k)
 521
 522         try:
 523             # required
 524             schedulers = config['schedulers']
 525             builders = config['builders']
 526             for k in builders:
 527                 if k['name'].startswith("_"):
 528                     errmsg = ("builder names must not start with an "
 529                               "underscore: " + k['name'])
 530                     log.err(errmsg)
 531                     raise ValueError(errmsg)
 532
 533             slavePortnum = config['slavePortnum']
 534             #slaves = config['slaves']
 535             #change_source = config['change_source']
 536
 537             # optional
 538             debugPassword = config.get('debugPassword')
 539             manhole = config.get('manhole')
 540             status = config.get('status', [])
 541             projectName = config.get('projectName')
 542             projectURL = config.get('projectURL')
 543             buildbotURL = config.get('buildbotURL')
 544             properties = config.get('properties', {})
 545             logCompressionLimit = config.get('logCompressionLimit')
 546             if logCompressionLimit is not None and not \
 547                     isinstance(logCompressionLimit, int):
 548                 raise ValueError("logCompressionLimit needs to be bool or int")
 549
 550         except KeyError, e:
 551             log.msg("config dictionary is missing a required parameter")
 552             log.msg("leaving old configuration in place")
 553             raise
 554
 555         #if "bots" in config:
 556         #    raise KeyError("c['bots'] is no longer accepted")
 557
 558         slaves = config.get('slaves', [])
 559         if "bots" in config:
 560             m = ("c['bots'] is deprecated as of 0.7.6 and will be "
 561                  "removed by 0.8.0 . Please use c['slaves'] instead.")
 562             log.msg(m)
 563             warnings.warn(m, DeprecationWarning)
 564             for name, passwd in config['bots']:
 565                 slaves.append(BuildSlave(name, passwd))
 566
 567         if "bots" not in config and "slaves" not in config:
 568             log.msg("config dictionary must have either 'bots' or 'slaves'")
 569             log.msg("leaving old configuration in place")
 570             raise KeyError("must have either 'bots' or 'slaves'")
 571
 572         #if "sources" in config:
 573         #    raise KeyError("c['sources'] is no longer accepted")
 574
 575         change_source = config.get('change_source', [])
 576         if isinstance(change_source, (list, tuple)):
 577             change_sources = change_source
 578         else:
 579             change_sources = [change_source]
 580         if "sources" in config:
 581             m = ("c['sources'] is deprecated as of 0.7.6 and will be "
 582                  "removed by 0.8.0 . Please use c['change_source'] instead.")
 583             log.msg(m)
 584             warnings.warn(m, DeprecationWarning)
 585             for s in config['sources']:
 586                 change_sources.append(s)
 587
 588         # do some validation first
 589         for s in slaves:
 590             assert interfaces.IBuildSlave.providedBy(s)
 591             if s.slavename in ("debug", "change", "status"):
 592                 raise KeyError(
 593                     "reserved name '%s' used for a bot" % s.slavename)
 594         if config.has_key('interlocks'):
 595             raise KeyError("c['interlocks'] is no longer accepted")
 596
 597         assert isinstance(change_sources, (list, tuple))
 598         for s in change_sources:
 599             assert interfaces.IChangeSource(s, None)
 600         # this assertion catches c['schedulers'] = Scheduler(), since
 601         # Schedulers are service.MultiServices and thus iterable.
 602         errmsg = "c['schedulers'] must be a list of Scheduler instances"
 603         assert isinstance(schedulers, (list, tuple)), errmsg
 604         for s in schedulers:
 605             assert interfaces.IScheduler(s, None), errmsg
 606         assert isinstance(status, (list, tuple))
 607         for s in status:
 608             assert interfaces.IStatusReceiver(s, None)
 609
 610         slavenames = [s.slavename for s in slaves]
 611         buildernames = []
 612         dirnames = []
 613         for b in builders:
 614             if type(b) is tuple:
 615                 raise ValueError("builder %s must be defined with a dict, "
 616                                  "not a tuple" % b[0])
 617             if b.has_key('slavename') and b['slavename'] not in slavenames:
 618                 raise ValueError("builder %s uses undefined slave %s" \
 619                                  % (b['name'], b['slavename']))
 620             for n in b.get('slavenames', []):
 621                 if n not in slavenames:
 622                     raise ValueError("builder %s uses undefined slave %s" \
 623                                      % (b['name'], n))
 624             if b['name'] in buildernames:
 625                 raise ValueError("duplicate builder name %s"
 626                                  % b['name'])
 627             buildernames.append(b['name'])
 628             if b['builddir'] in dirnames:
 629                 raise ValueError("builder %s reuses builddir %s"
 630                                  % (b['name'], b['builddir']))
 631             dirnames.append(b['builddir'])
 632
 633         unscheduled_buildernames = buildernames[:]
 634         schedulernames = []
 635         for s in schedulers:
 636             for b in s.listBuilderNames():
 637                 assert b in buildernames, \
 638                        "%s uses unknown builder %s" % (s, b)
 639                 if b in unscheduled_buildernames:
 640                     unscheduled_buildernames.remove(b)
 641
 642             if s.name in schedulernames:
 643                 # TODO: schedulers share a namespace with other Service
 644                 # children of the BuildMaster node, like status plugins, the
 645                 # Manhole, the ChangeMaster, and the BotMaster (although most
 646                 # of these don't have names)
 647                 msg = ("Schedulers must have unique names, but "
 648                        "'%s' was a duplicate" % (s.name,))
 649                 raise ValueError(msg)
 650             schedulernames.append(s.name)
 651
 652         if unscheduled_buildernames:
 653             log.msg("Warning: some Builders have no Schedulers to drive them:"
 654                     " %s" % (unscheduled_buildernames,))
 655
 656         # assert that all locks used by the Builds and their Steps are
 657         # uniquely named.
 658         lock_dict = {}
 659         for b in builders:
 660             for l in b.get('locks', []):
 661                 if isinstance(l, locks.LockAccess): # User specified access to the lock
 662                     l = l.lockid
 663                 if lock_dict.has_key(l.name):
 664                     if lock_dict[l.name] is not l:
 665                         raise ValueError("Two different locks (%s and %s) "
 666                                          "share the name %s"
 667                                          % (l, lock_dict[l.name], l.name))
 668                 else:
 669                     lock_dict[l.name] = l
 670             # TODO: this will break with any BuildFactory that doesn't use a
 671             # .steps list, but I think the verification step is more
 672             # important.
 673             for s in b['factory'].steps:
 674                 for l in s[1].get('locks', []):
 675                     if isinstance(l, locks.LockAccess): # User specified access to the lock
 676                         l = l.lockid
 677                     if lock_dict.has_key(l.name):
 678                         if lock_dict[l.name] is not l:
 679                             raise ValueError("Two different locks (%s and %s)"
 680                                              " share the name %s"
 681                                              % (l, lock_dict[l.name], l.name))
 682                     else:
 683                         lock_dict[l.name] = l
 684
 685         if not isinstance(properties, dict):
 686             raise ValueError("c['properties'] must be a dictionary")
 687
 688         # slavePortnum supposed to be a strports specification
 689         if type(slavePortnum) is int:
 690             slavePortnum = "tcp:%d" % slavePortnum
 691
 692         # now we're committed to implementing the new configuration, so do
 693         # it atomically
 694         # TODO: actually, this is spread across a couple of Deferreds, so it
 695         # really isn't atomic.
 696
 697         d = defer.succeed(None)
 698
 699         self.projectName = projectName
 700         self.projectURL = projectURL
 701         self.buildbotURL = buildbotURL
 702
 703         self.properties = Properties()
 704         self.properties.update(properties, self.configFileName)
 705         if logCompressionLimit is not None:
 706             self.status.logCompressionLimit = logCompressionLimit
 707
 708         # self.slaves: Disconnect any that were attached and removed from the
 709         # list. Update self.checker with the new list of passwords, including
 710         # debug/change/status.
 711         d.addCallback(lambda res: self.loadConfig_Slaves(slaves))
 712
 713         # self.debugPassword
 714         if debugPassword:
 715             self.checker.addUser("debug", debugPassword)
 716             self.debugPassword = debugPassword
 717
 718         # self.manhole
 719         if manhole != self.manhole:
 720             # changing
 721             if self.manhole:
 722                 # disownServiceParent may return a Deferred
 723                 d.addCallback(lambda res: self.manhole.disownServiceParent())
 724                 def _remove(res):
 725                     self.manhole = None
 726                     return res
 727                 d.addCallback(_remove)
 728             if manhole:
 729                 def _add(res):
 730                     self.manhole = manhole
 731                     manhole.setServiceParent(self)
 732                 d.addCallback(_add)
 733
 734         # add/remove self.botmaster.builders to match builders. The
 735         # botmaster will handle startup/shutdown issues.
 736         d.addCallback(lambda res: self.loadConfig_Builders(builders))
 737
 738         d.addCallback(lambda res: self.loadConfig_status(status))
 739
 740         # Schedulers are added after Builders in case they start right away
 741         d.addCallback(lambda res: self.loadConfig_Schedulers(schedulers))
 742         # and Sources go after Schedulers for the same reason
 743         d.addCallback(lambda res: self.loadConfig_Sources(change_sources))
 744
 745         # self.slavePort
 746         if self.slavePortnum != slavePortnum:
 747             if self.slavePort:
 748                 def closeSlavePort(res):
 749                     d1 = self.slavePort.disownServiceParent()
 750                     self.slavePort = None
 751                     return d1
 752                 d.addCallback(closeSlavePort)
 753             if slavePortnum is not None:
 754                 def openSlavePort(res):
 755                     self.slavePort = strports.service(slavePortnum,
 756                                                       self.slaveFactory)
 757                     self.slavePort.setServiceParent(self)
 758                 d.addCallback(openSlavePort)
 759                 log.msg("BuildMaster listening on port %s" % slavePortnum)
 760             self.slavePortnum = slavePortnum
 761
 762         log.msg("configuration update started")
 763         def _done(res):
 764             self.readConfig = True
 765             log.msg("configuration update complete")
 766         d.addCallback(_done)
 767         d.addCallback(lambda res: self.botmaster.maybeStartAllBuilds())
 768         return d
 769
 770     def loadConfig_Slaves(self, new_slaves):
 771         # set up the Checker with the names and passwords of all valid bots
 772         self.checker.users = {} # violates abstraction, oh well
 773         for s in new_slaves:
 774             self.checker.addUser(s.slavename, s.password)
 775         self.checker.addUser("change", "changepw")
 776         # let the BotMaster take care of the rest
 777         return self.botmaster.loadConfig_Slaves(new_slaves)
 778
 779     def loadConfig_Sources(self, sources):
 780         if not sources:
 781             log.msg("warning: no ChangeSources specified in c['change_source']")
 782         # shut down any that were removed, start any that were added
 783         deleted_sources = [s for s in self.change_svc if s not in sources]
 784         added_sources = [s for s in sources if s not in self.change_svc]
 785         dl = [self.change_svc.removeSource(s) for s in deleted_sources]
 786         def addNewOnes(res):
 787             [self.change_svc.addSource(s) for s in added_sources]
 788         d = defer.DeferredList(dl, fireOnOneErrback=1, consumeErrors=0)
 789         d.addCallback(addNewOnes)
 790         return d
 791
 792     def allSchedulers(self):
 793         return [child for child in self
 794                 if interfaces.IScheduler.providedBy(child)]
 795
 796
 797     def loadConfig_Schedulers(self, newschedulers):
 798         oldschedulers = self.allSchedulers()
 799         removed = [s for s in oldschedulers if s not in newschedulers]
 800         added = [s for s in newschedulers if s not in oldschedulers]
 801         dl = [defer.maybeDeferred(s.disownServiceParent) for s in removed]
 802         def addNewOnes(res):
 803             log.msg("adding %d new schedulers, removed %d" %
 804                     (len(added), len(dl)))
 805             for s in added:
 806                 s.setServiceParent(self)
 807         d = defer.DeferredList(dl, fireOnOneErrback=1)
 808         d.addCallback(addNewOnes)
 809         if removed or added:
 810             # notify Downstream schedulers to potentially pick up
 811             # new schedulers now that we have removed and added some
 812             def updateDownstreams(res):
 813                 log.msg("notifying downstream schedulers of changes")
 814                 for s in newschedulers:
 815                     if interfaces.IDownstreamScheduler.providedBy(s):
 816                         s.checkUpstreamScheduler()
 817             d.addCallback(updateDownstreams)
 818         return d
 819
 820     def loadConfig_Builders(self, newBuilderData):
 821         somethingChanged = False
 822         newList = {}
 823         newBuilderNames = []
 824         allBuilders = self.botmaster.builders.copy()
 825         for data in newBuilderData:
 826             name = data['name']
 827             newList[name] = data
 828             newBuilderNames.append(name)
 829
 830         # identify all that were removed
 831         for oldname in self.botmaster.getBuildernames():
 832             if oldname not in newList:
 833                 log.msg("removing old builder %s" % oldname)
 834                 del allBuilders[oldname]
 835                 somethingChanged = True
 836                 # announce the change
 837                 self.status.builderRemoved(oldname)
 838
 839         # everything in newList is either unchanged, changed, or new
 840         for name, data in newList.items():
 841             old = self.botmaster.builders.get(name)
 842             basedir = data['builddir'] # used on both master and slave
 843             #name, slave, builddir, factory = data
 844             if not old: # new
 845                 # category added after 0.6.2
 846                 category = data.get('category', None)
 847                 log.msg("adding new builder %s for category %s" %
 848                         (name, category))
 849                 statusbag = self.status.builderAdded(name, basedir, category)
 850                 builder = Builder(data, statusbag)
 851                 allBuilders[name] = builder
 852                 somethingChanged = True
 853             elif old.compareToSetup(data):
 854                 # changed: try to minimize the disruption and only modify the
 855                 # pieces that really changed
 856                 diffs = old.compareToSetup(data)
 857                 log.msg("updating builder %s: %s" % (name, "\n".join(diffs)))
 858
 859                 statusbag = old.builder_status
 860                 statusbag.saveYourself() # seems like a good idea
 861                 # TODO: if the basedir was changed, we probably need to make
 862                 # a new statusbag
 863                 new_builder = Builder(data, statusbag)
 864                 new_builder.consumeTheSoulOfYourPredecessor(old)
 865                 # that migrates any retained slavebuilders too
 866
 867                 # point out that the builder was updated. On the Waterfall,
 868                 # this will appear just after any currently-running builds.
 869                 statusbag.addPointEvent(["config", "updated"])
 870
 871                 allBuilders[name] = new_builder
 872                 somethingChanged = True
 873             else:
 874                 # unchanged: leave it alone
 875                 log.msg("builder %s is unchanged" % name)
 876                 pass
 877
 878         if somethingChanged:
 879             sortedAllBuilders = [allBuilders[name] for name in newBuilderNames]
 880             d = self.botmaster.setBuilders(sortedAllBuilders)
 881             return d
 882         return None
 883
 884     def loadConfig_status(self, status):
 885         dl = []
 886
 887         # remove old ones
 888         for s in self.statusTargets[:]:
 889             if not s in status:
 890                 log.msg("removing IStatusReceiver", s)
 891                 d = defer.maybeDeferred(s.disownServiceParent)
 892                 dl.append(d)
 893                 self.statusTargets.remove(s)
 894         # after those are finished going away, add new ones
 895         def addNewOnes(res):
 896             for s in status:
 897                 if not s in self.statusTargets:
 898                     log.msg("adding IStatusReceiver", s)
 899                     s.setServiceParent(self)
 900                     self.statusTargets.append(s)
 901         d = defer.DeferredList(dl, fireOnOneErrback=1)
 902         d.addCallback(addNewOnes)
 903         return d
 904
 905
 906     def addChange(self, change):
 907         for s in self.allSchedulers():
 908             s.addChange(change)
 909
 910     def submitBuildSet(self, bs):
 911         # determine the set of Builders to use
 912         builders = []
 913         for name in bs.builderNames:
 914             b = self.botmaster.builders.get(name)
 915             if b:
 916                 if b not in builders:
 917                     builders.append(b)
 918                 continue
 919             # TODO: add aliases like 'all'
 920             raise KeyError("no such builder named '%s'" % name)
 921
 922         # now tell the BuildSet to create BuildRequests for all those
 923         # Builders and submit them
 924         bs.start(builders)
 925         self.status.buildsetSubmitted(bs.status)
 926
 927
 928 class Control:
 929     implements(interfaces.IControl)
 930
 931     def __init__(self, master):
 932         self.master = master
 933
 934     def addChange(self, change):
 935         self.master.change_svc.addChange(change)
 936
 937     def submitBuildSet(self, bs):
 938         self.master.submitBuildSet(bs)
 939
 940     def getBuilder(self, name):
 941         b = self.master.botmaster.builders[name]
 942         return interfaces.IBuilderControl(b)
 943
 944 components.registerAdapter(Control, BuildMaster, interfaces.IControl)
 945
 946 # so anybody who can get a handle on the BuildMaster can cause a build with:
 947 #  IControl(master).getBuilder("full-2.3").requestBuild(buildrequest)