buildbot/master.py

   1 # -*- test-case-name: buildbot.test.test_run -*-
   2
   3 import os
   4 signal = None
   5 try:
   6     import signal
   7 except ImportError:
   8     pass
   9 from cPickle import load
  10 import warnings
  11
  12 from zope.interface import implements
  13 from twisted.python import log, components
  14 from twisted.internet import defer, reactor
  15 from twisted.spread import pb
  16 from twisted.cred import portal, checkers
  17 from twisted.application import service, strports
  18 from twisted.persisted import styles
  19
  20 # sibling imports
  21 from buildbot.util import now
  22 from buildbot.pbutil import NewCredPerspective
  23 from buildbot.process.builder import Builder, IDLE
  24 from buildbot.process.base import BuildRequest
  25 from buildbot.status.builder import Status
  26 from buildbot.changes.changes import Change, ChangeMaster
  27 from buildbot.sourcestamp import SourceStamp
  28 from buildbot.buildslave import BuildSlave
  29 from buildbot import interfaces
  30
  31 ########################################
  32
  33 class BotMaster(service.MultiService):
  34
  35     """This is the master-side service which manages remote buildbot slaves.
  36     It provides them with BuildSlaves, and distributes file change
  37     notification messages to them.
  38     """
  39
  40     debug = 0
  41
  42     def __init__(self):
  43         service.MultiService.__init__(self)
  44         self.builders = {}
  45         self.builderNames = []
  46         # builders maps Builder names to instances of bb.p.builder.Builder,
  47         # which is the master-side object that defines and controls a build.
  48         # They are added by calling botmaster.addBuilder() from the startup
  49         # code.
  50
  51         # self.slaves contains a ready BuildSlave instance for each
  52         # potential buildslave, i.e. all the ones listed in the config file.
  53         # If the slave is connected, self.slaves[slavename].slave will
  54         # contain a RemoteReference to their Bot instance. If it is not
  55         # connected, that attribute will hold None.
  56         self.slaves = {} # maps slavename to BuildSlave
  57         self.statusClientService = None
  58         self.watchers = {}
  59
  60         # self.locks holds the real Lock instances
  61         self.locks = {}
  62
  63     # these four are convenience functions for testing
  64
  65     def waitUntilBuilderAttached(self, name):
  66         b = self.builders[name]
  67         #if b.slaves:
  68         #    return defer.succeed(None)
  69         d = defer.Deferred()
  70         b.watchers['attach'].append(d)
  71         return d
  72
  73     def waitUntilBuilderDetached(self, name):
  74         b = self.builders.get(name)
  75         if not b or not b.slaves:
  76             return defer.succeed(None)
  77         d = defer.Deferred()
  78         b.watchers['detach'].append(d)
  79         return d
  80
  81     def waitUntilBuilderFullyDetached(self, name):
  82         b = self.builders.get(name)
  83         # TODO: this looks too deeply inside the Builder object
  84         if not b or not b.slaves:
  85             return defer.succeed(None)
  86         d = defer.Deferred()
  87         b.watchers['detach_all'].append(d)
  88         return d
  89
  90     def waitUntilBuilderIdle(self, name):
  91         b = self.builders[name]
  92         # TODO: this looks way too deeply inside the Builder object
  93         for sb in b.slaves:
  94             if sb.state != IDLE:
  95                 d = defer.Deferred()
  96                 b.watchers['idle'].append(d)
  97                 return d
  98         return defer.succeed(None)
  99
 100     def loadConfig_Slaves(self, new_slaves):
 101         old_slaves = [c for c in list(self)
 102                       if interfaces.IBuildSlave.providedBy(c)]
 103
 104         # identify added/removed slaves. For each slave we construct a tuple
 105         # of (name, password, class), and we consider the slave to be already
 106         # present if the tuples match. (we include the class to make sure
 107         # that BuildSlave(name,pw) is different than
 108         # SubclassOfBuildSlave(name,pw) ). If the password or class has
 109         # changed, we will remove the old version of the slave and replace it
 110         # with a new one. If anything else has changed, we just update the
 111         # old BuildSlave instance in place. If the name has changed, of
 112         # course, it looks exactly the same as deleting one slave and adding
 113         # an unrelated one.
 114         old_t = {}
 115         for s in old_slaves:
 116             old_t[(s.slavename, s.password, s.__class__)] = s
 117         new_t = {}
 118         for s in new_slaves:
 119             new_t[(s.slavename, s.password, s.__class__)] = s
 120         removed = [old_t[t]
 121                    for t in old_t
 122                    if t not in new_t]
 123         added = [new_t[t]
 124                  for t in new_t
 125                  if t not in old_t]
 126         remaining_t = [t
 127                        for t in new_t
 128                        if t in old_t]
 129         # removeSlave will hang up on the old bot
 130         dl = []
 131         for s in removed:
 132             dl.append(self.removeSlave(s))
 133         d = defer.DeferredList(dl, fireOnOneErrback=True)
 134         def _add(res):
 135             for s in added:
 136                 self.addSlave(s)
 137             for t in remaining_t:
 138                 old_t[t].update(new_t[t])
 139         d.addCallback(_add)
 140         return d
 141
 142     def addSlave(self, s):
 143         s.setServiceParent(self)
 144         s.setBotmaster(self)
 145         self.slaves[s.slavename] = s
 146
 147     def removeSlave(self, s):
 148         # TODO: technically, disownServiceParent could return a Deferred
 149         s.disownServiceParent()
 150         d = self.slaves[s.slavename].disconnect()
 151         del self.slaves[s.slavename]
 152         return d
 153
 154     def slaveLost(self, bot):
 155         for name, b in self.builders.items():
 156             if bot.slavename in b.slavenames:
 157                 b.detached(bot)
 158
 159     def getBuildersForSlave(self, slavename):
 160         return [b
 161                 for b in self.builders.values()
 162                 if slavename in b.slavenames]
 163
 164     def getBuildernames(self):
 165         return self.builderNames
 166
 167     def getBuilders(self):
 168         allBuilders = [self.builders[name] for name in self.builderNames]
 169         return allBuilders
 170
 171     def setBuilders(self, builders):
 172         self.builders = {}
 173         self.builderNames = []
 174         for b in builders:
 175             for slavename in b.slavenames:
 176                 # this is actually validated earlier
 177                 assert slavename in self.slaves
 178             self.builders[b.name] = b
 179             self.builderNames.append(b.name)
 180             b.setBotmaster(self)
 181         d = self._updateAllSlaves()
 182         return d
 183
 184     def _updateAllSlaves(self):
 185         """Notify all buildslaves about changes in their Builders."""
 186         dl = [s.updateSlave() for s in self.slaves.values()]
 187         return defer.DeferredList(dl)
 188
 189     def maybeStartAllBuilds(self):
 190         for b in self.builders.values():
 191             b.maybeStartBuild()
 192
 193     def getPerspective(self, slavename):
 194         return self.slaves[slavename]
 195
 196     def shutdownSlaves(self):
 197         # TODO: make this into a bot method rather than a builder method
 198         for b in self.slaves.values():
 199             b.shutdownSlave()
 200
 201     def stopService(self):
 202         for b in self.builders.values():
 203             b.builder_status.addPointEvent(["master", "shutdown"])
 204             b.builder_status.saveYourself()
 205         return service.Service.stopService(self)
 206
 207     def getLockByID(self, lockid):
 208         """Convert a Lock identifier into an actual Lock instance.
 209         @param lockid: a locks.MasterLock or locks.SlaveLock instance
 210         @return: a locks.RealMasterLock or locks.RealSlaveLock instance
 211         """
 212         if not lockid in self.locks:
 213             self.locks[lockid] = lockid.lockClass(lockid)
 214         # if the master.cfg file has changed maxCount= on the lock, the next
 215         # time a build is started, they'll get a new RealLock instance. Note
 216         # that this requires that MasterLock and SlaveLock (marker) instances
 217         # be hashable and that they should compare properly.
 218         return self.locks[lockid]
 219
 220 ########################################
 221
 222
 223
 224 class DebugPerspective(NewCredPerspective):
 225     def attached(self, mind):
 226         return self
 227     def detached(self, mind):
 228         pass
 229
 230     def perspective_requestBuild(self, buildername, reason, branch, revision):
 231         c = interfaces.IControl(self.master)
 232         bc = c.getBuilder(buildername)
 233         ss = SourceStamp(branch, revision)
 234         br = BuildRequest(reason, ss, buildername)
 235         bc.requestBuild(br)
 236
 237     def perspective_pingBuilder(self, buildername):
 238         c = interfaces.IControl(self.master)
 239         bc = c.getBuilder(buildername)
 240         bc.ping()
 241
 242     def perspective_fakeChange(self, file, revision=None, who="fakeUser",
 243                                branch=None):
 244         change = Change(who, [file], "some fake comments\n",
 245                         branch=branch, revision=revision)
 246         c = interfaces.IControl(self.master)
 247         c.addChange(change)
 248
 249     def perspective_setCurrentState(self, buildername, state):
 250         builder = self.botmaster.builders.get(buildername)
 251         if not builder: return
 252         if state == "offline":
 253             builder.statusbag.currentlyOffline()
 254         if state == "idle":
 255             builder.statusbag.currentlyIdle()
 256         if state == "waiting":
 257             builder.statusbag.currentlyWaiting(now()+10)
 258         if state == "building":
 259             builder.statusbag.currentlyBuilding(None)
 260     def perspective_reload(self):
 261         print "doing reload of the config file"
 262         self.master.loadTheConfigFile()
 263     def perspective_pokeIRC(self):
 264         print "saying something on IRC"
 265         from buildbot.status import words
 266         for s in self.master:
 267             if isinstance(s, words.IRC):
 268                 bot = s.f
 269                 for channel in bot.channels:
 270                     print " channel", channel
 271                     bot.p.msg(channel, "Ow, quit it")
 272
 273     def perspective_print(self, msg):
 274         print "debug", msg
 275
 276 class Dispatcher(styles.Versioned):
 277     implements(portal.IRealm)
 278     persistenceVersion = 2
 279
 280     def __init__(self):
 281         self.names = {}
 282
 283     def upgradeToVersion1(self):
 284         self.master = self.botmaster.parent
 285     def upgradeToVersion2(self):
 286         self.names = {}
 287
 288     def register(self, name, afactory):
 289         self.names[name] = afactory
 290     def unregister(self, name):
 291         del self.names[name]
 292
 293     def requestAvatar(self, avatarID, mind, interface):
 294         assert interface == pb.IPerspective
 295         afactory = self.names.get(avatarID)
 296         if afactory:
 297             p = afactory.getPerspective()
 298         elif avatarID == "debug":
 299             p = DebugPerspective()
 300             p.master = self.master
 301             p.botmaster = self.botmaster
 302         elif avatarID == "statusClient":
 303             p = self.statusClientService.getPerspective()
 304         else:
 305             # it must be one of the buildslaves: no other names will make it
 306             # past the checker
 307             p = self.botmaster.getPerspective(avatarID)
 308
 309         if not p:
 310             raise ValueError("no perspective for '%s'" % avatarID)
 311
 312         d = defer.maybeDeferred(p.attached, mind)
 313         d.addCallback(self._avatarAttached, mind)
 314         return d
 315
 316     def _avatarAttached(self, p, mind):
 317         return (pb.IPerspective, p, lambda p=p,mind=mind: p.detached(mind))
 318
 319 ########################################
 320
 321 # service hierarchy:
 322 #  BuildMaster
 323 #   BotMaster
 324 #   ChangeMaster
 325 #    all IChangeSource objects
 326 #   StatusClientService
 327 #   TCPClient(self.ircFactory)
 328 #   TCPServer(self.slaveFactory) -> dispatcher.requestAvatar
 329 #   TCPServer(self.site)
 330 #   UNIXServer(ResourcePublisher(self.site))
 331
 332
 333 class BuildMaster(service.MultiService, styles.Versioned):
 334     debug = 0
 335     persistenceVersion = 3
 336     manhole = None
 337     debugPassword = None
 338     projectName = "(unspecified)"
 339     projectURL = None
 340     buildbotURL = None
 341     change_svc = None
 342
 343     def __init__(self, basedir, configFileName="master.cfg"):
 344         service.MultiService.__init__(self)
 345         self.setName("buildmaster")
 346         self.basedir = basedir
 347         self.configFileName = configFileName
 348
 349         # the dispatcher is the realm in which all inbound connections are
 350         # looked up: slave builders, change notifications, status clients, and
 351         # the debug port
 352         dispatcher = Dispatcher()
 353         dispatcher.master = self
 354         self.dispatcher = dispatcher
 355         self.checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
 356         # the checker starts with no user/passwd pairs: they are added later
 357         p = portal.Portal(dispatcher)
 358         p.registerChecker(self.checker)
 359         self.slaveFactory = pb.PBServerFactory(p)
 360         self.slaveFactory.unsafeTracebacks = True # let them see exceptions
 361
 362         self.slavePortnum = None
 363         self.slavePort = None
 364
 365         self.botmaster = BotMaster()
 366         self.botmaster.setName("botmaster")
 367         self.botmaster.setServiceParent(self)
 368         dispatcher.botmaster = self.botmaster
 369
 370         self.status = Status(self.botmaster, self.basedir)
 371
 372         self.statusTargets = []
 373
 374         # this ChangeMaster is a dummy, only used by tests. In the real
 375         # buildmaster, where the BuildMaster instance is activated
 376         # (startService is called) by twistd, this attribute is overwritten.
 377         self.useChanges(ChangeMaster())
 378
 379         self.readConfig = False
 380
 381     def upgradeToVersion1(self):
 382         self.dispatcher = self.slaveFactory.root.portal.realm
 383
 384     def upgradeToVersion2(self): # post-0.4.3
 385         self.webServer = self.webTCPPort
 386         del self.webTCPPort
 387         self.webDistribServer = self.webUNIXPort
 388         del self.webUNIXPort
 389         self.configFileName = "master.cfg"
 390
 391     def upgradeToVersion3(self):
 392         # post 0.6.3, solely to deal with the 0.6.3 breakage. Starting with
 393         # 0.6.5 I intend to do away with .tap files altogether
 394         self.services = []
 395         self.namedServices = {}
 396         del self.change_svc
 397
 398     def startService(self):
 399         service.MultiService.startService(self)
 400         self.loadChanges() # must be done before loading the config file
 401         if not self.readConfig:
 402             # TODO: consider catching exceptions during this call to
 403             # loadTheConfigFile and bailing (reactor.stop) if it fails,
 404             # since without a config file we can't do anything except reload
 405             # the config file, and it would be nice for the user to discover
 406             # this quickly.
 407             self.loadTheConfigFile()
 408         if signal and hasattr(signal, "SIGHUP"):
 409             signal.signal(signal.SIGHUP, self._handleSIGHUP)
 410         for b in self.botmaster.builders.values():
 411             b.builder_status.addPointEvent(["master", "started"])
 412             b.builder_status.saveYourself()
 413
 414     def useChanges(self, changes):
 415         if self.change_svc:
 416             # TODO: can return a Deferred
 417             self.change_svc.disownServiceParent()
 418         self.change_svc = changes
 419         self.change_svc.basedir = self.basedir
 420         self.change_svc.setName("changemaster")
 421         self.dispatcher.changemaster = self.change_svc
 422         self.change_svc.setServiceParent(self)
 423
 424     def loadChanges(self):
 425         filename = os.path.join(self.basedir, "changes.pck")
 426         try:
 427             changes = load(open(filename, "rb"))
 428             styles.doUpgrade()
 429         except IOError:
 430             log.msg("changes.pck missing, using new one")
 431             changes = ChangeMaster()
 432         except EOFError:
 433             log.msg("corrupted changes.pck, using new one")
 434             changes = ChangeMaster()
 435         self.useChanges(changes)
 436
 437     def _handleSIGHUP(self, *args):
 438         reactor.callLater(0, self.loadTheConfigFile)
 439
 440     def getStatus(self):
 441         """
 442         @rtype: L{buildbot.status.builder.Status}
 443         """
 444         return self.status
 445
 446     def loadTheConfigFile(self, configFile=None):
 447         if not configFile:
 448             configFile = os.path.join(self.basedir, self.configFileName)
 449
 450         log.msg("loading configuration from %s" % configFile)
 451         configFile = os.path.expanduser(configFile)
 452
 453         try:
 454             f = open(configFile, "r")
 455         except IOError, e:
 456             log.msg("unable to open config file '%s'" % configFile)
 457             log.msg("leaving old configuration in place")
 458             log.err(e)
 459             return
 460
 461         try:
 462             self.loadConfig(f)
 463         except:
 464             log.msg("error during loadConfig")
 465             log.err()
 466             log.msg("The new config file is unusable, so I'll ignore it.")
 467             log.msg("I will keep using the previous config file instead.")
 468         f.close()
 469
 470     def loadConfig(self, f):
 471         """Internal function to load a specific configuration file. Any
 472         errors in the file will be signalled by raising an exception.
 473
 474         @return: a Deferred that will fire (with None) when the configuration
 475         changes have been completed. This may involve a round-trip to each
 476         buildslave that was involved."""
 477
 478         localDict = {'basedir': os.path.expanduser(self.basedir)}
 479         try:
 480             exec f in localDict
 481         except:
 482             log.msg("error while parsing config file")
 483             raise
 484
 485         try:
 486             config = localDict['BuildmasterConfig']
 487         except KeyError:
 488             log.err("missing config dictionary")
 489             log.err("config file must define BuildmasterConfig")
 490             raise
 491
 492         known_keys = ("bots", "slaves",
 493                       "sources", "change_source",
 494                       "schedulers", "builders",
 495                       "slavePortnum", "debugPassword", "manhole",
 496                       "status", "projectName", "projectURL", "buildbotURL",
 497                       )
 498         for k in config.keys():
 499             if k not in known_keys:
 500                 log.msg("unknown key '%s' defined in config dictionary" % k)
 501
 502         try:
 503             # required
 504             schedulers = config['schedulers']
 505             builders = config['builders']
 506             for k in builders:
 507                 if k['name'].startswith("_"):
 508                     errmsg = ("builder names must not start with an "
 509                               "underscore: " + k['name'])
 510                     log.err(errmsg)
 511                     raise ValueError(errmsg)
 512
 513             slavePortnum = config['slavePortnum']
 514             #slaves = config['slaves']
 515             #change_source = config['change_source']
 516
 517             # optional
 518             debugPassword = config.get('debugPassword')
 519             manhole = config.get('manhole')
 520             status = config.get('status', [])
 521             projectName = config.get('projectName')
 522             projectURL = config.get('projectURL')
 523             buildbotURL = config.get('buildbotURL')
 524
 525         except KeyError, e:
 526             log.msg("config dictionary is missing a required parameter")
 527             log.msg("leaving old configuration in place")
 528             raise
 529
 530         #if "bots" in config:
 531         #    raise KeyError("c['bots'] is no longer accepted")
 532
 533         slaves = config.get('slaves', [])
 534         if "bots" in config:
 535             m = ("c['bots'] is deprecated as of 0.7.6 and will be "
 536                  "removed by 0.8.0 . Please use c['slaves'] instead.")
 537             log.msg(m)
 538             warnings.warn(m, DeprecationWarning)
 539             for name, passwd in config['bots']:
 540                 slaves.append(BuildSlave(name, passwd))
 541
 542         if "bots" not in config and "slaves" not in config:
 543             log.msg("config dictionary must have either 'bots' or 'slaves'")
 544             log.msg("leaving old configuration in place")
 545             raise KeyError("must have either 'bots' or 'slaves'")
 546
 547         #if "sources" in config:
 548         #    raise KeyError("c['sources'] is no longer accepted")
 549
 550         change_source = config.get('change_source', [])
 551         if isinstance(change_source, (list, tuple)):
 552             change_sources = change_source
 553         else:
 554             change_sources = [change_source]
 555         if "sources" in config:
 556             m = ("c['sources'] is deprecated as of 0.7.6 and will be "
 557                  "removed by 0.8.0 . Please use c['change_source'] instead.")
 558             log.msg(m)
 559             warnings.warn(m, DeprecationWarning)
 560             for s in config['sources']:
 561                 change_sources.append(s)
 562
 563         # do some validation first
 564         for s in slaves:
 565             assert isinstance(s, BuildSlave)
 566             if s.slavename in ("debug", "change", "status"):
 567                 raise KeyError, "reserved name '%s' used for a bot" % s.slavename
 568         if config.has_key('interlocks'):
 569             raise KeyError("c['interlocks'] is no longer accepted")
 570
 571         assert isinstance(change_sources, (list, tuple))
 572         for s in change_sources:
 573             assert interfaces.IChangeSource(s, None)
 574         # this assertion catches c['schedulers'] = Scheduler(), since
 575         # Schedulers are service.MultiServices and thus iterable.
 576         errmsg = "c['schedulers'] must be a list of Scheduler instances"
 577         assert isinstance(schedulers, (list, tuple)), errmsg
 578         for s in schedulers:
 579             assert interfaces.IScheduler(s, None), errmsg
 580         assert isinstance(status, (list, tuple))
 581         for s in status:
 582             assert interfaces.IStatusReceiver(s, None)
 583
 584         slavenames = [s.slavename for s in slaves]
 585         buildernames = []
 586         dirnames = []
 587         for b in builders:
 588             if type(b) is tuple:
 589                 raise ValueError("builder %s must be defined with a dict, "
 590                                  "not a tuple" % b[0])
 591             if b.has_key('slavename') and b['slavename'] not in slavenames:
 592                 raise ValueError("builder %s uses undefined slave %s" \
 593                                  % (b['name'], b['slavename']))
 594             for n in b.get('slavenames', []):
 595                 if n not in slavenames:
 596                     raise ValueError("builder %s uses undefined slave %s" \
 597                                      % (b['name'], n))
 598             if b['name'] in buildernames:
 599                 raise ValueError("duplicate builder name %s"
 600                                  % b['name'])
 601             buildernames.append(b['name'])
 602             if b['builddir'] in dirnames:
 603                 raise ValueError("builder %s reuses builddir %s"
 604                                  % (b['name'], b['builddir']))
 605             dirnames.append(b['builddir'])
 606
 607         unscheduled_buildernames = buildernames[:]
 608         schedulernames = []
 609         for s in schedulers:
 610             for b in s.listBuilderNames():
 611                 assert b in buildernames, \
 612                        "%s uses unknown builder %s" % (s, b)
 613                 if b in unscheduled_buildernames:
 614                     unscheduled_buildernames.remove(b)
 615
 616             if s.name in schedulernames:
 617                 # TODO: schedulers share a namespace with other Service
 618                 # children of the BuildMaster node, like status plugins, the
 619                 # Manhole, the ChangeMaster, and the BotMaster (although most
 620                 # of these don't have names)
 621                 msg = ("Schedulers must have unique names, but "
 622                        "'%s' was a duplicate" % (s.name,))
 623                 raise ValueError(msg)
 624             schedulernames.append(s.name)
 625
 626         if unscheduled_buildernames:
 627             log.msg("Warning: some Builders have no Schedulers to drive them:"
 628                     " %s" % (unscheduled_buildernames,))
 629
 630         # assert that all locks used by the Builds and their Steps are
 631         # uniquely named.
 632         locks = {}
 633         for b in builders:
 634             for l in b.get('locks', []):
 635                 if locks.has_key(l.name):
 636                     if locks[l.name] is not l:
 637                         raise ValueError("Two different locks (%s and %s) "
 638                                          "share the name %s"
 639                                          % (l, locks[l.name], l.name))
 640                 else:
 641                     locks[l.name] = l
 642             # TODO: this will break with any BuildFactory that doesn't use a
 643             # .steps list, but I think the verification step is more
 644             # important.
 645             for s in b['factory'].steps:
 646                 for l in s[1].get('locks', []):
 647                     if locks.has_key(l.name):
 648                         if locks[l.name] is not l:
 649                             raise ValueError("Two different locks (%s and %s)"
 650                                              " share the name %s"
 651                                              % (l, locks[l.name], l.name))
 652                     else:
 653                         locks[l.name] = l
 654
 655         # slavePortnum supposed to be a strports specification
 656         if type(slavePortnum) is int:
 657             slavePortnum = "tcp:%d" % slavePortnum
 658
 659         # now we're committed to implementing the new configuration, so do
 660         # it atomically
 661         # TODO: actually, this is spread across a couple of Deferreds, so it
 662         # really isn't atomic.
 663
 664         d = defer.succeed(None)
 665
 666         self.projectName = projectName
 667         self.projectURL = projectURL
 668         self.buildbotURL = buildbotURL
 669
 670         # self.slaves: Disconnect any that were attached and removed from the
 671         # list. Update self.checker with the new list of passwords, including
 672         # debug/change/status.
 673         d.addCallback(lambda res: self.loadConfig_Slaves(slaves))
 674
 675         # self.debugPassword
 676         if debugPassword:
 677             self.checker.addUser("debug", debugPassword)
 678             self.debugPassword = debugPassword
 679
 680         # self.manhole
 681         if manhole != self.manhole:
 682             # changing
 683             if self.manhole:
 684                 # disownServiceParent may return a Deferred
 685                 d.addCallback(lambda res: self.manhole.disownServiceParent())
 686                 def _remove(res):
 687                     self.manhole = None
 688                     return res
 689                 d.addCallback(_remove)
 690             if manhole:
 691                 def _add(res):
 692                     self.manhole = manhole
 693                     manhole.setServiceParent(self)
 694                 d.addCallback(_add)
 695
 696         # add/remove self.botmaster.builders to match builders. The
 697         # botmaster will handle startup/shutdown issues.
 698         d.addCallback(lambda res: self.loadConfig_Builders(builders))
 699
 700         d.addCallback(lambda res: self.loadConfig_status(status))
 701
 702         # Schedulers are added after Builders in case they start right away
 703         d.addCallback(lambda res: self.loadConfig_Schedulers(schedulers))
 704         # and Sources go after Schedulers for the same reason
 705         d.addCallback(lambda res: self.loadConfig_Sources(change_sources))
 706
 707         # self.slavePort
 708         if self.slavePortnum != slavePortnum:
 709             if self.slavePort:
 710                 def closeSlavePort(res):
 711                     d1 = self.slavePort.disownServiceParent()
 712                     self.slavePort = None
 713                     return d1
 714                 d.addCallback(closeSlavePort)
 715             if slavePortnum is not None:
 716                 def openSlavePort(res):
 717                     self.slavePort = strports.service(slavePortnum,
 718                                                       self.slaveFactory)
 719                     self.slavePort.setServiceParent(self)
 720                 d.addCallback(openSlavePort)
 721                 log.msg("BuildMaster listening on port %s" % slavePortnum)
 722             self.slavePortnum = slavePortnum
 723
 724         log.msg("configuration update started")
 725         def _done(res):
 726             self.readConfig = True
 727             log.msg("configuration update complete")
 728         d.addCallback(_done)
 729         d.addCallback(lambda res: self.botmaster.maybeStartAllBuilds())
 730         return d
 731
 732     def loadConfig_Slaves(self, new_slaves):
 733         # set up the Checker with the names and passwords of all valid bots
 734         self.checker.users = {} # violates abstraction, oh well
 735         for s in new_slaves:
 736             self.checker.addUser(s.slavename, s.password)
 737         self.checker.addUser("change", "changepw")
 738         # let the BotMaster take care of the rest
 739         return self.botmaster.loadConfig_Slaves(new_slaves)
 740
 741     def loadConfig_Sources(self, sources):
 742         if not sources:
 743             log.msg("warning: no ChangeSources specified in c['change_source']")
 744         # shut down any that were removed, start any that were added
 745         deleted_sources = [s for s in self.change_svc if s not in sources]
 746         added_sources = [s for s in sources if s not in self.change_svc]
 747         dl = [self.change_svc.removeSource(s) for s in deleted_sources]
 748         def addNewOnes(res):
 749             [self.change_svc.addSource(s) for s in added_sources]
 750         d = defer.DeferredList(dl, fireOnOneErrback=1, consumeErrors=0)
 751         d.addCallback(addNewOnes)
 752         return d
 753
 754     def allSchedulers(self):
 755         return [child for child in self
 756                 if interfaces.IScheduler.providedBy(child)]
 757
 758
 759     def loadConfig_Schedulers(self, newschedulers):
 760         oldschedulers = self.allSchedulers()
 761         removed = [s for s in oldschedulers if s not in newschedulers]
 762         added = [s for s in newschedulers if s not in oldschedulers]
 763         dl = [defer.maybeDeferred(s.disownServiceParent) for s in removed]
 764         def addNewOnes(res):
 765             for s in added:
 766                 s.setServiceParent(self)
 767         d = defer.DeferredList(dl, fireOnOneErrback=1)
 768         d.addCallback(addNewOnes)
 769         return d
 770
 771     def loadConfig_Builders(self, newBuilderData):
 772         somethingChanged = False
 773         newList = {}
 774         newBuilderNames = []
 775         allBuilders = self.botmaster.builders.copy()
 776         for data in newBuilderData:
 777             name = data['name']
 778             newList[name] = data
 779             newBuilderNames.append(name)
 780
 781         # identify all that were removed
 782         for oldname in self.botmaster.getBuildernames():
 783             if oldname not in newList:
 784                 log.msg("removing old builder %s" % oldname)
 785                 del allBuilders[oldname]
 786                 somethingChanged = True
 787                 # announce the change
 788                 self.status.builderRemoved(oldname)
 789
 790         # everything in newList is either unchanged, changed, or new
 791         for name, data in newList.items():
 792             old = self.botmaster.builders.get(name)
 793             basedir = data['builddir'] # used on both master and slave
 794             #name, slave, builddir, factory = data
 795             if not old: # new
 796                 # category added after 0.6.2
 797                 category = data.get('category', None)
 798                 log.msg("adding new builder %s for category %s" %
 799                         (name, category))
 800                 statusbag = self.status.builderAdded(name, basedir, category)
 801                 builder = Builder(data, statusbag)
 802                 allBuilders[name] = builder
 803                 somethingChanged = True
 804             elif old.compareToSetup(data):
 805                 # changed: try to minimize the disruption and only modify the
 806                 # pieces that really changed
 807                 diffs = old.compareToSetup(data)
 808                 log.msg("updating builder %s: %s" % (name, "\n".join(diffs)))
 809
 810                 statusbag = old.builder_status
 811                 statusbag.saveYourself() # seems like a good idea
 812                 # TODO: if the basedir was changed, we probably need to make
 813                 # a new statusbag
 814                 new_builder = Builder(data, statusbag)
 815                 new_builder.consumeTheSoulOfYourPredecessor(old)
 816                 # that migrates any retained slavebuilders too
 817
 818                 # point out that the builder was updated. On the Waterfall,
 819                 # this will appear just after any currently-running builds.
 820                 statusbag.addPointEvent(["config", "updated"])
 821
 822                 allBuilders[name] = new_builder
 823                 somethingChanged = True
 824             else:
 825                 # unchanged: leave it alone
 826                 log.msg("builder %s is unchanged" % name)
 827                 pass
 828
 829         if somethingChanged:
 830             sortedAllBuilders = [allBuilders[name] for name in newBuilderNames]
 831             d = self.botmaster.setBuilders(sortedAllBuilders)
 832             return d
 833         return None
 834
 835     def loadConfig_status(self, status):
 836         dl = []
 837
 838         # remove old ones
 839         for s in self.statusTargets[:]:
 840             if not s in status:
 841                 log.msg("removing IStatusReceiver", s)
 842                 d = defer.maybeDeferred(s.disownServiceParent)
 843                 dl.append(d)
 844                 self.statusTargets.remove(s)
 845         # after those are finished going away, add new ones
 846         def addNewOnes(res):
 847             for s in status:
 848                 if not s in self.statusTargets:
 849                     log.msg("adding IStatusReceiver", s)
 850                     s.setServiceParent(self)
 851                     self.statusTargets.append(s)
 852         d = defer.DeferredList(dl, fireOnOneErrback=1)
 853         d.addCallback(addNewOnes)
 854         return d
 855
 856
 857     def addChange(self, change):
 858         for s in self.allSchedulers():
 859             s.addChange(change)
 860
 861     def submitBuildSet(self, bs):
 862         # determine the set of Builders to use
 863         builders = []
 864         for name in bs.builderNames:
 865             b = self.botmaster.builders.get(name)
 866             if b:
 867                 if b not in builders:
 868                     builders.append(b)
 869                 continue
 870             # TODO: add aliases like 'all'
 871             raise KeyError("no such builder named '%s'" % name)
 872
 873         # now tell the BuildSet to create BuildRequests for all those
 874         # Builders and submit them
 875         bs.start(builders)
 876         self.status.buildsetSubmitted(bs.status)
 877
 878
 879 class Control:
 880     implements(interfaces.IControl)
 881
 882     def __init__(self, master):
 883         self.master = master
 884
 885     def addChange(self, change):
 886         self.master.change_svc.addChange(change)
 887
 888     def submitBuildSet(self, bs):
 889         self.master.submitBuildSet(bs)
 890
 891     def getBuilder(self, name):
 892         b = self.master.botmaster.builders[name]
 893         return interfaces.IBuilderControl(b)
 894
 895 components.registerAdapter(Control, BuildMaster, interfaces.IControl)
 896
 897 # so anybody who can get a handle on the BuildMaster can cause a build with:
 898 #  IControl(master).getBuilder("full-2.3").requestBuild(buildrequest)
 899