buildbot/master.py

   1 # -*- test-case-name: buildbot.test.test_run -*-
   2
   3 import os
   4 signal = None
   5 try:
   6     import signal
   7 except ImportError:
   8     pass
   9 from cPickle import load
  10 import warnings
  11
  12 from zope.interface import implements
  13 from twisted.python import log, components
  14 from twisted.internet import defer, reactor
  15 from twisted.spread import pb
  16 from twisted.cred import portal, checkers
  17 from twisted.application import service, strports
  18 from twisted.persisted import styles
  19
  20 # sibling imports
  21 from buildbot.util import now
  22 from buildbot.pbutil import NewCredPerspective
  23 from buildbot.process.builder import Builder, IDLE
  24 from buildbot.process.base import BuildRequest
  25 from buildbot.status.builder import Status
  26 from buildbot.changes.changes import Change, ChangeMaster
  27 from buildbot.sourcestamp import SourceStamp
  28 from buildbot.buildslave import BuildSlave
  29 from buildbot import interfaces
  30
  31 ########################################
  32
  33 class BotMaster(service.Service):
  34
  35     """This is the master-side service which manages remote buildbot slaves.
  36     It provides them with BuildSlaves, and distributes file change
  37     notification messages to them.
  38     """
  39
  40     debug = 0
  41
  42     def __init__(self):
  43         self.builders = {}
  44         self.builderNames = []
  45         # builders maps Builder names to instances of bb.p.builder.Builder,
  46         # which is the master-side object that defines and controls a build.
  47         # They are added by calling botmaster.addBuilder() from the startup
  48         # code.
  49
  50         # self.slaves contains a ready BuildSlave instance for each
  51         # potential buildslave, i.e. all the ones listed in the config file.
  52         # If the slave is connected, self.slaves[slavename].slave will
  53         # contain a RemoteReference to their Bot instance. If it is not
  54         # connected, that attribute will hold None.
  55         self.slaves = {} # maps slavename to BuildSlave
  56         self.statusClientService = None
  57         self.watchers = {}
  58
  59         # self.locks holds the real Lock instances
  60         self.locks = {}
  61
  62     # these four are convenience functions for testing
  63
  64     def waitUntilBuilderAttached(self, name):
  65         b = self.builders[name]
  66         #if b.slaves:
  67         #    return defer.succeed(None)
  68         d = defer.Deferred()
  69         b.watchers['attach'].append(d)
  70         return d
  71
  72     def waitUntilBuilderDetached(self, name):
  73         b = self.builders.get(name)
  74         if not b or not b.slaves:
  75             return defer.succeed(None)
  76         d = defer.Deferred()
  77         b.watchers['detach'].append(d)
  78         return d
  79
  80     def waitUntilBuilderFullyDetached(self, name):
  81         b = self.builders.get(name)
  82         # TODO: this looks too deeply inside the Builder object
  83         if not b or not b.slaves:
  84             return defer.succeed(None)
  85         d = defer.Deferred()
  86         b.watchers['detach_all'].append(d)
  87         return d
  88
  89     def waitUntilBuilderIdle(self, name):
  90         b = self.builders[name]
  91         # TODO: this looks way too deeply inside the Builder object
  92         for sb in b.slaves:
  93             if sb.state != IDLE:
  94                 d = defer.Deferred()
  95                 b.watchers['idle'].append(d)
  96                 return d
  97         return defer.succeed(None)
  98
  99
 100     def addSlave(self, slave):
 101         slave.setBotmaster(self)
 102         self.slaves[slave.slavename] = slave
 103
 104     def removeSlave(self, slavename):
 105         d = self.slaves[slavename].disconnect()
 106         del self.slaves[slavename]
 107         return d
 108
 109     def slaveLost(self, bot):
 110         for name, b in self.builders.items():
 111             if bot.slavename in b.slavenames:
 112                 b.detached(bot)
 113
 114     def getBuildersForSlave(self, slavename):
 115         return [b
 116                 for b in self.builders.values()
 117                 if slavename in b.slavenames]
 118
 119     def getBuildernames(self):
 120         return self.builderNames
 121
 122     def getBuilders(self):
 123         allBuilders = [self.builders[name] for name in self.builderNames]
 124         return allBuilders
 125
 126     def setBuilders(self, builders):
 127         self.builders = {}
 128         self.builderNames = []
 129         for b in builders:
 130             for slavename in b.slavenames:
 131                 # this is actually validated earlier
 132                 assert slavename in self.slaves
 133             self.builders[b.name] = b
 134             self.builderNames.append(b.name)
 135             b.setBotmaster(self)
 136         d = self._updateAllSlaves()
 137         return d
 138
 139     def _updateAllSlaves(self):
 140         """Notify all buildslaves about changes in their Builders."""
 141         dl = [s.updateSlave() for s in self.slaves.values()]
 142         return defer.DeferredList(dl)
 143
 144     def maybeStartAllBuilds(self):
 145         for b in self.builders.values():
 146             b.maybeStartBuild()
 147
 148     def getPerspective(self, slavename):
 149         return self.slaves[slavename]
 150
 151     def shutdownSlaves(self):
 152         # TODO: make this into a bot method rather than a builder method
 153         for b in self.slaves.values():
 154             b.shutdownSlave()
 155
 156     def stopService(self):
 157         for b in self.builders.values():
 158             b.builder_status.addPointEvent(["master", "shutdown"])
 159             b.builder_status.saveYourself()
 160         return service.Service.stopService(self)
 161
 162     def getLockByID(self, lockid):
 163         """Convert a Lock identifier into an actual Lock instance.
 164         @param lockid: a locks.MasterLock or locks.SlaveLock instance
 165         @return: a locks.RealMasterLock or locks.RealSlaveLock instance
 166         """
 167         if not lockid in self.locks:
 168             self.locks[lockid] = lockid.lockClass(lockid)
 169         # if the master.cfg file has changed maxCount= on the lock, the next
 170         # time a build is started, they'll get a new RealLock instance. Note
 171         # that this requires that MasterLock and SlaveLock (marker) instances
 172         # be hashable and that they should compare properly.
 173         return self.locks[lockid]
 174
 175 ########################################
 176
 177
 178
 179 class DebugPerspective(NewCredPerspective):
 180     def attached(self, mind):
 181         return self
 182     def detached(self, mind):
 183         pass
 184
 185     def perspective_requestBuild(self, buildername, reason, branch, revision):
 186         c = interfaces.IControl(self.master)
 187         bc = c.getBuilder(buildername)
 188         ss = SourceStamp(branch, revision)
 189         br = BuildRequest(reason, ss, buildername)
 190         bc.requestBuild(br)
 191
 192     def perspective_pingBuilder(self, buildername):
 193         c = interfaces.IControl(self.master)
 194         bc = c.getBuilder(buildername)
 195         bc.ping()
 196
 197     def perspective_fakeChange(self, file, revision=None, who="fakeUser",
 198                                branch=None):
 199         change = Change(who, [file], "some fake comments\n",
 200                         branch=branch, revision=revision)
 201         c = interfaces.IControl(self.master)
 202         c.addChange(change)
 203
 204     def perspective_setCurrentState(self, buildername, state):
 205         builder = self.botmaster.builders.get(buildername)
 206         if not builder: return
 207         if state == "offline":
 208             builder.statusbag.currentlyOffline()
 209         if state == "idle":
 210             builder.statusbag.currentlyIdle()
 211         if state == "waiting":
 212             builder.statusbag.currentlyWaiting(now()+10)
 213         if state == "building":
 214             builder.statusbag.currentlyBuilding(None)
 215     def perspective_reload(self):
 216         print "doing reload of the config file"
 217         self.master.loadTheConfigFile()
 218     def perspective_pokeIRC(self):
 219         print "saying something on IRC"
 220         from buildbot.status import words
 221         for s in self.master:
 222             if isinstance(s, words.IRC):
 223                 bot = s.f
 224                 for channel in bot.channels:
 225                     print " channel", channel
 226                     bot.p.msg(channel, "Ow, quit it")
 227
 228     def perspective_print(self, msg):
 229         print "debug", msg
 230
 231 class Dispatcher(styles.Versioned):
 232     implements(portal.IRealm)
 233     persistenceVersion = 2
 234
 235     def __init__(self):
 236         self.names = {}
 237
 238     def upgradeToVersion1(self):
 239         self.master = self.botmaster.parent
 240     def upgradeToVersion2(self):
 241         self.names = {}
 242
 243     def register(self, name, afactory):
 244         self.names[name] = afactory
 245     def unregister(self, name):
 246         del self.names[name]
 247
 248     def requestAvatar(self, avatarID, mind, interface):
 249         assert interface == pb.IPerspective
 250         afactory = self.names.get(avatarID)
 251         if afactory:
 252             p = afactory.getPerspective()
 253         elif avatarID == "debug":
 254             p = DebugPerspective()
 255             p.master = self.master
 256             p.botmaster = self.botmaster
 257         elif avatarID == "statusClient":
 258             p = self.statusClientService.getPerspective()
 259         else:
 260             # it must be one of the buildslaves: no other names will make it
 261             # past the checker
 262             p = self.botmaster.getPerspective(avatarID)
 263
 264         if not p:
 265             raise ValueError("no perspective for '%s'" % avatarID)
 266
 267         d = defer.maybeDeferred(p.attached, mind)
 268         d.addCallback(self._avatarAttached, mind)
 269         return d
 270
 271     def _avatarAttached(self, p, mind):
 272         return (pb.IPerspective, p, lambda p=p,mind=mind: p.detached(mind))
 273
 274 ########################################
 275
 276 # service hierarchy:
 277 #  BuildMaster
 278 #   BotMaster
 279 #   ChangeMaster
 280 #    all IChangeSource objects
 281 #   StatusClientService
 282 #   TCPClient(self.ircFactory)
 283 #   TCPServer(self.slaveFactory) -> dispatcher.requestAvatar
 284 #   TCPServer(self.site)
 285 #   UNIXServer(ResourcePublisher(self.site))
 286
 287
 288 class BuildMaster(service.MultiService, styles.Versioned):
 289     debug = 0
 290     persistenceVersion = 3
 291     manhole = None
 292     debugPassword = None
 293     projectName = "(unspecified)"
 294     projectURL = None
 295     buildbotURL = None
 296     change_svc = None
 297
 298     def __init__(self, basedir, configFileName="master.cfg"):
 299         service.MultiService.__init__(self)
 300         self.setName("buildmaster")
 301         self.basedir = basedir
 302         self.configFileName = configFileName
 303
 304         # the dispatcher is the realm in which all inbound connections are
 305         # looked up: slave builders, change notifications, status clients, and
 306         # the debug port
 307         dispatcher = Dispatcher()
 308         dispatcher.master = self
 309         self.dispatcher = dispatcher
 310         self.checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
 311         # the checker starts with no user/passwd pairs: they are added later
 312         p = portal.Portal(dispatcher)
 313         p.registerChecker(self.checker)
 314         self.slaveFactory = pb.PBServerFactory(p)
 315         self.slaveFactory.unsafeTracebacks = True # let them see exceptions
 316
 317         self.slavePortnum = None
 318         self.slavePort = None
 319
 320         self.botmaster = BotMaster()
 321         self.botmaster.setName("botmaster")
 322         self.botmaster.setServiceParent(self)
 323         dispatcher.botmaster = self.botmaster
 324
 325         self.status = Status(self.botmaster, self.basedir)
 326
 327         self.statusTargets = []
 328
 329         self.slaves = []
 330         # this ChangeMaster is a dummy, only used by tests. In the real
 331         # buildmaster, where the BuildMaster instance is activated
 332         # (startService is called) by twistd, this attribute is overwritten.
 333         self.useChanges(ChangeMaster())
 334
 335         self.readConfig = False
 336
 337     def upgradeToVersion1(self):
 338         self.dispatcher = self.slaveFactory.root.portal.realm
 339
 340     def upgradeToVersion2(self): # post-0.4.3
 341         self.webServer = self.webTCPPort
 342         del self.webTCPPort
 343         self.webDistribServer = self.webUNIXPort
 344         del self.webUNIXPort
 345         self.configFileName = "master.cfg"
 346
 347     def upgradeToVersion3(self):
 348         # post 0.6.3, solely to deal with the 0.6.3 breakage. Starting with
 349         # 0.6.5 I intend to do away with .tap files altogether
 350         self.services = []
 351         self.namedServices = {}
 352         del self.change_svc
 353
 354     def startService(self):
 355         service.MultiService.startService(self)
 356         self.loadChanges() # must be done before loading the config file
 357         if not self.readConfig:
 358             # TODO: consider catching exceptions during this call to
 359             # loadTheConfigFile and bailing (reactor.stop) if it fails,
 360             # since without a config file we can't do anything except reload
 361             # the config file, and it would be nice for the user to discover
 362             # this quickly.
 363             self.loadTheConfigFile()
 364         if signal and hasattr(signal, "SIGHUP"):
 365             signal.signal(signal.SIGHUP, self._handleSIGHUP)
 366         for b in self.botmaster.builders.values():
 367             b.builder_status.addPointEvent(["master", "started"])
 368             b.builder_status.saveYourself()
 369
 370     def useChanges(self, changes):
 371         if self.change_svc:
 372             # TODO: can return a Deferred
 373             self.change_svc.disownServiceParent()
 374         self.change_svc = changes
 375         self.change_svc.basedir = self.basedir
 376         self.change_svc.setName("changemaster")
 377         self.dispatcher.changemaster = self.change_svc
 378         self.change_svc.setServiceParent(self)
 379
 380     def loadChanges(self):
 381         filename = os.path.join(self.basedir, "changes.pck")
 382         try:
 383             changes = load(open(filename, "rb"))
 384             styles.doUpgrade()
 385         except IOError:
 386             log.msg("changes.pck missing, using new one")
 387             changes = ChangeMaster()
 388         except EOFError:
 389             log.msg("corrupted changes.pck, using new one")
 390             changes = ChangeMaster()
 391         self.useChanges(changes)
 392
 393     def _handleSIGHUP(self, *args):
 394         reactor.callLater(0, self.loadTheConfigFile)
 395
 396     def getStatus(self):
 397         """
 398         @rtype: L{buildbot.status.builder.Status}
 399         """
 400         return self.status
 401
 402     def loadTheConfigFile(self, configFile=None):
 403         if not configFile:
 404             configFile = os.path.join(self.basedir, self.configFileName)
 405
 406         log.msg("loading configuration from %s" % configFile)
 407         configFile = os.path.expanduser(configFile)
 408
 409         try:
 410             f = open(configFile, "r")
 411         except IOError, e:
 412             log.msg("unable to open config file '%s'" % configFile)
 413             log.msg("leaving old configuration in place")
 414             log.err(e)
 415             return
 416
 417         try:
 418             self.loadConfig(f)
 419         except:
 420             log.msg("error during loadConfig")
 421             log.err()
 422             log.msg("The new config file is unusable, so I'll ignore it.")
 423             log.msg("I will keep using the previous config file instead.")
 424         f.close()
 425
 426     def loadConfig(self, f):
 427         """Internal function to load a specific configuration file. Any
 428         errors in the file will be signalled by raising an exception.
 429
 430         @return: a Deferred that will fire (with None) when the configuration
 431         changes have been completed. This may involve a round-trip to each
 432         buildslave that was involved."""
 433
 434         localDict = {'basedir': os.path.expanduser(self.basedir)}
 435         try:
 436             exec f in localDict
 437         except:
 438             log.msg("error while parsing config file")
 439             raise
 440
 441         try:
 442             config = localDict['BuildmasterConfig']
 443         except KeyError:
 444             log.err("missing config dictionary")
 445             log.err("config file must define BuildmasterConfig")
 446             raise
 447
 448         known_keys = ("bots", "slaves",
 449                       "sources", "change_source",
 450                       "schedulers", "builders",
 451                       "slavePortnum", "debugPassword", "manhole",
 452                       "status", "projectName", "projectURL", "buildbotURL",
 453                       )
 454         for k in config.keys():
 455             if k not in known_keys:
 456                 log.msg("unknown key '%s' defined in config dictionary" % k)
 457
 458         try:
 459             # required
 460             schedulers = config['schedulers']
 461             builders = config['builders']
 462             slavePortnum = config['slavePortnum']
 463             #slaves = config['slaves']
 464             #change_source = config['change_source']
 465
 466             # optional
 467             debugPassword = config.get('debugPassword')
 468             manhole = config.get('manhole')
 469             status = config.get('status', [])
 470             projectName = config.get('projectName')
 471             projectURL = config.get('projectURL')
 472             buildbotURL = config.get('buildbotURL')
 473
 474         except KeyError, e:
 475             log.msg("config dictionary is missing a required parameter")
 476             log.msg("leaving old configuration in place")
 477             raise
 478
 479         #if "bots" in config:
 480         #    raise KeyError("c['bots'] is no longer accepted")
 481
 482         slaves = config.get('slaves', [])
 483         if "bots" in config:
 484             m = ("c['bots'] is deprecated as of 0.7.6 and will be "
 485                  "removed by 0.8.0 . Please use c['slaves'] instead.")
 486             log.msg(m)
 487             warnings.warn(m, DeprecationWarning)
 488             for name, passwd in config['bots']:
 489                 slaves.append(BuildSlave(name, passwd))
 490
 491         if "bots" not in config and "slaves" not in config:
 492             log.msg("config dictionary must have either 'bots' or 'slaves'")
 493             log.msg("leaving old configuration in place")
 494             raise KeyError("must have either 'bots' or 'slaves'")
 495
 496         #if "sources" in config:
 497         #    raise KeyError("c['sources'] is no longer accepted")
 498
 499         change_source = config.get('change_source', [])
 500         if isinstance(change_source, (list, tuple)):
 501             change_sources = change_source
 502         else:
 503             change_sources = [change_source]
 504         if "sources" in config:
 505             m = ("c['sources'] is deprecated as of 0.7.6 and will be "
 506                  "removed by 0.8.0 . Please use c['change_source'] instead.")
 507             log.msg(m)
 508             warnings.warn(m, DeprecationWarning)
 509             for s in config['sources']:
 510                 change_sources.append(s)
 511
 512         # do some validation first
 513         for s in slaves:
 514             assert isinstance(s, BuildSlave)
 515             if s.slavename in ("debug", "change", "status"):
 516                 raise KeyError, "reserved name '%s' used for a bot" % s.slavename
 517         if config.has_key('interlocks'):
 518             raise KeyError("c['interlocks'] is no longer accepted")
 519
 520         assert isinstance(change_sources, (list, tuple))
 521         for s in change_sources:
 522             assert interfaces.IChangeSource(s, None)
 523         # this assertion catches c['schedulers'] = Scheduler(), since
 524         # Schedulers are service.MultiServices and thus iterable.
 525         errmsg = "c['schedulers'] must be a list of Scheduler instances"
 526         assert isinstance(schedulers, (list, tuple)), errmsg
 527         for s in schedulers:
 528             assert interfaces.IScheduler(s, None), errmsg
 529         assert isinstance(status, (list, tuple))
 530         for s in status:
 531             assert interfaces.IStatusReceiver(s, None)
 532
 533         slavenames = [s.slavename for s in slaves]
 534         buildernames = []
 535         dirnames = []
 536         for b in builders:
 537             if type(b) is tuple:
 538                 raise ValueError("builder %s must be defined with a dict, "
 539                                  "not a tuple" % b[0])
 540             if b.has_key('slavename') and b['slavename'] not in slavenames:
 541                 raise ValueError("builder %s uses undefined slave %s" \
 542                                  % (b['name'], b['slavename']))
 543             for n in b.get('slavenames', []):
 544                 if n not in slavenames:
 545                     raise ValueError("builder %s uses undefined slave %s" \
 546                                      % (b['name'], n))
 547             if b['name'] in buildernames:
 548                 raise ValueError("duplicate builder name %s"
 549                                  % b['name'])
 550             buildernames.append(b['name'])
 551             if b['builddir'] in dirnames:
 552                 raise ValueError("builder %s reuses builddir %s"
 553                                  % (b['name'], b['builddir']))
 554             dirnames.append(b['builddir'])
 555
 556         unscheduled_buildernames = buildernames[:]
 557         schedulernames = []
 558         for s in schedulers:
 559             for b in s.listBuilderNames():
 560                 assert b in buildernames, \
 561                        "%s uses unknown builder %s" % (s, b)
 562                 if b in unscheduled_buildernames:
 563                     unscheduled_buildernames.remove(b)
 564
 565             if s.name in schedulernames:
 566                 # TODO: schedulers share a namespace with other Service
 567                 # children of the BuildMaster node, like status plugins, the
 568                 # Manhole, the ChangeMaster, and the BotMaster (although most
 569                 # of these don't have names)
 570                 msg = ("Schedulers must have unique names, but "
 571                        "'%s' was a duplicate" % (s.name,))
 572                 raise ValueError(msg)
 573             schedulernames.append(s.name)
 574
 575         if unscheduled_buildernames:
 576             log.msg("Warning: some Builders have no Schedulers to drive them:"
 577                     " %s" % (unscheduled_buildernames,))
 578
 579         # assert that all locks used by the Builds and their Steps are
 580         # uniquely named.
 581         locks = {}
 582         for b in builders:
 583             for l in b.get('locks', []):
 584                 if locks.has_key(l.name):
 585                     if locks[l.name] is not l:
 586                         raise ValueError("Two different locks (%s and %s) "
 587                                          "share the name %s"
 588                                          % (l, locks[l.name], l.name))
 589                 else:
 590                     locks[l.name] = l
 591             # TODO: this will break with any BuildFactory that doesn't use a
 592             # .steps list, but I think the verification step is more
 593             # important.
 594             for s in b['factory'].steps:
 595                 for l in s[1].get('locks', []):
 596                     if locks.has_key(l.name):
 597                         if locks[l.name] is not l:
 598                             raise ValueError("Two different locks (%s and %s)"
 599                                              " share the name %s"
 600                                              % (l, locks[l.name], l.name))
 601                     else:
 602                         locks[l.name] = l
 603
 604         # slavePortnum supposed to be a strports specification
 605         if type(slavePortnum) is int:
 606             slavePortnum = "tcp:%d" % slavePortnum
 607
 608         # now we're committed to implementing the new configuration, so do
 609         # it atomically
 610         # TODO: actually, this is spread across a couple of Deferreds, so it
 611         # really isn't atomic.
 612
 613         d = defer.succeed(None)
 614
 615         self.projectName = projectName
 616         self.projectURL = projectURL
 617         self.buildbotURL = buildbotURL
 618
 619         # self.slaves: Disconnect any that were attached and removed from the
 620         # list. Update self.checker with the new list of passwords, including
 621         # debug/change/status.
 622         d.addCallback(lambda res: self.loadConfig_Slaves(slaves))
 623
 624         # self.debugPassword
 625         if debugPassword:
 626             self.checker.addUser("debug", debugPassword)
 627             self.debugPassword = debugPassword
 628
 629         # self.manhole
 630         if manhole != self.manhole:
 631             # changing
 632             if self.manhole:
 633                 # disownServiceParent may return a Deferred
 634                 d.addCallback(lambda res: self.manhole.disownServiceParent())
 635                 def _remove(res):
 636                     self.manhole = None
 637                     return res
 638                 d.addCallback(_remove)
 639             if manhole:
 640                 def _add(res):
 641                     self.manhole = manhole
 642                     manhole.setServiceParent(self)
 643                 d.addCallback(_add)
 644
 645         # add/remove self.botmaster.builders to match builders. The
 646         # botmaster will handle startup/shutdown issues.
 647         d.addCallback(lambda res: self.loadConfig_Builders(builders))
 648
 649         d.addCallback(lambda res: self.loadConfig_status(status))
 650
 651         # Schedulers are added after Builders in case they start right away
 652         d.addCallback(lambda res: self.loadConfig_Schedulers(schedulers))
 653         # and Sources go after Schedulers for the same reason
 654         d.addCallback(lambda res: self.loadConfig_Sources(change_sources))
 655
 656         # self.slavePort
 657         if self.slavePortnum != slavePortnum:
 658             if self.slavePort:
 659                 def closeSlavePort(res):
 660                     d1 = self.slavePort.disownServiceParent()
 661                     self.slavePort = None
 662                     return d1
 663                 d.addCallback(closeSlavePort)
 664             if slavePortnum is not None:
 665                 def openSlavePort(res):
 666                     self.slavePort = strports.service(slavePortnum,
 667                                                       self.slaveFactory)
 668                     self.slavePort.setServiceParent(self)
 669                 d.addCallback(openSlavePort)
 670                 log.msg("BuildMaster listening on port %s" % slavePortnum)
 671             self.slavePortnum = slavePortnum
 672
 673         log.msg("configuration update started")
 674         def _done(res):
 675             self.readConfig = True
 676             log.msg("configuration update complete")
 677         d.addCallback(_done)
 678         d.addCallback(lambda res: self.botmaster.maybeStartAllBuilds())
 679         return d
 680
 681     def loadConfig_Slaves(self, new_slaves):
 682         # set up the Checker with the names and passwords of all valid bots
 683         self.checker.users = {} # violates abstraction, oh well
 684         for s in new_slaves:
 685             self.checker.addUser(s.slavename, s.password)
 686         self.checker.addUser("change", "changepw")
 687
 688         # identify new/old slaves. For each slave we construct a tuple of
 689         # (name, password, class), and we consider the slave to be already
 690         # present if the tuples match. (we include the class to make sure
 691         # that BuildSlave(name,pw) is different than
 692         # SubclassOfBuildSlave(name,pw) ). If the password or class has
 693         # changed, we will remove the old version of the slave and replace it
 694         # with a new one. If anything else has changed, we just update the
 695         # old BuildSlave instance in place. If the name has changed, of
 696         # course, it looks exactly the same as deleting one slave and adding
 697         # an unrelated one.
 698         old_t = {}
 699         for s in self.slaves:
 700             old_t[(s.slavename, s.password, s.__class__)] = s
 701         new_t = {}
 702         for s in new_slaves:
 703             new_t[(s.slavename, s.password, s.__class__)] = s
 704         removed = [old_t[t]
 705                    for t in old_t
 706                    if t not in new_t]
 707         added = [new_t[t]
 708                  for t in new_t
 709                  if t not in old_t]
 710         remaining_t = [t
 711                        for t in new_t
 712                        if t in old_t]
 713         # removeSlave will hang up on the old bot
 714         dl = [self.botmaster.removeSlave(s.slavename) for s in removed]
 715         d = defer.DeferredList(dl, fireOnOneErrback=True)
 716         def _add(res):
 717             for s in added:
 718                 self.botmaster.addSlave(s)
 719             for t in remaining_t:
 720                 old_t[t].update(new_t[t])
 721             self.slaves = new_slaves
 722         d.addCallback(_add)
 723         return d
 724
 725     def loadConfig_Sources(self, sources):
 726         if not sources:
 727             log.msg("warning: no ChangeSources specified in c['change_source']")
 728         # shut down any that were removed, start any that were added
 729         deleted_sources = [s for s in self.change_svc if s not in sources]
 730         added_sources = [s for s in sources if s not in self.change_svc]
 731         dl = [self.change_svc.removeSource(s) for s in deleted_sources]
 732         def addNewOnes(res):
 733             [self.change_svc.addSource(s) for s in added_sources]
 734         d = defer.DeferredList(dl, fireOnOneErrback=1, consumeErrors=0)
 735         d.addCallback(addNewOnes)
 736         return d
 737
 738     def allSchedulers(self):
 739         return [child for child in self
 740                 if interfaces.IScheduler.providedBy(child)]
 741
 742
 743     def loadConfig_Schedulers(self, newschedulers):
 744         oldschedulers = self.allSchedulers()
 745         removed = [s for s in oldschedulers if s not in newschedulers]
 746         added = [s for s in newschedulers if s not in oldschedulers]
 747         dl = [defer.maybeDeferred(s.disownServiceParent) for s in removed]
 748         def addNewOnes(res):
 749             for s in added:
 750                 s.setServiceParent(self)
 751         d = defer.DeferredList(dl, fireOnOneErrback=1)
 752         d.addCallback(addNewOnes)
 753         return d
 754
 755     def loadConfig_Builders(self, newBuilderData):
 756         somethingChanged = False
 757         newList = {}
 758         newBuilderNames = []
 759         allBuilders = self.botmaster.builders.copy()
 760         for data in newBuilderData:
 761             name = data['name']
 762             newList[name] = data
 763             newBuilderNames.append(name)
 764
 765         # identify all that were removed
 766         for oldname in self.botmaster.getBuildernames():
 767             if oldname not in newList:
 768                 log.msg("removing old builder %s" % oldname)
 769                 del allBuilders[oldname]
 770                 somethingChanged = True
 771                 # announce the change
 772                 self.status.builderRemoved(oldname)
 773
 774         # everything in newList is either unchanged, changed, or new
 775         for name, data in newList.items():
 776             old = self.botmaster.builders.get(name)
 777             basedir = data['builddir'] # used on both master and slave
 778             #name, slave, builddir, factory = data
 779             if not old: # new
 780                 # category added after 0.6.2
 781                 category = data.get('category', None)
 782                 log.msg("adding new builder %s for category %s" %
 783                         (name, category))
 784                 statusbag = self.status.builderAdded(name, basedir, category)
 785                 builder = Builder(data, statusbag)
 786                 allBuilders[name] = builder
 787                 somethingChanged = True
 788             elif old.compareToSetup(data):
 789                 # changed: try to minimize the disruption and only modify the
 790                 # pieces that really changed
 791                 diffs = old.compareToSetup(data)
 792                 log.msg("updating builder %s: %s" % (name, "\n".join(diffs)))
 793
 794                 statusbag = old.builder_status
 795                 statusbag.saveYourself() # seems like a good idea
 796                 # TODO: if the basedir was changed, we probably need to make
 797                 # a new statusbag
 798                 new_builder = Builder(data, statusbag)
 799                 new_builder.consumeTheSoulOfYourPredecessor(old)
 800                 # that migrates any retained slavebuilders too
 801
 802                 # point out that the builder was updated. On the Waterfall,
 803                 # this will appear just after any currently-running builds.
 804                 statusbag.addPointEvent(["config", "updated"])
 805
 806                 allBuilders[name] = new_builder
 807                 somethingChanged = True
 808             else:
 809                 # unchanged: leave it alone
 810                 log.msg("builder %s is unchanged" % name)
 811                 pass
 812
 813         if somethingChanged:
 814             sortedAllBuilders = [allBuilders[name] for name in newBuilderNames]
 815             d = self.botmaster.setBuilders(sortedAllBuilders)
 816             return d
 817         return None
 818
 819     def loadConfig_status(self, status):
 820         dl = []
 821
 822         # remove old ones
 823         for s in self.statusTargets[:]:
 824             if not s in status:
 825                 log.msg("removing IStatusReceiver", s)
 826                 d = defer.maybeDeferred(s.disownServiceParent)
 827                 dl.append(d)
 828                 self.statusTargets.remove(s)
 829         # after those are finished going away, add new ones
 830         def addNewOnes(res):
 831             for s in status:
 832                 if not s in self.statusTargets:
 833                     log.msg("adding IStatusReceiver", s)
 834                     s.setServiceParent(self)
 835                     self.statusTargets.append(s)
 836         d = defer.DeferredList(dl, fireOnOneErrback=1)
 837         d.addCallback(addNewOnes)
 838         return d
 839
 840
 841     def addChange(self, change):
 842         for s in self.allSchedulers():
 843             s.addChange(change)
 844
 845     def submitBuildSet(self, bs):
 846         # determine the set of Builders to use
 847         builders = []
 848         for name in bs.builderNames:
 849             b = self.botmaster.builders.get(name)
 850             if b:
 851                 if b not in builders:
 852                     builders.append(b)
 853                 continue
 854             # TODO: add aliases like 'all'
 855             raise KeyError("no such builder named '%s'" % name)
 856
 857         # now tell the BuildSet to create BuildRequests for all those
 858         # Builders and submit them
 859         bs.start(builders)
 860         self.status.buildsetSubmitted(bs.status)
 861
 862
 863 class Control:
 864     implements(interfaces.IControl)
 865
 866     def __init__(self, master):
 867         self.master = master
 868
 869     def addChange(self, change):
 870         self.master.change_svc.addChange(change)
 871
 872     def submitBuildSet(self, bs):
 873         self.master.submitBuildSet(bs)
 874
 875     def getBuilder(self, name):
 876         b = self.master.botmaster.builders[name]
 877         return interfaces.IBuilderControl(b)
 878
 879 components.registerAdapter(Control, BuildMaster, interfaces.IControl)
 880
 881 # so anybody who can get a handle on the BuildMaster can cause a build with:
 882 #  IControl(master).getBuilder("full-2.3").requestBuild(buildrequest)
 883