buildbot/master.py

   1 # -*- test-case-name: buildbot.test.test_run -*-
   2
   3 import os
   4 signal = None
   5 try:
   6     import signal
   7 except ImportError:
   8     pass
   9 from cPickle import load
  10 import warnings
  11
  12 from zope.interface import implements
  13 from twisted.python import log, components
  14 from twisted.internet import defer, reactor
  15 from twisted.spread import pb
  16 from twisted.cred import portal, checkers
  17 from twisted.application import service, strports
  18 from twisted.persisted import styles
  19
  20 import buildbot
  21 # sibling imports
  22 from buildbot.util import now
  23 from buildbot.pbutil import NewCredPerspective
  24 from buildbot.process.builder import Builder, IDLE
  25 from buildbot.process.base import BuildRequest
  26 from buildbot.status.builder import Status
  27 from buildbot.changes.changes import Change, ChangeMaster
  28 from buildbot.sourcestamp import SourceStamp
  29 from buildbot.buildslave import BuildSlave
  30 from buildbot import interfaces, locks
  31 from buildbot.process.properties import Properties
  32
  33 ########################################
  34
  35 class BotMaster(service.MultiService):
  36
  37     """This is the master-side service which manages remote buildbot slaves.
  38     It provides them with BuildSlaves, and distributes file change
  39     notification messages to them.
  40     """
  41
  42     debug = 0
  43
  44     def __init__(self):
  45         service.MultiService.__init__(self)
  46         self.builders = {}
  47         self.builderNames = []
  48         # builders maps Builder names to instances of bb.p.builder.Builder,
  49         # which is the master-side object that defines and controls a build.
  50         # They are added by calling botmaster.addBuilder() from the startup
  51         # code.
  52
  53         # self.slaves contains a ready BuildSlave instance for each
  54         # potential buildslave, i.e. all the ones listed in the config file.
  55         # If the slave is connected, self.slaves[slavename].slave will
  56         # contain a RemoteReference to their Bot instance. If it is not
  57         # connected, that attribute will hold None.
  58         self.slaves = {} # maps slavename to BuildSlave
  59         self.statusClientService = None
  60         self.watchers = {}
  61
  62         # self.locks holds the real Lock instances
  63         self.locks = {}
  64
  65     # these four are convenience functions for testing
  66
  67     def waitUntilBuilderAttached(self, name):
  68         b = self.builders[name]
  69         #if b.slaves:
  70         #    return defer.succeed(None)
  71         d = defer.Deferred()
  72         b.watchers['attach'].append(d)
  73         return d
  74
  75     def waitUntilBuilderDetached(self, name):
  76         b = self.builders.get(name)
  77         if not b or not b.slaves:
  78             return defer.succeed(None)
  79         d = defer.Deferred()
  80         b.watchers['detach'].append(d)
  81         return d
  82
  83     def waitUntilBuilderFullyDetached(self, name):
  84         b = self.builders.get(name)
  85         # TODO: this looks too deeply inside the Builder object
  86         if not b or not b.slaves:
  87             return defer.succeed(None)
  88         d = defer.Deferred()
  89         b.watchers['detach_all'].append(d)
  90         return d
  91
  92     def waitUntilBuilderIdle(self, name):
  93         b = self.builders[name]
  94         # TODO: this looks way too deeply inside the Builder object
  95         for sb in b.slaves:
  96             if sb.state != IDLE:
  97                 d = defer.Deferred()
  98                 b.watchers['idle'].append(d)
  99                 return d
 100         return defer.succeed(None)
 101
 102     def loadConfig_Slaves(self, new_slaves):
 103         old_slaves = [c for c in list(self)
 104                       if interfaces.IBuildSlave.providedBy(c)]
 105
 106         # identify added/removed slaves. For each slave we construct a tuple
 107         # of (name, password, class), and we consider the slave to be already
 108         # present if the tuples match. (we include the class to make sure
 109         # that BuildSlave(name,pw) is different than
 110         # SubclassOfBuildSlave(name,pw) ). If the password or class has
 111         # changed, we will remove the old version of the slave and replace it
 112         # with a new one. If anything else has changed, we just update the
 113         # old BuildSlave instance in place. If the name has changed, of
 114         # course, it looks exactly the same as deleting one slave and adding
 115         # an unrelated one.
 116         old_t = {}
 117         for s in old_slaves:
 118             old_t[(s.slavename, s.password, s.__class__)] = s
 119         new_t = {}
 120         for s in new_slaves:
 121             new_t[(s.slavename, s.password, s.__class__)] = s
 122         removed = [old_t[t]
 123                    for t in old_t
 124                    if t not in new_t]
 125         added = [new_t[t]
 126                  for t in new_t
 127                  if t not in old_t]
 128         remaining_t = [t
 129                        for t in new_t
 130                        if t in old_t]
 131         # removeSlave will hang up on the old bot
 132         dl = []
 133         for s in removed:
 134             dl.append(self.removeSlave(s))
 135         d = defer.DeferredList(dl, fireOnOneErrback=True)
 136         def _add(res):
 137             for s in added:
 138                 self.addSlave(s)
 139             for t in remaining_t:
 140                 old_t[t].update(new_t[t])
 141         d.addCallback(_add)
 142         return d
 143
 144     def addSlave(self, s):
 145         s.setServiceParent(self)
 146         s.setBotmaster(self)
 147         self.slaves[s.slavename] = s
 148
 149     def removeSlave(self, s):
 150         # TODO: technically, disownServiceParent could return a Deferred
 151         s.disownServiceParent()
 152         d = self.slaves[s.slavename].disconnect()
 153         del self.slaves[s.slavename]
 154         return d
 155
 156     def slaveLost(self, bot):
 157         for name, b in self.builders.items():
 158             if bot.slavename in b.slavenames:
 159                 b.detached(bot)
 160
 161     def getBuildersForSlave(self, slavename):
 162         return [b
 163                 for b in self.builders.values()
 164                 if slavename in b.slavenames]
 165
 166     def getBuildernames(self):
 167         return self.builderNames
 168
 169     def getBuilders(self):
 170         allBuilders = [self.builders[name] for name in self.builderNames]
 171         return allBuilders
 172
 173     def setBuilders(self, builders):
 174         self.builders = {}
 175         self.builderNames = []
 176         for b in builders:
 177             for slavename in b.slavenames:
 178                 # this is actually validated earlier
 179                 assert slavename in self.slaves
 180             self.builders[b.name] = b
 181             self.builderNames.append(b.name)
 182             b.setBotmaster(self)
 183         d = self._updateAllSlaves()
 184         return d
 185
 186     def _updateAllSlaves(self):
 187         """Notify all buildslaves about changes in their Builders."""
 188         dl = [s.updateSlave() for s in self.slaves.values()]
 189         return defer.DeferredList(dl)
 190
 191     def maybeStartAllBuilds(self):
 192         builders = self.builders.values()
 193         def _sortfunc(b1, b2):
 194             t1 = b1.getOldestRequestTime()
 195             t2 = b2.getOldestRequestTime()
 196             # If t1 or t2 is None, then there are no build requests,
 197             # so sort it at the end
 198             if t1 is None:
 199                 return 1
 200             if t2 is None:
 201                 return -1
 202             return cmp(t1, t2)
 203         builders.sort(cmp=_sortfunc)
 204         for b in builders:
 205             b.maybeStartBuild()
 206
 207     def getPerspective(self, slavename):
 208         return self.slaves[slavename]
 209
 210     def shutdownSlaves(self):
 211         # TODO: make this into a bot method rather than a builder method
 212         for b in self.slaves.values():
 213             b.shutdownSlave()
 214
 215     def stopService(self):
 216         for b in self.builders.values():
 217             b.builder_status.addPointEvent(["master", "shutdown"])
 218             b.builder_status.saveYourself()
 219         return service.Service.stopService(self)
 220
 221     def getLockByID(self, lockid):
 222         """Convert a Lock identifier into an actual Lock instance.
 223         @param lockid: a locks.MasterLock or locks.SlaveLock instance
 224         @return: a locks.RealMasterLock or locks.RealSlaveLock instance
 225         """
 226         assert isinstance(lockid, (locks.MasterLock, locks.SlaveLock))
 227         if not lockid in self.locks:
 228             self.locks[lockid] = lockid.lockClass(lockid)
 229         # if the master.cfg file has changed maxCount= on the lock, the next
 230         # time a build is started, they'll get a new RealLock instance. Note
 231         # that this requires that MasterLock and SlaveLock (marker) instances
 232         # be hashable and that they should compare properly.
 233         return self.locks[lockid]
 234
 235 ########################################
 236
 237
 238
 239 class DebugPerspective(NewCredPerspective):
 240     def attached(self, mind):
 241         return self
 242     def detached(self, mind):
 243         pass
 244
 245     def perspective_requestBuild(self, buildername, reason, branch, revision, properties={}):
 246         c = interfaces.IControl(self.master)
 247         bc = c.getBuilder(buildername)
 248         ss = SourceStamp(branch, revision)
 249         bpr = Properties()
 250         bpr.update(properties, "remote requestBuild")
 251         br = BuildRequest(reason, ss, builderName=buildername, properties=bpr)
 252         bc.requestBuild(br)
 253
 254     def perspective_pingBuilder(self, buildername):
 255         c = interfaces.IControl(self.master)
 256         bc = c.getBuilder(buildername)
 257         bc.ping()
 258
 259     def perspective_fakeChange(self, file, revision=None, who="fakeUser",
 260                                branch=None):
 261         change = Change(who, [file], "some fake comments\n",
 262                         branch=branch, revision=revision)
 263         c = interfaces.IControl(self.master)
 264         c.addChange(change)
 265
 266     def perspective_setCurrentState(self, buildername, state):
 267         builder = self.botmaster.builders.get(buildername)
 268         if not builder: return
 269         if state == "offline":
 270             builder.statusbag.currentlyOffline()
 271         if state == "idle":
 272             builder.statusbag.currentlyIdle()
 273         if state == "waiting":
 274             builder.statusbag.currentlyWaiting(now()+10)
 275         if state == "building":
 276             builder.statusbag.currentlyBuilding(None)
 277     def perspective_reload(self):
 278         print "doing reload of the config file"
 279         self.master.loadTheConfigFile()
 280     def perspective_pokeIRC(self):
 281         print "saying something on IRC"
 282         from buildbot.status import words
 283         for s in self.master:
 284             if isinstance(s, words.IRC):
 285                 bot = s.f
 286                 for channel in bot.channels:
 287                     print " channel", channel
 288                     bot.p.msg(channel, "Ow, quit it")
 289
 290     def perspective_print(self, msg):
 291         print "debug", msg
 292
 293 class Dispatcher(styles.Versioned):
 294     implements(portal.IRealm)
 295     persistenceVersion = 2
 296
 297     def __init__(self):
 298         self.names = {}
 299
 300     def upgradeToVersion1(self):
 301         self.master = self.botmaster.parent
 302     def upgradeToVersion2(self):
 303         self.names = {}
 304
 305     def register(self, name, afactory):
 306         self.names[name] = afactory
 307     def unregister(self, name):
 308         del self.names[name]
 309
 310     def requestAvatar(self, avatarID, mind, interface):
 311         assert interface == pb.IPerspective
 312         afactory = self.names.get(avatarID)
 313         if afactory:
 314             p = afactory.getPerspective()
 315         elif avatarID == "debug":
 316             p = DebugPerspective()
 317             p.master = self.master
 318             p.botmaster = self.botmaster
 319         elif avatarID == "statusClient":
 320             p = self.statusClientService.getPerspective()
 321         else:
 322             # it must be one of the buildslaves: no other names will make it
 323             # past the checker
 324             p = self.botmaster.getPerspective(avatarID)
 325
 326         if not p:
 327             raise ValueError("no perspective for '%s'" % avatarID)
 328
 329         d = defer.maybeDeferred(p.attached, mind)
 330         d.addCallback(self._avatarAttached, mind)
 331         return d
 332
 333     def _avatarAttached(self, p, mind):
 334         return (pb.IPerspective, p, lambda p=p,mind=mind: p.detached(mind))
 335
 336 ########################################
 337
 338 # service hierarchy:
 339 #  BuildMaster
 340 #   BotMaster
 341 #   ChangeMaster
 342 #    all IChangeSource objects
 343 #   StatusClientService
 344 #   TCPClient(self.ircFactory)
 345 #   TCPServer(self.slaveFactory) -> dispatcher.requestAvatar
 346 #   TCPServer(self.site)
 347 #   UNIXServer(ResourcePublisher(self.site))
 348
 349
 350 class BuildMaster(service.MultiService, styles.Versioned):
 351     debug = 0
 352     persistenceVersion = 3
 353     manhole = None
 354     debugPassword = None
 355     projectName = "(unspecified)"
 356     projectURL = None
 357     buildbotURL = None
 358     change_svc = None
 359     properties = Properties()
 360
 361     def __init__(self, basedir, configFileName="master.cfg"):
 362         service.MultiService.__init__(self)
 363         self.setName("buildmaster")
 364         self.basedir = basedir
 365         self.configFileName = configFileName
 366
 367         # the dispatcher is the realm in which all inbound connections are
 368         # looked up: slave builders, change notifications, status clients, and
 369         # the debug port
 370         dispatcher = Dispatcher()
 371         dispatcher.master = self
 372         self.dispatcher = dispatcher
 373         self.checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
 374         # the checker starts with no user/passwd pairs: they are added later
 375         p = portal.Portal(dispatcher)
 376         p.registerChecker(self.checker)
 377         self.slaveFactory = pb.PBServerFactory(p)
 378         self.slaveFactory.unsafeTracebacks = True # let them see exceptions
 379
 380         self.slavePortnum = None
 381         self.slavePort = None
 382
 383         self.botmaster = BotMaster()
 384         self.botmaster.setName("botmaster")
 385         self.botmaster.setServiceParent(self)
 386         dispatcher.botmaster = self.botmaster
 387
 388         self.status = Status(self.botmaster, self.basedir)
 389
 390         self.statusTargets = []
 391
 392         # this ChangeMaster is a dummy, only used by tests. In the real
 393         # buildmaster, where the BuildMaster instance is activated
 394         # (startService is called) by twistd, this attribute is overwritten.
 395         self.useChanges(ChangeMaster())
 396
 397         self.readConfig = False
 398
 399     def upgradeToVersion1(self):
 400         self.dispatcher = self.slaveFactory.root.portal.realm
 401
 402     def upgradeToVersion2(self): # post-0.4.3
 403         self.webServer = self.webTCPPort
 404         del self.webTCPPort
 405         self.webDistribServer = self.webUNIXPort
 406         del self.webUNIXPort
 407         self.configFileName = "master.cfg"
 408
 409     def upgradeToVersion3(self):
 410         # post 0.6.3, solely to deal with the 0.6.3 breakage. Starting with
 411         # 0.6.5 I intend to do away with .tap files altogether
 412         self.services = []
 413         self.namedServices = {}
 414         del self.change_svc
 415
 416     def startService(self):
 417         service.MultiService.startService(self)
 418         self.loadChanges() # must be done before loading the config file
 419         if not self.readConfig:
 420             # TODO: consider catching exceptions during this call to
 421             # loadTheConfigFile and bailing (reactor.stop) if it fails,
 422             # since without a config file we can't do anything except reload
 423             # the config file, and it would be nice for the user to discover
 424             # this quickly.
 425             self.loadTheConfigFile()
 426         if signal and hasattr(signal, "SIGHUP"):
 427             signal.signal(signal.SIGHUP, self._handleSIGHUP)
 428         for b in self.botmaster.builders.values():
 429             b.builder_status.addPointEvent(["master", "started"])
 430             b.builder_status.saveYourself()
 431
 432     def useChanges(self, changes):
 433         if self.change_svc:
 434             # TODO: can return a Deferred
 435             self.change_svc.disownServiceParent()
 436         self.change_svc = changes
 437         self.change_svc.basedir = self.basedir
 438         self.change_svc.setName("changemaster")
 439         self.dispatcher.changemaster = self.change_svc
 440         self.change_svc.setServiceParent(self)
 441
 442     def loadChanges(self):
 443         filename = os.path.join(self.basedir, "changes.pck")
 444         try:
 445             changes = load(open(filename, "rb"))
 446             styles.doUpgrade()
 447         except IOError:
 448             log.msg("changes.pck missing, using new one")
 449             changes = ChangeMaster()
 450         except EOFError:
 451             log.msg("corrupted changes.pck, using new one")
 452             changes = ChangeMaster()
 453         self.useChanges(changes)
 454
 455     def _handleSIGHUP(self, *args):
 456         reactor.callLater(0, self.loadTheConfigFile)
 457
 458     def getStatus(self):
 459         """
 460         @rtype: L{buildbot.status.builder.Status}
 461         """
 462         return self.status
 463
 464     def loadTheConfigFile(self, configFile=None):
 465         if not configFile:
 466             configFile = os.path.join(self.basedir, self.configFileName)
 467
 468         log.msg("Creating BuildMaster -- buildbot.version: %s" % buildbot.version)
 469         log.msg("loading configuration from %s" % configFile)
 470         configFile = os.path.expanduser(configFile)
 471
 472         try:
 473             f = open(configFile, "r")
 474         except IOError, e:
 475             log.msg("unable to open config file '%s'" % configFile)
 476             log.msg("leaving old configuration in place")
 477             log.err(e)
 478             return
 479
 480         try:
 481             self.loadConfig(f)
 482         except:
 483             log.msg("error during loadConfig")
 484             log.err()
 485             log.msg("The new config file is unusable, so I'll ignore it.")
 486             log.msg("I will keep using the previous config file instead.")
 487         f.close()
 488
 489     def loadConfig(self, f):
 490         """Internal function to load a specific configuration file. Any
 491         errors in the file will be signalled by raising an exception.
 492
 493         @return: a Deferred that will fire (with None) when the configuration
 494         changes have been completed. This may involve a round-trip to each
 495         buildslave that was involved."""
 496
 497         localDict = {'basedir': os.path.expanduser(self.basedir)}
 498         try:
 499             exec f in localDict
 500         except:
 501             log.msg("error while parsing config file")
 502             raise
 503
 504         try:
 505             config = localDict['BuildmasterConfig']
 506         except KeyError:
 507             log.err("missing config dictionary")
 508             log.err("config file must define BuildmasterConfig")
 509             raise
 510
 511         known_keys = ("bots", "slaves",
 512                       "sources", "change_source",
 513                       "schedulers", "builders",
 514                       "slavePortnum", "debugPassword", "manhole",
 515                       "status", "projectName", "projectURL", "buildbotURL",
 516                       "properties"
 517                       )
 518         for k in config.keys():
 519             if k not in known_keys:
 520                 log.msg("unknown key '%s' defined in config dictionary" % k)
 521
 522         try:
 523             # required
 524             schedulers = config['schedulers']
 525             builders = config['builders']
 526             for k in builders:
 527                 if k['name'].startswith("_"):
 528                     errmsg = ("builder names must not start with an "
 529                               "underscore: " + k['name'])
 530                     log.err(errmsg)
 531                     raise ValueError(errmsg)
 532
 533             slavePortnum = config['slavePortnum']
 534             #slaves = config['slaves']
 535             #change_source = config['change_source']
 536
 537             # optional
 538             debugPassword = config.get('debugPassword')
 539             manhole = config.get('manhole')
 540             status = config.get('status', [])
 541             projectName = config.get('projectName')
 542             projectURL = config.get('projectURL')
 543             buildbotURL = config.get('buildbotURL')
 544             properties = config.get('properties', {})
 545
 546         except KeyError, e:
 547             log.msg("config dictionary is missing a required parameter")
 548             log.msg("leaving old configuration in place")
 549             raise
 550
 551         #if "bots" in config:
 552         #    raise KeyError("c['bots'] is no longer accepted")
 553
 554         slaves = config.get('slaves', [])
 555         if "bots" in config:
 556             m = ("c['bots'] is deprecated as of 0.7.6 and will be "
 557                  "removed by 0.8.0 . Please use c['slaves'] instead.")
 558             log.msg(m)
 559             warnings.warn(m, DeprecationWarning)
 560             for name, passwd in config['bots']:
 561                 slaves.append(BuildSlave(name, passwd))
 562
 563         if "bots" not in config and "slaves" not in config:
 564             log.msg("config dictionary must have either 'bots' or 'slaves'")
 565             log.msg("leaving old configuration in place")
 566             raise KeyError("must have either 'bots' or 'slaves'")
 567
 568         #if "sources" in config:
 569         #    raise KeyError("c['sources'] is no longer accepted")
 570
 571         change_source = config.get('change_source', [])
 572         if isinstance(change_source, (list, tuple)):
 573             change_sources = change_source
 574         else:
 575             change_sources = [change_source]
 576         if "sources" in config:
 577             m = ("c['sources'] is deprecated as of 0.7.6 and will be "
 578                  "removed by 0.8.0 . Please use c['change_source'] instead.")
 579             log.msg(m)
 580             warnings.warn(m, DeprecationWarning)
 581             for s in config['sources']:
 582                 change_sources.append(s)
 583
 584         # do some validation first
 585         for s in slaves:
 586             assert isinstance(s, BuildSlave)
 587             if s.slavename in ("debug", "change", "status"):
 588                 raise KeyError, "reserved name '%s' used for a bot" % s.slavename
 589         if config.has_key('interlocks'):
 590             raise KeyError("c['interlocks'] is no longer accepted")
 591
 592         assert isinstance(change_sources, (list, tuple))
 593         for s in change_sources:
 594             assert interfaces.IChangeSource(s, None)
 595         # this assertion catches c['schedulers'] = Scheduler(), since
 596         # Schedulers are service.MultiServices and thus iterable.
 597         errmsg = "c['schedulers'] must be a list of Scheduler instances"
 598         assert isinstance(schedulers, (list, tuple)), errmsg
 599         for s in schedulers:
 600             assert interfaces.IScheduler(s, None), errmsg
 601         assert isinstance(status, (list, tuple))
 602         for s in status:
 603             assert interfaces.IStatusReceiver(s, None)
 604
 605         slavenames = [s.slavename for s in slaves]
 606         buildernames = []
 607         dirnames = []
 608         for b in builders:
 609             if type(b) is tuple:
 610                 raise ValueError("builder %s must be defined with a dict, "
 611                                  "not a tuple" % b[0])
 612             if b.has_key('slavename') and b['slavename'] not in slavenames:
 613                 raise ValueError("builder %s uses undefined slave %s" \
 614                                  % (b['name'], b['slavename']))
 615             for n in b.get('slavenames', []):
 616                 if n not in slavenames:
 617                     raise ValueError("builder %s uses undefined slave %s" \
 618                                      % (b['name'], n))
 619             if b['name'] in buildernames:
 620                 raise ValueError("duplicate builder name %s"
 621                                  % b['name'])
 622             buildernames.append(b['name'])
 623             if b['builddir'] in dirnames:
 624                 raise ValueError("builder %s reuses builddir %s"
 625                                  % (b['name'], b['builddir']))
 626             dirnames.append(b['builddir'])
 627
 628         unscheduled_buildernames = buildernames[:]
 629         schedulernames = []
 630         for s in schedulers:
 631             for b in s.listBuilderNames():
 632                 assert b in buildernames, \
 633                        "%s uses unknown builder %s" % (s, b)
 634                 if b in unscheduled_buildernames:
 635                     unscheduled_buildernames.remove(b)
 636
 637             if s.name in schedulernames:
 638                 # TODO: schedulers share a namespace with other Service
 639                 # children of the BuildMaster node, like status plugins, the
 640                 # Manhole, the ChangeMaster, and the BotMaster (although most
 641                 # of these don't have names)
 642                 msg = ("Schedulers must have unique names, but "
 643                        "'%s' was a duplicate" % (s.name,))
 644                 raise ValueError(msg)
 645             schedulernames.append(s.name)
 646
 647         if unscheduled_buildernames:
 648             log.msg("Warning: some Builders have no Schedulers to drive them:"
 649                     " %s" % (unscheduled_buildernames,))
 650
 651         # assert that all locks used by the Builds and their Steps are
 652         # uniquely named.
 653         lock_dict = {}
 654         for b in builders:
 655             for l in b.get('locks', []):
 656                 if isinstance(l, locks.LockAccess): # User specified access to the lock
 657                     l = l.lockid
 658                 if lock_dict.has_key(l.name):
 659                     if lock_dict[l.name] is not l:
 660                         raise ValueError("Two different locks (%s and %s) "
 661                                          "share the name %s"
 662                                          % (l, lock_dict[l.name], l.name))
 663                 else:
 664                     lock_dict[l.name] = l
 665             # TODO: this will break with any BuildFactory that doesn't use a
 666             # .steps list, but I think the verification step is more
 667             # important.
 668             for s in b['factory'].steps:
 669                 for l in s[1].get('locks', []):
 670                     if isinstance(l, locks.LockAccess): # User specified access to the lock
 671                         l = l.lockid
 672                     if lock_dict.has_key(l.name):
 673                         if lock_dict[l.name] is not l:
 674                             raise ValueError("Two different locks (%s and %s)"
 675                                              " share the name %s"
 676                                              % (l, lock_dict[l.name], l.name))
 677                     else:
 678                         lock_dict[l.name] = l
 679
 680         if not isinstance(properties, dict):
 681             raise ValueError("c['properties'] must be a dictionary")
 682
 683         # slavePortnum supposed to be a strports specification
 684         if type(slavePortnum) is int:
 685             slavePortnum = "tcp:%d" % slavePortnum
 686
 687         # now we're committed to implementing the new configuration, so do
 688         # it atomically
 689         # TODO: actually, this is spread across a couple of Deferreds, so it
 690         # really isn't atomic.
 691
 692         d = defer.succeed(None)
 693
 694         self.projectName = projectName
 695         self.projectURL = projectURL
 696         self.buildbotURL = buildbotURL
 697
 698         self.properties = Properties()
 699         self.properties.update(properties, self.configFileName)
 700
 701         # self.slaves: Disconnect any that were attached and removed from the
 702         # list. Update self.checker with the new list of passwords, including
 703         # debug/change/status.
 704         d.addCallback(lambda res: self.loadConfig_Slaves(slaves))
 705
 706         # self.debugPassword
 707         if debugPassword:
 708             self.checker.addUser("debug", debugPassword)
 709             self.debugPassword = debugPassword
 710
 711         # self.manhole
 712         if manhole != self.manhole:
 713             # changing
 714             if self.manhole:
 715                 # disownServiceParent may return a Deferred
 716                 d.addCallback(lambda res: self.manhole.disownServiceParent())
 717                 def _remove(res):
 718                     self.manhole = None
 719                     return res
 720                 d.addCallback(_remove)
 721             if manhole:
 722                 def _add(res):
 723                     self.manhole = manhole
 724                     manhole.setServiceParent(self)
 725                 d.addCallback(_add)
 726
 727         # add/remove self.botmaster.builders to match builders. The
 728         # botmaster will handle startup/shutdown issues.
 729         d.addCallback(lambda res: self.loadConfig_Builders(builders))
 730
 731         d.addCallback(lambda res: self.loadConfig_status(status))
 732
 733         # Schedulers are added after Builders in case they start right away
 734         d.addCallback(lambda res: self.loadConfig_Schedulers(schedulers))
 735         # and Sources go after Schedulers for the same reason
 736         d.addCallback(lambda res: self.loadConfig_Sources(change_sources))
 737
 738         # self.slavePort
 739         if self.slavePortnum != slavePortnum:
 740             if self.slavePort:
 741                 def closeSlavePort(res):
 742                     d1 = self.slavePort.disownServiceParent()
 743                     self.slavePort = None
 744                     return d1
 745                 d.addCallback(closeSlavePort)
 746             if slavePortnum is not None:
 747                 def openSlavePort(res):
 748                     self.slavePort = strports.service(slavePortnum,
 749                                                       self.slaveFactory)
 750                     self.slavePort.setServiceParent(self)
 751                 d.addCallback(openSlavePort)
 752                 log.msg("BuildMaster listening on port %s" % slavePortnum)
 753             self.slavePortnum = slavePortnum
 754
 755         log.msg("configuration update started")
 756         def _done(res):
 757             self.readConfig = True
 758             log.msg("configuration update complete")
 759         d.addCallback(_done)
 760         d.addCallback(lambda res: self.botmaster.maybeStartAllBuilds())
 761         return d
 762
 763     def loadConfig_Slaves(self, new_slaves):
 764         # set up the Checker with the names and passwords of all valid bots
 765         self.checker.users = {} # violates abstraction, oh well
 766         for s in new_slaves:
 767             self.checker.addUser(s.slavename, s.password)
 768         self.checker.addUser("change", "changepw")
 769         # let the BotMaster take care of the rest
 770         return self.botmaster.loadConfig_Slaves(new_slaves)
 771
 772     def loadConfig_Sources(self, sources):
 773         if not sources:
 774             log.msg("warning: no ChangeSources specified in c['change_source']")
 775         # shut down any that were removed, start any that were added
 776         deleted_sources = [s for s in self.change_svc if s not in sources]
 777         added_sources = [s for s in sources if s not in self.change_svc]
 778         dl = [self.change_svc.removeSource(s) for s in deleted_sources]
 779         def addNewOnes(res):
 780             [self.change_svc.addSource(s) for s in added_sources]
 781         d = defer.DeferredList(dl, fireOnOneErrback=1, consumeErrors=0)
 782         d.addCallback(addNewOnes)
 783         return d
 784
 785     def allSchedulers(self):
 786         return [child for child in self
 787                 if interfaces.IScheduler.providedBy(child)]
 788
 789
 790     def loadConfig_Schedulers(self, newschedulers):
 791         oldschedulers = self.allSchedulers()
 792         removed = [s for s in oldschedulers if s not in newschedulers]
 793         added = [s for s in newschedulers if s not in oldschedulers]
 794         dl = [defer.maybeDeferred(s.disownServiceParent) for s in removed]
 795         def addNewOnes(res):
 796             log.msg("adding %d new schedulers, removed %d" %
 797                     (len(added), len(dl)))
 798             for s in added:
 799                 s.setServiceParent(self)
 800         d = defer.DeferredList(dl, fireOnOneErrback=1)
 801         d.addCallback(addNewOnes)
 802         if removed or added:
 803             # notify Downstream schedulers to potentially pick up
 804             # new schedulers now that we have removed and added some
 805             def updateDownstreams(res):
 806                 log.msg("notifying downstream schedulers of changes")
 807                 for s in newschedulers:
 808                     if interfaces.IDownstreamScheduler.providedBy(s):
 809                         s.updateSchedulers()
 810             d.addCallback(updateDownstreams)
 811         return d
 812
 813     def loadConfig_Builders(self, newBuilderData):
 814         somethingChanged = False
 815         newList = {}
 816         newBuilderNames = []
 817         allBuilders = self.botmaster.builders.copy()
 818         for data in newBuilderData:
 819             name = data['name']
 820             newList[name] = data
 821             newBuilderNames.append(name)
 822
 823         # identify all that were removed
 824         for oldname in self.botmaster.getBuildernames():
 825             if oldname not in newList:
 826                 log.msg("removing old builder %s" % oldname)
 827                 del allBuilders[oldname]
 828                 somethingChanged = True
 829                 # announce the change
 830                 self.status.builderRemoved(oldname)
 831
 832         # everything in newList is either unchanged, changed, or new
 833         for name, data in newList.items():
 834             old = self.botmaster.builders.get(name)
 835             basedir = data['builddir'] # used on both master and slave
 836             #name, slave, builddir, factory = data
 837             if not old: # new
 838                 # category added after 0.6.2
 839                 category = data.get('category', None)
 840                 log.msg("adding new builder %s for category %s" %
 841                         (name, category))
 842                 statusbag = self.status.builderAdded(name, basedir, category)
 843                 builder = Builder(data, statusbag)
 844                 allBuilders[name] = builder
 845                 somethingChanged = True
 846             elif old.compareToSetup(data):
 847                 # changed: try to minimize the disruption and only modify the
 848                 # pieces that really changed
 849                 diffs = old.compareToSetup(data)
 850                 log.msg("updating builder %s: %s" % (name, "\n".join(diffs)))
 851
 852                 statusbag = old.builder_status
 853                 statusbag.saveYourself() # seems like a good idea
 854                 # TODO: if the basedir was changed, we probably need to make
 855                 # a new statusbag
 856                 new_builder = Builder(data, statusbag)
 857                 new_builder.consumeTheSoulOfYourPredecessor(old)
 858                 # that migrates any retained slavebuilders too
 859
 860                 # point out that the builder was updated. On the Waterfall,
 861                 # this will appear just after any currently-running builds.
 862                 statusbag.addPointEvent(["config", "updated"])
 863
 864                 allBuilders[name] = new_builder
 865                 somethingChanged = True
 866             else:
 867                 # unchanged: leave it alone
 868                 log.msg("builder %s is unchanged" % name)
 869                 pass
 870
 871         if somethingChanged:
 872             sortedAllBuilders = [allBuilders[name] for name in newBuilderNames]
 873             d = self.botmaster.setBuilders(sortedAllBuilders)
 874             return d
 875         return None
 876
 877     def loadConfig_status(self, status):
 878         dl = []
 879
 880         # remove old ones
 881         for s in self.statusTargets[:]:
 882             if not s in status:
 883                 log.msg("removing IStatusReceiver", s)
 884                 d = defer.maybeDeferred(s.disownServiceParent)
 885                 dl.append(d)
 886                 self.statusTargets.remove(s)
 887         # after those are finished going away, add new ones
 888         def addNewOnes(res):
 889             for s in status:
 890                 if not s in self.statusTargets:
 891                     log.msg("adding IStatusReceiver", s)
 892                     s.setServiceParent(self)
 893                     self.statusTargets.append(s)
 894         d = defer.DeferredList(dl, fireOnOneErrback=1)
 895         d.addCallback(addNewOnes)
 896         return d
 897
 898
 899     def addChange(self, change):
 900         for s in self.allSchedulers():
 901             s.addChange(change)
 902
 903     def submitBuildSet(self, bs):
 904         # determine the set of Builders to use
 905         builders = []
 906         for name in bs.builderNames:
 907             b = self.botmaster.builders.get(name)
 908             if b:
 909                 if b not in builders:
 910                     builders.append(b)
 911                 continue
 912             # TODO: add aliases like 'all'
 913             raise KeyError("no such builder named '%s'" % name)
 914
 915         # now tell the BuildSet to create BuildRequests for all those
 916         # Builders and submit them
 917         bs.start(builders)
 918         self.status.buildsetSubmitted(bs.status)
 919
 920
 921 class Control:
 922     implements(interfaces.IControl)
 923
 924     def __init__(self, master):
 925         self.master = master
 926
 927     def addChange(self, change):
 928         self.master.change_svc.addChange(change)
 929
 930     def submitBuildSet(self, bs):
 931         self.master.submitBuildSet(bs)
 932
 933     def getBuilder(self, name):
 934         b = self.master.botmaster.builders[name]
 935         return interfaces.IBuilderControl(b)
 936
 937 components.registerAdapter(Control, BuildMaster, interfaces.IControl)
 938
 939 # so anybody who can get a handle on the BuildMaster can cause a build with:
 940 #  IControl(master).getBuilder("full-2.3").requestBuild(buildrequest)
 941