buildbot/master.py

   1 # -*- test-case-name: buildbot.test.test_run -*-
   2
   3 import os
   4 signal = None
   5 try:
   6     import signal
   7 except ImportError:
   8     pass
   9 from cPickle import load
  10 import warnings
  11
  12 from zope.interface import implements
  13 from twisted.python import log, components
  14 from twisted.python.failure import Failure
  15 from twisted.internet import defer, reactor
  16 from twisted.spread import pb
  17 from twisted.cred import portal, checkers
  18 from twisted.application import service, strports
  19 from twisted.persisted import styles
  20
  21 import buildbot
  22 # sibling imports
  23 from buildbot.util import now
  24 from buildbot.pbutil import NewCredPerspective
  25 from buildbot.process.builder import Builder, IDLE
  26 from buildbot.process.base import BuildRequest
  27 from buildbot.status.builder import Status
  28 from buildbot.changes.changes import Change, ChangeMaster, TestChangeMaster
  29 from buildbot.sourcestamp import SourceStamp
  30 from buildbot.buildslave import BuildSlave
  31 from buildbot import interfaces, locks
  32 from buildbot.process.properties import Properties
  33
  34 ########################################
  35
  36 class BotMaster(service.MultiService):
  37
  38     """This is the master-side service which manages remote buildbot slaves.
  39     It provides them with BuildSlaves, and distributes file change
  40     notification messages to them.
  41     """
  42
  43     debug = 0
  44
  45     def __init__(self):
  46         service.MultiService.__init__(self)
  47         self.builders = {}
  48         self.builderNames = []
  49         # builders maps Builder names to instances of bb.p.builder.Builder,
  50         # which is the master-side object that defines and controls a build.
  51         # They are added by calling botmaster.addBuilder() from the startup
  52         # code.
  53
  54         # self.slaves contains a ready BuildSlave instance for each
  55         # potential buildslave, i.e. all the ones listed in the config file.
  56         # If the slave is connected, self.slaves[slavename].slave will
  57         # contain a RemoteReference to their Bot instance. If it is not
  58         # connected, that attribute will hold None.
  59         self.slaves = {} # maps slavename to BuildSlave
  60         self.statusClientService = None
  61         self.watchers = {}
  62
  63         # self.locks holds the real Lock instances
  64         self.locks = {}
  65
  66         # self.mergeRequests is the callable override for merging build
  67         # requests
  68         self.mergeRequests = None
  69
  70         # self.prioritizeBuilders is the callable override for builder order
  71         # traversal
  72         self.prioritizeBuilders = None
  73
  74     # these four are convenience functions for testing
  75
  76     def waitUntilBuilderAttached(self, name):
  77         b = self.builders[name]
  78         #if b.slaves:
  79         #    return defer.succeed(None)
  80         d = defer.Deferred()
  81         b.watchers['attach'].append(d)
  82         return d
  83
  84     def waitUntilBuilderDetached(self, name):
  85         b = self.builders.get(name)
  86         if not b or not b.slaves:
  87             return defer.succeed(None)
  88         d = defer.Deferred()
  89         b.watchers['detach'].append(d)
  90         return d
  91
  92     def waitUntilBuilderFullyDetached(self, name):
  93         b = self.builders.get(name)
  94         # TODO: this looks too deeply inside the Builder object
  95         if not b or not b.slaves:
  96             return defer.succeed(None)
  97         d = defer.Deferred()
  98         b.watchers['detach_all'].append(d)
  99         return d
 100
 101     def waitUntilBuilderIdle(self, name):
 102         b = self.builders[name]
 103         # TODO: this looks way too deeply inside the Builder object
 104         for sb in b.slaves:
 105             if sb.state != IDLE:
 106                 d = defer.Deferred()
 107                 b.watchers['idle'].append(d)
 108                 return d
 109         return defer.succeed(None)
 110
 111     def loadConfig_Slaves(self, new_slaves):
 112         old_slaves = [c for c in list(self)
 113                       if interfaces.IBuildSlave.providedBy(c)]
 114
 115         # identify added/removed slaves. For each slave we construct a tuple
 116         # of (name, password, class), and we consider the slave to be already
 117         # present if the tuples match. (we include the class to make sure
 118         # that BuildSlave(name,pw) is different than
 119         # SubclassOfBuildSlave(name,pw) ). If the password or class has
 120         # changed, we will remove the old version of the slave and replace it
 121         # with a new one. If anything else has changed, we just update the
 122         # old BuildSlave instance in place. If the name has changed, of
 123         # course, it looks exactly the same as deleting one slave and adding
 124         # an unrelated one.
 125         old_t = {}
 126         for s in old_slaves:
 127             old_t[(s.slavename, s.password, s.__class__)] = s
 128         new_t = {}
 129         for s in new_slaves:
 130             new_t[(s.slavename, s.password, s.__class__)] = s
 131         removed = [old_t[t]
 132                    for t in old_t
 133                    if t not in new_t]
 134         added = [new_t[t]
 135                  for t in new_t
 136                  if t not in old_t]
 137         remaining_t = [t
 138                        for t in new_t
 139                        if t in old_t]
 140         # removeSlave will hang up on the old bot
 141         dl = []
 142         for s in removed:
 143             dl.append(self.removeSlave(s))
 144         d = defer.DeferredList(dl, fireOnOneErrback=True)
 145         def _add(res):
 146             for s in added:
 147                 self.addSlave(s)
 148             for t in remaining_t:
 149                 old_t[t].update(new_t[t])
 150         d.addCallback(_add)
 151         return d
 152
 153     def addSlave(self, s):
 154         s.setServiceParent(self)
 155         s.setBotmaster(self)
 156         self.slaves[s.slavename] = s
 157
 158     def removeSlave(self, s):
 159         # TODO: technically, disownServiceParent could return a Deferred
 160         s.disownServiceParent()
 161         d = self.slaves[s.slavename].disconnect()
 162         del self.slaves[s.slavename]
 163         return d
 164
 165     def slaveLost(self, bot):
 166         for name, b in self.builders.items():
 167             if bot.slavename in b.slavenames:
 168                 b.detached(bot)
 169
 170     def getBuildersForSlave(self, slavename):
 171         return [b
 172                 for b in self.builders.values()
 173                 if slavename in b.slavenames]
 174
 175     def getBuildernames(self):
 176         return self.builderNames
 177
 178     def getBuilders(self):
 179         allBuilders = [self.builders[name] for name in self.builderNames]
 180         return allBuilders
 181
 182     def setBuilders(self, builders):
 183         self.builders = {}
 184         self.builderNames = []
 185         for b in builders:
 186             for slavename in b.slavenames:
 187                 # this is actually validated earlier
 188                 assert slavename in self.slaves
 189             self.builders[b.name] = b
 190             self.builderNames.append(b.name)
 191             b.setBotmaster(self)
 192         d = self._updateAllSlaves()
 193         return d
 194
 195     def _updateAllSlaves(self):
 196         """Notify all buildslaves about changes in their Builders."""
 197         dl = [s.updateSlave() for s in self.slaves.values()]
 198         return defer.DeferredList(dl)
 199
 200     def maybeStartAllBuilds(self):
 201         builders = self.builders.values()
 202         if self.prioritizeBuilders is not None:
 203             try:
 204                 builders = self.prioritizeBuilders(self.parent, builders)
 205             except:
 206                 log.msg("Exception prioritizing builders")
 207                 log.err(Failure())
 208                 return
 209         else:
 210             def _sortfunc(b1, b2):
 211                 t1 = b1.getOldestRequestTime()
 212                 t2 = b2.getOldestRequestTime()
 213                 # If t1 or t2 is None, then there are no build requests,
 214                 # so sort it at the end
 215                 if t1 is None:
 216                     return 1
 217                 if t2 is None:
 218                     return -1
 219                 return cmp(t1, t2)
 220             builders.sort(cmp=_sortfunc)
 221         try:
 222             for b in builders:
 223                 b.maybeStartBuild()
 224         except:
 225             log.msg("Exception starting builds")
 226             log.err(Failure())
 227
 228     def shouldMergeRequests(self, builder, req1, req2):
 229         """Determine whether two BuildRequests should be merged for
 230         the given builder.
 231
 232         """
 233         if self.mergeRequests is not None:
 234             return self.mergeRequests(builder, req1, req2)
 235         return req1.canBeMergedWith(req2)
 236
 237     def getPerspective(self, slavename):
 238         return self.slaves[slavename]
 239
 240     def shutdownSlaves(self):
 241         # TODO: make this into a bot method rather than a builder method
 242         for b in self.slaves.values():
 243             b.shutdownSlave()
 244
 245     def stopService(self):
 246         for b in self.builders.values():
 247             b.builder_status.addPointEvent(["master", "shutdown"])
 248             b.builder_status.saveYourself()
 249         return service.Service.stopService(self)
 250
 251     def getLockByID(self, lockid):
 252         """Convert a Lock identifier into an actual Lock instance.
 253         @param lockid: a locks.MasterLock or locks.SlaveLock instance
 254         @return: a locks.RealMasterLock or locks.RealSlaveLock instance
 255         """
 256         assert isinstance(lockid, (locks.MasterLock, locks.SlaveLock))
 257         if not lockid in self.locks:
 258             self.locks[lockid] = lockid.lockClass(lockid)
 259         # if the master.cfg file has changed maxCount= on the lock, the next
 260         # time a build is started, they'll get a new RealLock instance. Note
 261         # that this requires that MasterLock and SlaveLock (marker) instances
 262         # be hashable and that they should compare properly.
 263         return self.locks[lockid]
 264
 265 ########################################
 266
 267
 268
 269 class DebugPerspective(NewCredPerspective):
 270     def attached(self, mind):
 271         return self
 272     def detached(self, mind):
 273         pass
 274
 275     def perspective_requestBuild(self, buildername, reason, branch, revision, properties={}):
 276         c = interfaces.IControl(self.master)
 277         bc = c.getBuilder(buildername)
 278         ss = SourceStamp(branch, revision)
 279         bpr = Properties()
 280         bpr.update(properties, "remote requestBuild")
 281         br = BuildRequest(reason, ss, builderName=buildername, properties=bpr)
 282         bc.requestBuild(br)
 283
 284     def perspective_pingBuilder(self, buildername):
 285         c = interfaces.IControl(self.master)
 286         bc = c.getBuilder(buildername)
 287         bc.ping()
 288
 289     def perspective_fakeChange(self, file, revision=None, who="fakeUser",
 290                                branch=None):
 291         change = Change(who, [file], "some fake comments\n",
 292                         branch=branch, revision=revision)
 293         c = interfaces.IControl(self.master)
 294         c.addChange(change)
 295
 296     def perspective_setCurrentState(self, buildername, state):
 297         builder = self.botmaster.builders.get(buildername)
 298         if not builder: return
 299         if state == "offline":
 300             builder.statusbag.currentlyOffline()
 301         if state == "idle":
 302             builder.statusbag.currentlyIdle()
 303         if state == "waiting":
 304             builder.statusbag.currentlyWaiting(now()+10)
 305         if state == "building":
 306             builder.statusbag.currentlyBuilding(None)
 307     def perspective_reload(self):
 308         print "doing reload of the config file"
 309         self.master.loadTheConfigFile()
 310     def perspective_pokeIRC(self):
 311         print "saying something on IRC"
 312         from buildbot.status import words
 313         for s in self.master:
 314             if isinstance(s, words.IRC):
 315                 bot = s.f
 316                 for channel in bot.channels:
 317                     print " channel", channel
 318                     bot.p.msg(channel, "Ow, quit it")
 319
 320     def perspective_print(self, msg):
 321         print "debug", msg
 322
 323 class Dispatcher(styles.Versioned):
 324     implements(portal.IRealm)
 325     persistenceVersion = 2
 326
 327     def __init__(self):
 328         self.names = {}
 329
 330     def upgradeToVersion1(self):
 331         self.master = self.botmaster.parent
 332     def upgradeToVersion2(self):
 333         self.names = {}
 334
 335     def register(self, name, afactory):
 336         self.names[name] = afactory
 337     def unregister(self, name):
 338         del self.names[name]
 339
 340     def requestAvatar(self, avatarID, mind, interface):
 341         assert interface == pb.IPerspective
 342         afactory = self.names.get(avatarID)
 343         if afactory:
 344             p = afactory.getPerspective()
 345         elif avatarID == "debug":
 346             p = DebugPerspective()
 347             p.master = self.master
 348             p.botmaster = self.botmaster
 349         elif avatarID == "statusClient":
 350             p = self.statusClientService.getPerspective()
 351         else:
 352             # it must be one of the buildslaves: no other names will make it
 353             # past the checker
 354             p = self.botmaster.getPerspective(avatarID)
 355
 356         if not p:
 357             raise ValueError("no perspective for '%s'" % avatarID)
 358
 359         d = defer.maybeDeferred(p.attached, mind)
 360         d.addCallback(self._avatarAttached, mind)
 361         return d
 362
 363     def _avatarAttached(self, p, mind):
 364         return (pb.IPerspective, p, lambda p=p,mind=mind: p.detached(mind))
 365
 366 ########################################
 367
 368 # service hierarchy:
 369 #  BuildMaster
 370 #   BotMaster
 371 #   ChangeMaster
 372 #    all IChangeSource objects
 373 #   StatusClientService
 374 #   TCPClient(self.ircFactory)
 375 #   TCPServer(self.slaveFactory) -> dispatcher.requestAvatar
 376 #   TCPServer(self.site)
 377 #   UNIXServer(ResourcePublisher(self.site))
 378
 379
 380 class BuildMaster(service.MultiService, styles.Versioned):
 381     debug = 0
 382     persistenceVersion = 3
 383     manhole = None
 384     debugPassword = None
 385     projectName = "(unspecified)"
 386     projectURL = None
 387     buildbotURL = None
 388     change_svc = None
 389     properties = Properties()
 390
 391     def __init__(self, basedir, configFileName="master.cfg"):
 392         service.MultiService.__init__(self)
 393         self.setName("buildmaster")
 394         self.basedir = basedir
 395         self.configFileName = configFileName
 396
 397         # the dispatcher is the realm in which all inbound connections are
 398         # looked up: slave builders, change notifications, status clients, and
 399         # the debug port
 400         dispatcher = Dispatcher()
 401         dispatcher.master = self
 402         self.dispatcher = dispatcher
 403         self.checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
 404         # the checker starts with no user/passwd pairs: they are added later
 405         p = portal.Portal(dispatcher)
 406         p.registerChecker(self.checker)
 407         self.slaveFactory = pb.PBServerFactory(p)
 408         self.slaveFactory.unsafeTracebacks = True # let them see exceptions
 409
 410         self.slavePortnum = None
 411         self.slavePort = None
 412
 413         self.botmaster = BotMaster()
 414         self.botmaster.setName("botmaster")
 415         self.botmaster.setServiceParent(self)
 416         dispatcher.botmaster = self.botmaster
 417
 418         self.status = Status(self.botmaster, self.basedir)
 419
 420         self.statusTargets = []
 421
 422         # this ChangeMaster is a dummy, only used by tests. In the real
 423         # buildmaster, where the BuildMaster instance is activated
 424         # (startService is called) by twistd, this attribute is overwritten.
 425         self.useChanges(TestChangeMaster())
 426
 427         self.readConfig = False
 428
 429     def upgradeToVersion1(self):
 430         self.dispatcher = self.slaveFactory.root.portal.realm
 431
 432     def upgradeToVersion2(self): # post-0.4.3
 433         self.webServer = self.webTCPPort
 434         del self.webTCPPort
 435         self.webDistribServer = self.webUNIXPort
 436         del self.webUNIXPort
 437         self.configFileName = "master.cfg"
 438
 439     def upgradeToVersion3(self):
 440         # post 0.6.3, solely to deal with the 0.6.3 breakage. Starting with
 441         # 0.6.5 I intend to do away with .tap files altogether
 442         self.services = []
 443         self.namedServices = {}
 444         del self.change_svc
 445
 446     def startService(self):
 447         service.MultiService.startService(self)
 448         self.loadChanges() # must be done before loading the config file
 449         if not self.readConfig:
 450             # TODO: consider catching exceptions during this call to
 451             # loadTheConfigFile and bailing (reactor.stop) if it fails,
 452             # since without a config file we can't do anything except reload
 453             # the config file, and it would be nice for the user to discover
 454             # this quickly.
 455             self.loadTheConfigFile()
 456         if signal and hasattr(signal, "SIGHUP"):
 457             signal.signal(signal.SIGHUP, self._handleSIGHUP)
 458         for b in self.botmaster.builders.values():
 459             b.builder_status.addPointEvent(["master", "started"])
 460             b.builder_status.saveYourself()
 461
 462     def useChanges(self, changes):
 463         if self.change_svc:
 464             # TODO: can return a Deferred
 465             self.change_svc.disownServiceParent()
 466         self.change_svc = changes
 467         self.change_svc.basedir = self.basedir
 468         self.change_svc.setName("changemaster")
 469         self.dispatcher.changemaster = self.change_svc
 470         self.change_svc.setServiceParent(self)
 471
 472     def loadChanges(self):
 473         filename = os.path.join(self.basedir, "changes.pck")
 474         try:
 475             changes = load(open(filename, "rb"))
 476             styles.doUpgrade()
 477         except IOError:
 478             log.msg("changes.pck missing, using new one")
 479             changes = ChangeMaster()
 480         except EOFError:
 481             log.msg("corrupted changes.pck, using new one")
 482             changes = ChangeMaster()
 483         self.useChanges(changes)
 484
 485     def _handleSIGHUP(self, *args):
 486         reactor.callLater(0, self.loadTheConfigFile)
 487
 488     def getStatus(self):
 489         """
 490         @rtype: L{buildbot.status.builder.Status}
 491         """
 492         return self.status
 493
 494     def loadTheConfigFile(self, configFile=None):
 495         if not configFile:
 496             configFile = os.path.join(self.basedir, self.configFileName)
 497
 498         log.msg("Creating BuildMaster -- buildbot.version: %s" % buildbot.version)
 499         log.msg("loading configuration from %s" % configFile)
 500         configFile = os.path.expanduser(configFile)
 501
 502         try:
 503             f = open(configFile, "r")
 504         except IOError, e:
 505             log.msg("unable to open config file '%s'" % configFile)
 506             log.msg("leaving old configuration in place")
 507             log.err(e)
 508             return
 509
 510         try:
 511             self.loadConfig(f)
 512         except:
 513             log.msg("error during loadConfig")
 514             log.err()
 515             log.msg("The new config file is unusable, so I'll ignore it.")
 516             log.msg("I will keep using the previous config file instead.")
 517         f.close()
 518
 519     def loadConfig(self, f):
 520         """Internal function to load a specific configuration file. Any
 521         errors in the file will be signalled by raising an exception.
 522
 523         @return: a Deferred that will fire (with None) when the configuration
 524         changes have been completed. This may involve a round-trip to each
 525         buildslave that was involved."""
 526
 527         localDict = {'basedir': os.path.expanduser(self.basedir)}
 528         try:
 529             exec f in localDict
 530         except:
 531             log.msg("error while parsing config file")
 532             raise
 533
 534         try:
 535             config = localDict['BuildmasterConfig']
 536         except KeyError:
 537             log.err("missing config dictionary")
 538             log.err("config file must define BuildmasterConfig")
 539             raise
 540
 541         known_keys = ("bots", "slaves",
 542                       "sources", "change_source",
 543                       "schedulers", "builders", "mergeRequests",
 544                       "slavePortnum", "debugPassword", "logCompressionLimit",
 545                       "manhole", "status", "projectName", "projectURL",
 546                       "buildbotURL", "properties", "prioritizeBuilders",
 547                       "eventHorizon", "buildCacheSize", "logHorizon", "buildHorizon",
 548                       "changeHorizon",
 549                       )
 550         for k in config.keys():
 551             if k not in known_keys:
 552                 log.msg("unknown key '%s' defined in config dictionary" % k)
 553
 554         try:
 555             # required
 556             schedulers = config['schedulers']
 557             builders = config['builders']
 558             for k in builders:
 559                 if k['name'].startswith("_"):
 560                     errmsg = ("builder names must not start with an "
 561                               "underscore: " + k['name'])
 562                     log.err(errmsg)
 563                     raise ValueError(errmsg)
 564
 565             slavePortnum = config['slavePortnum']
 566             #slaves = config['slaves']
 567             #change_source = config['change_source']
 568
 569             # optional
 570             debugPassword = config.get('debugPassword')
 571             manhole = config.get('manhole')
 572             status = config.get('status', [])
 573             projectName = config.get('projectName')
 574             projectURL = config.get('projectURL')
 575             buildbotURL = config.get('buildbotURL')
 576             properties = config.get('properties', {})
 577             buildCacheSize = config.get('buildCacheSize', None)
 578             eventHorizon = config.get('eventHorizon', None)
 579             logHorizon = config.get('logHorizon', None)
 580             buildHorizon = config.get('buildHorizon', None)
 581             logCompressionLimit = config.get('logCompressionLimit')
 582             if logCompressionLimit is not None and not \
 583                     isinstance(logCompressionLimit, int):
 584                 raise ValueError("logCompressionLimit needs to be bool or int")
 585             mergeRequests = config.get('mergeRequests')
 586             if mergeRequests is not None and not callable(mergeRequests):
 587                 raise ValueError("mergeRequests must be a callable")
 588             prioritizeBuilders = config.get('prioritizeBuilders')
 589             if prioritizeBuilders is not None and not callable(prioritizeBuilders):
 590                 raise ValueError("prioritizeBuilders must be callable")
 591             changeHorizon = config.get("changeHorizon")
 592             if changeHorizon is not None and not isinstance(changeHorizon, int):
 593                 raise ValueError("changeHorizon needs to be an int")
 594
 595         except KeyError, e:
 596             log.msg("config dictionary is missing a required parameter")
 597             log.msg("leaving old configuration in place")
 598             raise
 599
 600         #if "bots" in config:
 601         #    raise KeyError("c['bots'] is no longer accepted")
 602
 603         slaves = config.get('slaves', [])
 604         if "bots" in config:
 605             m = ("c['bots'] is deprecated as of 0.7.6 and will be "
 606                  "removed by 0.8.0 . Please use c['slaves'] instead.")
 607             log.msg(m)
 608             warnings.warn(m, DeprecationWarning)
 609             for name, passwd in config['bots']:
 610                 slaves.append(BuildSlave(name, passwd))
 611
 612         if "bots" not in config and "slaves" not in config:
 613             log.msg("config dictionary must have either 'bots' or 'slaves'")
 614             log.msg("leaving old configuration in place")
 615             raise KeyError("must have either 'bots' or 'slaves'")
 616
 617         #if "sources" in config:
 618         #    raise KeyError("c['sources'] is no longer accepted")
 619
 620         if changeHorizon is not None:
 621             self.change_svc.changeHorizon = changeHorizon
 622
 623         change_source = config.get('change_source', [])
 624         if isinstance(change_source, (list, tuple)):
 625             change_sources = change_source
 626         else:
 627             change_sources = [change_source]
 628         if "sources" in config:
 629             m = ("c['sources'] is deprecated as of 0.7.6 and will be "
 630                  "removed by 0.8.0 . Please use c['change_source'] instead.")
 631             log.msg(m)
 632             warnings.warn(m, DeprecationWarning)
 633             for s in config['sources']:
 634                 change_sources.append(s)
 635
 636         # do some validation first
 637         for s in slaves:
 638             assert interfaces.IBuildSlave.providedBy(s)
 639             if s.slavename in ("debug", "change", "status"):
 640                 raise KeyError(
 641                     "reserved name '%s' used for a bot" % s.slavename)
 642         if config.has_key('interlocks'):
 643             raise KeyError("c['interlocks'] is no longer accepted")
 644
 645         assert isinstance(change_sources, (list, tuple))
 646         for s in change_sources:
 647             assert interfaces.IChangeSource(s, None)
 648         # this assertion catches c['schedulers'] = Scheduler(), since
 649         # Schedulers are service.MultiServices and thus iterable.
 650         errmsg = "c['schedulers'] must be a list of Scheduler instances"
 651         assert isinstance(schedulers, (list, tuple)), errmsg
 652         for s in schedulers:
 653             assert interfaces.IScheduler(s, None), errmsg
 654         assert isinstance(status, (list, tuple))
 655         for s in status:
 656             assert interfaces.IStatusReceiver(s, None)
 657
 658         slavenames = [s.slavename for s in slaves]
 659         buildernames = []
 660         dirnames = []
 661         for b in builders:
 662             if type(b) is tuple:
 663                 raise ValueError("builder %s must be defined with a dict, "
 664                                  "not a tuple" % b[0])
 665             if b.has_key('slavename') and b['slavename'] not in slavenames:
 666                 raise ValueError("builder %s uses undefined slave %s" \
 667                                  % (b['name'], b['slavename']))
 668             for n in b.get('slavenames', []):
 669                 if n not in slavenames:
 670                     raise ValueError("builder %s uses undefined slave %s" \
 671                                      % (b['name'], n))
 672             if b['name'] in buildernames:
 673                 raise ValueError("duplicate builder name %s"
 674                                  % b['name'])
 675             buildernames.append(b['name'])
 676             if b['builddir'] in dirnames:
 677                 raise ValueError("builder %s reuses builddir %s"
 678                                  % (b['name'], b['builddir']))
 679             dirnames.append(b['builddir'])
 680
 681         unscheduled_buildernames = buildernames[:]
 682         schedulernames = []
 683         for s in schedulers:
 684             for b in s.listBuilderNames():
 685                 assert b in buildernames, \
 686                        "%s uses unknown builder %s" % (s, b)
 687                 if b in unscheduled_buildernames:
 688                     unscheduled_buildernames.remove(b)
 689
 690             if s.name in schedulernames:
 691                 # TODO: schedulers share a namespace with other Service
 692                 # children of the BuildMaster node, like status plugins, the
 693                 # Manhole, the ChangeMaster, and the BotMaster (although most
 694                 # of these don't have names)
 695                 msg = ("Schedulers must have unique names, but "
 696                        "'%s' was a duplicate" % (s.name,))
 697                 raise ValueError(msg)
 698             schedulernames.append(s.name)
 699
 700         if unscheduled_buildernames:
 701             log.msg("Warning: some Builders have no Schedulers to drive them:"
 702                     " %s" % (unscheduled_buildernames,))
 703
 704         # assert that all locks used by the Builds and their Steps are
 705         # uniquely named.
 706         lock_dict = {}
 707         for b in builders:
 708             for l in b.get('locks', []):
 709                 if isinstance(l, locks.LockAccess): # User specified access to the lock
 710                     l = l.lockid
 711                 if lock_dict.has_key(l.name):
 712                     if lock_dict[l.name] is not l:
 713                         raise ValueError("Two different locks (%s and %s) "
 714                                          "share the name %s"
 715                                          % (l, lock_dict[l.name], l.name))
 716                 else:
 717                     lock_dict[l.name] = l
 718             # TODO: this will break with any BuildFactory that doesn't use a
 719             # .steps list, but I think the verification step is more
 720             # important.
 721             for s in b['factory'].steps:
 722                 for l in s[1].get('locks', []):
 723                     if isinstance(l, locks.LockAccess): # User specified access to the lock
 724                         l = l.lockid
 725                     if lock_dict.has_key(l.name):
 726                         if lock_dict[l.name] is not l:
 727                             raise ValueError("Two different locks (%s and %s)"
 728                                              " share the name %s"
 729                                              % (l, lock_dict[l.name], l.name))
 730                     else:
 731                         lock_dict[l.name] = l
 732
 733         if not isinstance(properties, dict):
 734             raise ValueError("c['properties'] must be a dictionary")
 735
 736         # slavePortnum supposed to be a strports specification
 737         if type(slavePortnum) is int:
 738             slavePortnum = "tcp:%d" % slavePortnum
 739
 740         # now we're committed to implementing the new configuration, so do
 741         # it atomically
 742         # TODO: actually, this is spread across a couple of Deferreds, so it
 743         # really isn't atomic.
 744
 745         d = defer.succeed(None)
 746
 747         self.projectName = projectName
 748         self.projectURL = projectURL
 749         self.buildbotURL = buildbotURL
 750
 751         self.properties = Properties()
 752         self.properties.update(properties, self.configFileName)
 753         if logCompressionLimit is not None:
 754             self.status.logCompressionLimit = logCompressionLimit
 755         if mergeRequests is not None:
 756             self.botmaster.mergeRequests = mergeRequests
 757         if prioritizeBuilders is not None:
 758             self.botmaster.prioritizeBuilders = prioritizeBuilders
 759
 760         self.buildCacheSize = buildCacheSize
 761         self.eventHorizon = eventHorizon
 762         self.logHorizon = logHorizon
 763         self.buildHorizon = buildHorizon
 764
 765         # self.slaves: Disconnect any that were attached and removed from the
 766         # list. Update self.checker with the new list of passwords, including
 767         # debug/change/status.
 768         d.addCallback(lambda res: self.loadConfig_Slaves(slaves))
 769
 770         # self.debugPassword
 771         if debugPassword:
 772             self.checker.addUser("debug", debugPassword)
 773             self.debugPassword = debugPassword
 774
 775         # self.manhole
 776         if manhole != self.manhole:
 777             # changing
 778             if self.manhole:
 779                 # disownServiceParent may return a Deferred
 780                 d.addCallback(lambda res: self.manhole.disownServiceParent())
 781                 def _remove(res):
 782                     self.manhole = None
 783                     return res
 784                 d.addCallback(_remove)
 785             if manhole:
 786                 def _add(res):
 787                     self.manhole = manhole
 788                     manhole.setServiceParent(self)
 789                 d.addCallback(_add)
 790
 791         # add/remove self.botmaster.builders to match builders. The
 792         # botmaster will handle startup/shutdown issues.
 793         d.addCallback(lambda res: self.loadConfig_Builders(builders))
 794
 795         d.addCallback(lambda res: self.loadConfig_status(status))
 796
 797         # Schedulers are added after Builders in case they start right away
 798         d.addCallback(lambda res: self.loadConfig_Schedulers(schedulers))
 799         # and Sources go after Schedulers for the same reason
 800         d.addCallback(lambda res: self.loadConfig_Sources(change_sources))
 801
 802         # self.slavePort
 803         if self.slavePortnum != slavePortnum:
 804             if self.slavePort:
 805                 def closeSlavePort(res):
 806                     d1 = self.slavePort.disownServiceParent()
 807                     self.slavePort = None
 808                     return d1
 809                 d.addCallback(closeSlavePort)
 810             if slavePortnum is not None:
 811                 def openSlavePort(res):
 812                     self.slavePort = strports.service(slavePortnum,
 813                                                       self.slaveFactory)
 814                     self.slavePort.setServiceParent(self)
 815                 d.addCallback(openSlavePort)
 816                 log.msg("BuildMaster listening on port %s" % slavePortnum)
 817             self.slavePortnum = slavePortnum
 818
 819         log.msg("configuration update started")
 820         def _done(res):
 821             self.readConfig = True
 822             log.msg("configuration update complete")
 823         d.addCallback(_done)
 824         d.addCallback(lambda res: self.botmaster.maybeStartAllBuilds())
 825         return d
 826
 827     def loadConfig_Slaves(self, new_slaves):
 828         # set up the Checker with the names and passwords of all valid bots
 829         self.checker.users = {} # violates abstraction, oh well
 830         for s in new_slaves:
 831             self.checker.addUser(s.slavename, s.password)
 832         self.checker.addUser("change", "changepw")
 833         # let the BotMaster take care of the rest
 834         return self.botmaster.loadConfig_Slaves(new_slaves)
 835
 836     def loadConfig_Sources(self, sources):
 837         if not sources:
 838             log.msg("warning: no ChangeSources specified in c['change_source']")
 839         # shut down any that were removed, start any that were added
 840         deleted_sources = [s for s in self.change_svc if s not in sources]
 841         added_sources = [s for s in sources if s not in self.change_svc]
 842         log.msg("adding %d new changesources, removing %d" %
 843                 (len(added_sources), len(deleted_sources)))
 844         dl = [self.change_svc.removeSource(s) for s in deleted_sources]
 845         def addNewOnes(res):
 846             [self.change_svc.addSource(s) for s in added_sources]
 847         d = defer.DeferredList(dl, fireOnOneErrback=1, consumeErrors=0)
 848         d.addCallback(addNewOnes)
 849         return d
 850
 851     def allSchedulers(self):
 852         return [child for child in self
 853                 if interfaces.IScheduler.providedBy(child)]
 854
 855
 856     def loadConfig_Schedulers(self, newschedulers):
 857         oldschedulers = self.allSchedulers()
 858         removed = [s for s in oldschedulers if s not in newschedulers]
 859         added = [s for s in newschedulers if s not in oldschedulers]
 860         dl = [defer.maybeDeferred(s.disownServiceParent) for s in removed]
 861         def addNewOnes(res):
 862             log.msg("adding %d new schedulers, removed %d" %
 863                     (len(added), len(dl)))
 864             for s in added:
 865                 s.setServiceParent(self)
 866         d = defer.DeferredList(dl, fireOnOneErrback=1)
 867         d.addCallback(addNewOnes)
 868         if removed or added:
 869             # notify Downstream schedulers to potentially pick up
 870             # new schedulers now that we have removed and added some
 871             def updateDownstreams(res):
 872                 log.msg("notifying downstream schedulers of changes")
 873                 for s in newschedulers:
 874                     if interfaces.IDownstreamScheduler.providedBy(s):
 875                         s.checkUpstreamScheduler()
 876             d.addCallback(updateDownstreams)
 877         return d
 878
 879     def loadConfig_Builders(self, newBuilderData):
 880         somethingChanged = False
 881         newList = {}
 882         newBuilderNames = []
 883         allBuilders = self.botmaster.builders.copy()
 884         for data in newBuilderData:
 885             name = data['name']
 886             newList[name] = data
 887             newBuilderNames.append(name)
 888
 889         # identify all that were removed
 890         for oldname in self.botmaster.getBuildernames():
 891             if oldname not in newList:
 892                 log.msg("removing old builder %s" % oldname)
 893                 del allBuilders[oldname]
 894                 somethingChanged = True
 895                 # announce the change
 896                 self.status.builderRemoved(oldname)
 897
 898         # everything in newList is either unchanged, changed, or new
 899         for name, data in newList.items():
 900             old = self.botmaster.builders.get(name)
 901             basedir = data['builddir'] # used on both master and slave
 902             #name, slave, builddir, factory = data
 903             if not old: # new
 904                 # category added after 0.6.2
 905                 category = data.get('category', None)
 906                 log.msg("adding new builder %s for category %s" %
 907                         (name, category))
 908                 statusbag = self.status.builderAdded(name, basedir, category)
 909                 builder = Builder(data, statusbag)
 910                 allBuilders[name] = builder
 911                 somethingChanged = True
 912             elif old.compareToSetup(data):
 913                 # changed: try to minimize the disruption and only modify the
 914                 # pieces that really changed
 915                 diffs = old.compareToSetup(data)
 916                 log.msg("updating builder %s: %s" % (name, "\n".join(diffs)))
 917
 918                 statusbag = old.builder_status
 919                 statusbag.saveYourself() # seems like a good idea
 920                 # TODO: if the basedir was changed, we probably need to make
 921                 # a new statusbag
 922                 new_builder = Builder(data, statusbag)
 923                 new_builder.consumeTheSoulOfYourPredecessor(old)
 924                 # that migrates any retained slavebuilders too
 925
 926                 # point out that the builder was updated. On the Waterfall,
 927                 # this will appear just after any currently-running builds.
 928                 statusbag.addPointEvent(["config", "updated"])
 929
 930                 allBuilders[name] = new_builder
 931                 somethingChanged = True
 932             else:
 933                 # unchanged: leave it alone
 934                 log.msg("builder %s is unchanged" % name)
 935                 pass
 936
 937         # regardless of whether anything changed, get each builder status
 938         # to update its config
 939         for builder in allBuilders.values():
 940             builder.builder_status.reconfigFromBuildmaster(self)
 941
 942         # and then tell the botmaster if anything's changed
 943         if somethingChanged:
 944             sortedAllBuilders = [allBuilders[name] for name in newBuilderNames]
 945             d = self.botmaster.setBuilders(sortedAllBuilders)
 946             return d
 947         return None
 948
 949     def loadConfig_status(self, status):
 950         dl = []
 951
 952         # remove old ones
 953         for s in self.statusTargets[:]:
 954             if not s in status:
 955                 log.msg("removing IStatusReceiver", s)
 956                 d = defer.maybeDeferred(s.disownServiceParent)
 957                 dl.append(d)
 958                 self.statusTargets.remove(s)
 959         # after those are finished going away, add new ones
 960         def addNewOnes(res):
 961             for s in status:
 962                 if not s in self.statusTargets:
 963                     log.msg("adding IStatusReceiver", s)
 964                     s.setServiceParent(self)
 965                     self.statusTargets.append(s)
 966         d = defer.DeferredList(dl, fireOnOneErrback=1)
 967         d.addCallback(addNewOnes)
 968         return d
 969
 970
 971     def addChange(self, change):
 972         for s in self.allSchedulers():
 973             s.addChange(change)
 974
 975     def submitBuildSet(self, bs):
 976         # determine the set of Builders to use
 977         builders = []
 978         for name in bs.builderNames:
 979             b = self.botmaster.builders.get(name)
 980             if b:
 981                 if b not in builders:
 982                     builders.append(b)
 983                 continue
 984             # TODO: add aliases like 'all'
 985             raise KeyError("no such builder named '%s'" % name)
 986
 987         # now tell the BuildSet to create BuildRequests for all those
 988         # Builders and submit them
 989         bs.start(builders)
 990         self.status.buildsetSubmitted(bs.status)
 991
 992
 993 class Control:
 994     implements(interfaces.IControl)
 995
 996     def __init__(self, master):
 997         self.master = master
 998
 999     def addChange(self, change):
1000         self.master.change_svc.addChange(change)
1001
1002     def submitBuildSet(self, bs):
1003         self.master.submitBuildSet(bs)
1004
1005     def getBuilder(self, name):
1006         b = self.master.botmaster.builders[name]
1007         return interfaces.IBuilderControl(b)
1008
1009 components.registerAdapter(Control, BuildMaster, interfaces.IControl)
1010
1011 # so anybody who can get a handle on the BuildMaster can cause a build with:
1012 #  IControl(master).getBuilder("full-2.3").requestBuild(buildrequest)