buildbot/master.py

   1 # -*- test-case-name: buildbot.test.test_run -*-
   2
   3 import os
   4 signal = None
   5 try:
   6     import signal
   7 except ImportError:
   8     pass
   9 from cPickle import load
  10 import warnings
  11
  12 from zope.interface import implements
  13 from twisted.python import log, components
  14 from twisted.internet import defer, reactor
  15 from twisted.spread import pb
  16 from twisted.cred import portal, checkers
  17 from twisted.application import service, strports
  18 from twisted.persisted import styles
  19
  20 import buildbot
  21 # sibling imports
  22 from buildbot.util import now
  23 from buildbot.pbutil import NewCredPerspective
  24 from buildbot.process.builder import Builder, IDLE
  25 from buildbot.process.base import BuildRequest
  26 from buildbot.status.builder import Status
  27 from buildbot.changes.changes import Change, ChangeMaster
  28 from buildbot.sourcestamp import SourceStamp
  29 from buildbot.buildslave import BuildSlave
  30 from buildbot import interfaces, locks
  31 from buildbot.process.properties import Properties
  32
  33 ########################################
  34
  35 class BotMaster(service.MultiService):
  36
  37     """This is the master-side service which manages remote buildbot slaves.
  38     It provides them with BuildSlaves, and distributes file change
  39     notification messages to them.
  40     """
  41
  42     debug = 0
  43
  44     def __init__(self):
  45         service.MultiService.__init__(self)
  46         self.builders = {}
  47         self.builderNames = []
  48         # builders maps Builder names to instances of bb.p.builder.Builder,
  49         # which is the master-side object that defines and controls a build.
  50         # They are added by calling botmaster.addBuilder() from the startup
  51         # code.
  52
  53         # self.slaves contains a ready BuildSlave instance for each
  54         # potential buildslave, i.e. all the ones listed in the config file.
  55         # If the slave is connected, self.slaves[slavename].slave will
  56         # contain a RemoteReference to their Bot instance. If it is not
  57         # connected, that attribute will hold None.
  58         self.slaves = {} # maps slavename to BuildSlave
  59         self.statusClientService = None
  60         self.watchers = {}
  61
  62         # self.locks holds the real Lock instances
  63         self.locks = {}
  64
  65     # these four are convenience functions for testing
  66
  67     def waitUntilBuilderAttached(self, name):
  68         b = self.builders[name]
  69         #if b.slaves:
  70         #    return defer.succeed(None)
  71         d = defer.Deferred()
  72         b.watchers['attach'].append(d)
  73         return d
  74
  75     def waitUntilBuilderDetached(self, name):
  76         b = self.builders.get(name)
  77         if not b or not b.slaves:
  78             return defer.succeed(None)
  79         d = defer.Deferred()
  80         b.watchers['detach'].append(d)
  81         return d
  82
  83     def waitUntilBuilderFullyDetached(self, name):
  84         b = self.builders.get(name)
  85         # TODO: this looks too deeply inside the Builder object
  86         if not b or not b.slaves:
  87             return defer.succeed(None)
  88         d = defer.Deferred()
  89         b.watchers['detach_all'].append(d)
  90         return d
  91
  92     def waitUntilBuilderIdle(self, name):
  93         b = self.builders[name]
  94         # TODO: this looks way too deeply inside the Builder object
  95         for sb in b.slaves:
  96             if sb.state != IDLE:
  97                 d = defer.Deferred()
  98                 b.watchers['idle'].append(d)
  99                 return d
 100         return defer.succeed(None)
 101
 102     def loadConfig_Slaves(self, new_slaves):
 103         old_slaves = [c for c in list(self)
 104                       if interfaces.IBuildSlave.providedBy(c)]
 105
 106         # identify added/removed slaves. For each slave we construct a tuple
 107         # of (name, password, class), and we consider the slave to be already
 108         # present if the tuples match. (we include the class to make sure
 109         # that BuildSlave(name,pw) is different than
 110         # SubclassOfBuildSlave(name,pw) ). If the password or class has
 111         # changed, we will remove the old version of the slave and replace it
 112         # with a new one. If anything else has changed, we just update the
 113         # old BuildSlave instance in place. If the name has changed, of
 114         # course, it looks exactly the same as deleting one slave and adding
 115         # an unrelated one.
 116         old_t = {}
 117         for s in old_slaves:
 118             old_t[(s.slavename, s.password, s.__class__)] = s
 119         new_t = {}
 120         for s in new_slaves:
 121             new_t[(s.slavename, s.password, s.__class__)] = s
 122         removed = [old_t[t]
 123                    for t in old_t
 124                    if t not in new_t]
 125         added = [new_t[t]
 126                  for t in new_t
 127                  if t not in old_t]
 128         remaining_t = [t
 129                        for t in new_t
 130                        if t in old_t]
 131         # removeSlave will hang up on the old bot
 132         dl = []
 133         for s in removed:
 134             dl.append(self.removeSlave(s))
 135         d = defer.DeferredList(dl, fireOnOneErrback=True)
 136         def _add(res):
 137             for s in added:
 138                 self.addSlave(s)
 139             for t in remaining_t:
 140                 old_t[t].update(new_t[t])
 141         d.addCallback(_add)
 142         return d
 143
 144     def addSlave(self, s):
 145         s.setServiceParent(self)
 146         s.setBotmaster(self)
 147         self.slaves[s.slavename] = s
 148
 149     def removeSlave(self, s):
 150         # TODO: technically, disownServiceParent could return a Deferred
 151         s.disownServiceParent()
 152         d = self.slaves[s.slavename].disconnect()
 153         del self.slaves[s.slavename]
 154         return d
 155
 156     def slaveLost(self, bot):
 157         for name, b in self.builders.items():
 158             if bot.slavename in b.slavenames:
 159                 b.detached(bot)
 160
 161     def getBuildersForSlave(self, slavename):
 162         return [b
 163                 for b in self.builders.values()
 164                 if slavename in b.slavenames]
 165
 166     def getBuildernames(self):
 167         return self.builderNames
 168
 169     def getBuilders(self):
 170         allBuilders = [self.builders[name] for name in self.builderNames]
 171         return allBuilders
 172
 173     def setBuilders(self, builders):
 174         self.builders = {}
 175         self.builderNames = []
 176         for b in builders:
 177             for slavename in b.slavenames:
 178                 # this is actually validated earlier
 179                 assert slavename in self.slaves
 180             self.builders[b.name] = b
 181             self.builderNames.append(b.name)
 182             b.setBotmaster(self)
 183         d = self._updateAllSlaves()
 184         return d
 185
 186     def _updateAllSlaves(self):
 187         """Notify all buildslaves about changes in their Builders."""
 188         dl = [s.updateSlave() for s in self.slaves.values()]
 189         return defer.DeferredList(dl)
 190
 191     def maybeStartAllBuilds(self):
 192         for b in self.builders.values():
 193             b.maybeStartBuild()
 194
 195     def getPerspective(self, slavename):
 196         return self.slaves[slavename]
 197
 198     def shutdownSlaves(self):
 199         # TODO: make this into a bot method rather than a builder method
 200         for b in self.slaves.values():
 201             b.shutdownSlave()
 202
 203     def stopService(self):
 204         for b in self.builders.values():
 205             b.builder_status.addPointEvent(["master", "shutdown"])
 206             b.builder_status.saveYourself()
 207         return service.Service.stopService(self)
 208
 209     def getLockByID(self, lockid):
 210         """Convert a Lock identifier into an actual Lock instance.
 211         @param lockid: a locks.MasterLock or locks.SlaveLock instance
 212         @return: a locks.RealMasterLock or locks.RealSlaveLock instance
 213         """
 214         assert isinstance(lockid, (locks.MasterLock, locks.SlaveLock))
 215         if not lockid in self.locks:
 216             self.locks[lockid] = lockid.lockClass(lockid)
 217         # if the master.cfg file has changed maxCount= on the lock, the next
 218         # time a build is started, they'll get a new RealLock instance. Note
 219         # that this requires that MasterLock and SlaveLock (marker) instances
 220         # be hashable and that they should compare properly.
 221         return self.locks[lockid]
 222
 223 ########################################
 224
 225
 226
 227 class DebugPerspective(NewCredPerspective):
 228     def attached(self, mind):
 229         return self
 230     def detached(self, mind):
 231         pass
 232
 233     def perspective_requestBuild(self, buildername, reason, branch, revision, properties={}):
 234         c = interfaces.IControl(self.master)
 235         bc = c.getBuilder(buildername)
 236         ss = SourceStamp(branch, revision)
 237         bpr = Properties()
 238         bpr.update(properties, "remote requestBuild")
 239         br = BuildRequest(reason, ss, builderName=buildername, properties=bpr)
 240         bc.requestBuild(br)
 241
 242     def perspective_pingBuilder(self, buildername):
 243         c = interfaces.IControl(self.master)
 244         bc = c.getBuilder(buildername)
 245         bc.ping()
 246
 247     def perspective_fakeChange(self, file, revision=None, who="fakeUser",
 248                                branch=None):
 249         change = Change(who, [file], "some fake comments\n",
 250                         branch=branch, revision=revision)
 251         c = interfaces.IControl(self.master)
 252         c.addChange(change)
 253
 254     def perspective_setCurrentState(self, buildername, state):
 255         builder = self.botmaster.builders.get(buildername)
 256         if not builder: return
 257         if state == "offline":
 258             builder.statusbag.currentlyOffline()
 259         if state == "idle":
 260             builder.statusbag.currentlyIdle()
 261         if state == "waiting":
 262             builder.statusbag.currentlyWaiting(now()+10)
 263         if state == "building":
 264             builder.statusbag.currentlyBuilding(None)
 265     def perspective_reload(self):
 266         print "doing reload of the config file"
 267         self.master.loadTheConfigFile()
 268     def perspective_pokeIRC(self):
 269         print "saying something on IRC"
 270         from buildbot.status import words
 271         for s in self.master:
 272             if isinstance(s, words.IRC):
 273                 bot = s.f
 274                 for channel in bot.channels:
 275                     print " channel", channel
 276                     bot.p.msg(channel, "Ow, quit it")
 277
 278     def perspective_print(self, msg):
 279         print "debug", msg
 280
 281 class Dispatcher(styles.Versioned):
 282     implements(portal.IRealm)
 283     persistenceVersion = 2
 284
 285     def __init__(self):
 286         self.names = {}
 287
 288     def upgradeToVersion1(self):
 289         self.master = self.botmaster.parent
 290     def upgradeToVersion2(self):
 291         self.names = {}
 292
 293     def register(self, name, afactory):
 294         self.names[name] = afactory
 295     def unregister(self, name):
 296         del self.names[name]
 297
 298     def requestAvatar(self, avatarID, mind, interface):
 299         assert interface == pb.IPerspective
 300         afactory = self.names.get(avatarID)
 301         if afactory:
 302             p = afactory.getPerspective()
 303         elif avatarID == "debug":
 304             p = DebugPerspective()
 305             p.master = self.master
 306             p.botmaster = self.botmaster
 307         elif avatarID == "statusClient":
 308             p = self.statusClientService.getPerspective()
 309         else:
 310             # it must be one of the buildslaves: no other names will make it
 311             # past the checker
 312             p = self.botmaster.getPerspective(avatarID)
 313
 314         if not p:
 315             raise ValueError("no perspective for '%s'" % avatarID)
 316
 317         d = defer.maybeDeferred(p.attached, mind)
 318         d.addCallback(self._avatarAttached, mind)
 319         return d
 320
 321     def _avatarAttached(self, p, mind):
 322         return (pb.IPerspective, p, lambda p=p,mind=mind: p.detached(mind))
 323
 324 ########################################
 325
 326 # service hierarchy:
 327 #  BuildMaster
 328 #   BotMaster
 329 #   ChangeMaster
 330 #    all IChangeSource objects
 331 #   StatusClientService
 332 #   TCPClient(self.ircFactory)
 333 #   TCPServer(self.slaveFactory) -> dispatcher.requestAvatar
 334 #   TCPServer(self.site)
 335 #   UNIXServer(ResourcePublisher(self.site))
 336
 337
 338 class BuildMaster(service.MultiService, styles.Versioned):
 339     debug = 0
 340     persistenceVersion = 3
 341     manhole = None
 342     debugPassword = None
 343     projectName = "(unspecified)"
 344     projectURL = None
 345     buildbotURL = None
 346     change_svc = None
 347     properties = Properties()
 348
 349     def __init__(self, basedir, configFileName="master.cfg"):
 350         service.MultiService.__init__(self)
 351         self.setName("buildmaster")
 352         self.basedir = basedir
 353         self.configFileName = configFileName
 354
 355         # the dispatcher is the realm in which all inbound connections are
 356         # looked up: slave builders, change notifications, status clients, and
 357         # the debug port
 358         dispatcher = Dispatcher()
 359         dispatcher.master = self
 360         self.dispatcher = dispatcher
 361         self.checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
 362         # the checker starts with no user/passwd pairs: they are added later
 363         p = portal.Portal(dispatcher)
 364         p.registerChecker(self.checker)
 365         self.slaveFactory = pb.PBServerFactory(p)
 366         self.slaveFactory.unsafeTracebacks = True # let them see exceptions
 367
 368         self.slavePortnum = None
 369         self.slavePort = None
 370
 371         self.botmaster = BotMaster()
 372         self.botmaster.setName("botmaster")
 373         self.botmaster.setServiceParent(self)
 374         dispatcher.botmaster = self.botmaster
 375
 376         self.status = Status(self.botmaster, self.basedir)
 377
 378         self.statusTargets = []
 379
 380         # this ChangeMaster is a dummy, only used by tests. In the real
 381         # buildmaster, where the BuildMaster instance is activated
 382         # (startService is called) by twistd, this attribute is overwritten.
 383         self.useChanges(ChangeMaster())
 384
 385         self.readConfig = False
 386
 387     def upgradeToVersion1(self):
 388         self.dispatcher = self.slaveFactory.root.portal.realm
 389
 390     def upgradeToVersion2(self): # post-0.4.3
 391         self.webServer = self.webTCPPort
 392         del self.webTCPPort
 393         self.webDistribServer = self.webUNIXPort
 394         del self.webUNIXPort
 395         self.configFileName = "master.cfg"
 396
 397     def upgradeToVersion3(self):
 398         # post 0.6.3, solely to deal with the 0.6.3 breakage. Starting with
 399         # 0.6.5 I intend to do away with .tap files altogether
 400         self.services = []
 401         self.namedServices = {}
 402         del self.change_svc
 403
 404     def startService(self):
 405         service.MultiService.startService(self)
 406         self.loadChanges() # must be done before loading the config file
 407         if not self.readConfig:
 408             # TODO: consider catching exceptions during this call to
 409             # loadTheConfigFile and bailing (reactor.stop) if it fails,
 410             # since without a config file we can't do anything except reload
 411             # the config file, and it would be nice for the user to discover
 412             # this quickly.
 413             self.loadTheConfigFile()
 414         if signal and hasattr(signal, "SIGHUP"):
 415             signal.signal(signal.SIGHUP, self._handleSIGHUP)
 416         for b in self.botmaster.builders.values():
 417             b.builder_status.addPointEvent(["master", "started"])
 418             b.builder_status.saveYourself()
 419
 420     def useChanges(self, changes):
 421         if self.change_svc:
 422             # TODO: can return a Deferred
 423             self.change_svc.disownServiceParent()
 424         self.change_svc = changes
 425         self.change_svc.basedir = self.basedir
 426         self.change_svc.setName("changemaster")
 427         self.dispatcher.changemaster = self.change_svc
 428         self.change_svc.setServiceParent(self)
 429
 430     def loadChanges(self):
 431         filename = os.path.join(self.basedir, "changes.pck")
 432         try:
 433             changes = load(open(filename, "rb"))
 434             styles.doUpgrade()
 435         except IOError:
 436             log.msg("changes.pck missing, using new one")
 437             changes = ChangeMaster()
 438         except EOFError:
 439             log.msg("corrupted changes.pck, using new one")
 440             changes = ChangeMaster()
 441         self.useChanges(changes)
 442
 443     def _handleSIGHUP(self, *args):
 444         reactor.callLater(0, self.loadTheConfigFile)
 445
 446     def getStatus(self):
 447         """
 448         @rtype: L{buildbot.status.builder.Status}
 449         """
 450         return self.status
 451
 452     def loadTheConfigFile(self, configFile=None):
 453         if not configFile:
 454             configFile = os.path.join(self.basedir, self.configFileName)
 455
 456         log.msg("Creating BuildMaster -- buildbot.version: %s" % buildbot.version)
 457         log.msg("loading configuration from %s" % configFile)
 458         configFile = os.path.expanduser(configFile)
 459
 460         try:
 461             f = open(configFile, "r")
 462         except IOError, e:
 463             log.msg("unable to open config file '%s'" % configFile)
 464             log.msg("leaving old configuration in place")
 465             log.err(e)
 466             return
 467
 468         try:
 469             self.loadConfig(f)
 470         except:
 471             log.msg("error during loadConfig")
 472             log.err()
 473             log.msg("The new config file is unusable, so I'll ignore it.")
 474             log.msg("I will keep using the previous config file instead.")
 475         f.close()
 476
 477     def loadConfig(self, f):
 478         """Internal function to load a specific configuration file. Any
 479         errors in the file will be signalled by raising an exception.
 480
 481         @return: a Deferred that will fire (with None) when the configuration
 482         changes have been completed. This may involve a round-trip to each
 483         buildslave that was involved."""
 484
 485         localDict = {'basedir': os.path.expanduser(self.basedir)}
 486         try:
 487             exec f in localDict
 488         except:
 489             log.msg("error while parsing config file")
 490             raise
 491
 492         try:
 493             config = localDict['BuildmasterConfig']
 494         except KeyError:
 495             log.err("missing config dictionary")
 496             log.err("config file must define BuildmasterConfig")
 497             raise
 498
 499         known_keys = ("bots", "slaves",
 500                       "sources", "change_source",
 501                       "schedulers", "builders",
 502                       "slavePortnum", "debugPassword", "manhole",
 503                       "status", "projectName", "projectURL", "buildbotURL",
 504                       "properties"
 505                       )
 506         for k in config.keys():
 507             if k not in known_keys:
 508                 log.msg("unknown key '%s' defined in config dictionary" % k)
 509
 510         try:
 511             # required
 512             schedulers = config['schedulers']
 513             builders = config['builders']
 514             for k in builders:
 515                 if k['name'].startswith("_"):
 516                     errmsg = ("builder names must not start with an "
 517                               "underscore: " + k['name'])
 518                     log.err(errmsg)
 519                     raise ValueError(errmsg)
 520
 521             slavePortnum = config['slavePortnum']
 522             #slaves = config['slaves']
 523             #change_source = config['change_source']
 524
 525             # optional
 526             debugPassword = config.get('debugPassword')
 527             manhole = config.get('manhole')
 528             status = config.get('status', [])
 529             projectName = config.get('projectName')
 530             projectURL = config.get('projectURL')
 531             buildbotURL = config.get('buildbotURL')
 532             properties = config.get('properties', {})
 533
 534         except KeyError, e:
 535             log.msg("config dictionary is missing a required parameter")
 536             log.msg("leaving old configuration in place")
 537             raise
 538
 539         #if "bots" in config:
 540         #    raise KeyError("c['bots'] is no longer accepted")
 541
 542         slaves = config.get('slaves', [])
 543         if "bots" in config:
 544             m = ("c['bots'] is deprecated as of 0.7.6 and will be "
 545                  "removed by 0.8.0 . Please use c['slaves'] instead.")
 546             log.msg(m)
 547             warnings.warn(m, DeprecationWarning)
 548             for name, passwd in config['bots']:
 549                 slaves.append(BuildSlave(name, passwd))
 550
 551         if "bots" not in config and "slaves" not in config:
 552             log.msg("config dictionary must have either 'bots' or 'slaves'")
 553             log.msg("leaving old configuration in place")
 554             raise KeyError("must have either 'bots' or 'slaves'")
 555
 556         #if "sources" in config:
 557         #    raise KeyError("c['sources'] is no longer accepted")
 558
 559         change_source = config.get('change_source', [])
 560         if isinstance(change_source, (list, tuple)):
 561             change_sources = change_source
 562         else:
 563             change_sources = [change_source]
 564         if "sources" in config:
 565             m = ("c['sources'] is deprecated as of 0.7.6 and will be "
 566                  "removed by 0.8.0 . Please use c['change_source'] instead.")
 567             log.msg(m)
 568             warnings.warn(m, DeprecationWarning)
 569             for s in config['sources']:
 570                 change_sources.append(s)
 571
 572         # do some validation first
 573         for s in slaves:
 574             assert isinstance(s, BuildSlave)
 575             if s.slavename in ("debug", "change", "status"):
 576                 raise KeyError, "reserved name '%s' used for a bot" % s.slavename
 577         if config.has_key('interlocks'):
 578             raise KeyError("c['interlocks'] is no longer accepted")
 579
 580         assert isinstance(change_sources, (list, tuple))
 581         for s in change_sources:
 582             assert interfaces.IChangeSource(s, None)
 583         # this assertion catches c['schedulers'] = Scheduler(), since
 584         # Schedulers are service.MultiServices and thus iterable.
 585         errmsg = "c['schedulers'] must be a list of Scheduler instances"
 586         assert isinstance(schedulers, (list, tuple)), errmsg
 587         for s in schedulers:
 588             assert interfaces.IScheduler(s, None), errmsg
 589         assert isinstance(status, (list, tuple))
 590         for s in status:
 591             assert interfaces.IStatusReceiver(s, None)
 592
 593         slavenames = [s.slavename for s in slaves]
 594         buildernames = []
 595         dirnames = []
 596         for b in builders:
 597             if type(b) is tuple:
 598                 raise ValueError("builder %s must be defined with a dict, "
 599                                  "not a tuple" % b[0])
 600             if b.has_key('slavename') and b['slavename'] not in slavenames:
 601                 raise ValueError("builder %s uses undefined slave %s" \
 602                                  % (b['name'], b['slavename']))
 603             for n in b.get('slavenames', []):
 604                 if n not in slavenames:
 605                     raise ValueError("builder %s uses undefined slave %s" \
 606                                      % (b['name'], n))
 607             if b['name'] in buildernames:
 608                 raise ValueError("duplicate builder name %s"
 609                                  % b['name'])
 610             buildernames.append(b['name'])
 611             if b['builddir'] in dirnames:
 612                 raise ValueError("builder %s reuses builddir %s"
 613                                  % (b['name'], b['builddir']))
 614             dirnames.append(b['builddir'])
 615
 616         unscheduled_buildernames = buildernames[:]
 617         schedulernames = []
 618         for s in schedulers:
 619             for b in s.listBuilderNames():
 620                 assert b in buildernames, \
 621                        "%s uses unknown builder %s" % (s, b)
 622                 if b in unscheduled_buildernames:
 623                     unscheduled_buildernames.remove(b)
 624
 625             if s.name in schedulernames:
 626                 # TODO: schedulers share a namespace with other Service
 627                 # children of the BuildMaster node, like status plugins, the
 628                 # Manhole, the ChangeMaster, and the BotMaster (although most
 629                 # of these don't have names)
 630                 msg = ("Schedulers must have unique names, but "
 631                        "'%s' was a duplicate" % (s.name,))
 632                 raise ValueError(msg)
 633             schedulernames.append(s.name)
 634
 635         if unscheduled_buildernames:
 636             log.msg("Warning: some Builders have no Schedulers to drive them:"
 637                     " %s" % (unscheduled_buildernames,))
 638
 639         # assert that all locks used by the Builds and their Steps are
 640         # uniquely named.
 641         lock_dict = {}
 642         for b in builders:
 643             for l in b.get('locks', []):
 644                 if isinstance(l, locks.LockAccess): # User specified access to the lock
 645                     l = l.lockid
 646                 if lock_dict.has_key(l.name):
 647                     if lock_dict[l.name] is not l:
 648                         raise ValueError("Two different locks (%s and %s) "
 649                                          "share the name %s"
 650                                          % (l, lock_dict[l.name], l.name))
 651                 else:
 652                     lock_dict[l.name] = l
 653             # TODO: this will break with any BuildFactory that doesn't use a
 654             # .steps list, but I think the verification step is more
 655             # important.
 656             for s in b['factory'].steps:
 657                 for l in s[1].get('locks', []):
 658                     if isinstance(l, locks.LockAccess): # User specified access to the lock
 659                         l = l.lockid
 660                     if lock_dict.has_key(l.name):
 661                         if lock_dict[l.name] is not l:
 662                             raise ValueError("Two different locks (%s and %s)"
 663                                              " share the name %s"
 664                                              % (l, lock_dict[l.name], l.name))
 665                     else:
 666                         lock_dict[l.name] = l
 667
 668         if not isinstance(properties, dict):
 669             raise ValueError("c['properties'] must be a dictionary")
 670
 671         # slavePortnum supposed to be a strports specification
 672         if type(slavePortnum) is int:
 673             slavePortnum = "tcp:%d" % slavePortnum
 674
 675         # now we're committed to implementing the new configuration, so do
 676         # it atomically
 677         # TODO: actually, this is spread across a couple of Deferreds, so it
 678         # really isn't atomic.
 679
 680         d = defer.succeed(None)
 681
 682         self.projectName = projectName
 683         self.projectURL = projectURL
 684         self.buildbotURL = buildbotURL
 685
 686         self.properties = Properties()
 687         self.properties.update(properties, self.configFileName)
 688
 689         # self.slaves: Disconnect any that were attached and removed from the
 690         # list. Update self.checker with the new list of passwords, including
 691         # debug/change/status.
 692         d.addCallback(lambda res: self.loadConfig_Slaves(slaves))
 693
 694         # self.debugPassword
 695         if debugPassword:
 696             self.checker.addUser("debug", debugPassword)
 697             self.debugPassword = debugPassword
 698
 699         # self.manhole
 700         if manhole != self.manhole:
 701             # changing
 702             if self.manhole:
 703                 # disownServiceParent may return a Deferred
 704                 d.addCallback(lambda res: self.manhole.disownServiceParent())
 705                 def _remove(res):
 706                     self.manhole = None
 707                     return res
 708                 d.addCallback(_remove)
 709             if manhole:
 710                 def _add(res):
 711                     self.manhole = manhole
 712                     manhole.setServiceParent(self)
 713                 d.addCallback(_add)
 714
 715         # add/remove self.botmaster.builders to match builders. The
 716         # botmaster will handle startup/shutdown issues.
 717         d.addCallback(lambda res: self.loadConfig_Builders(builders))
 718
 719         d.addCallback(lambda res: self.loadConfig_status(status))
 720
 721         # Schedulers are added after Builders in case they start right away
 722         d.addCallback(lambda res: self.loadConfig_Schedulers(schedulers))
 723         # and Sources go after Schedulers for the same reason
 724         d.addCallback(lambda res: self.loadConfig_Sources(change_sources))
 725
 726         # self.slavePort
 727         if self.slavePortnum != slavePortnum:
 728             if self.slavePort:
 729                 def closeSlavePort(res):
 730                     d1 = self.slavePort.disownServiceParent()
 731                     self.slavePort = None
 732                     return d1
 733                 d.addCallback(closeSlavePort)
 734             if slavePortnum is not None:
 735                 def openSlavePort(res):
 736                     self.slavePort = strports.service(slavePortnum,
 737                                                       self.slaveFactory)
 738                     self.slavePort.setServiceParent(self)
 739                 d.addCallback(openSlavePort)
 740                 log.msg("BuildMaster listening on port %s" % slavePortnum)
 741             self.slavePortnum = slavePortnum
 742
 743         log.msg("configuration update started")
 744         def _done(res):
 745             self.readConfig = True
 746             log.msg("configuration update complete")
 747         d.addCallback(_done)
 748         d.addCallback(lambda res: self.botmaster.maybeStartAllBuilds())
 749         return d
 750
 751     def loadConfig_Slaves(self, new_slaves):
 752         # set up the Checker with the names and passwords of all valid bots
 753         self.checker.users = {} # violates abstraction, oh well
 754         for s in new_slaves:
 755             self.checker.addUser(s.slavename, s.password)
 756         self.checker.addUser("change", "changepw")
 757         # let the BotMaster take care of the rest
 758         return self.botmaster.loadConfig_Slaves(new_slaves)
 759
 760     def loadConfig_Sources(self, sources):
 761         if not sources:
 762             log.msg("warning: no ChangeSources specified in c['change_source']")
 763         # shut down any that were removed, start any that were added
 764         deleted_sources = [s for s in self.change_svc if s not in sources]
 765         added_sources = [s for s in sources if s not in self.change_svc]
 766         dl = [self.change_svc.removeSource(s) for s in deleted_sources]
 767         def addNewOnes(res):
 768             [self.change_svc.addSource(s) for s in added_sources]
 769         d = defer.DeferredList(dl, fireOnOneErrback=1, consumeErrors=0)
 770         d.addCallback(addNewOnes)
 771         return d
 772
 773     def allSchedulers(self):
 774         return [child for child in self
 775                 if interfaces.IScheduler.providedBy(child)]
 776
 777
 778     def loadConfig_Schedulers(self, newschedulers):
 779         oldschedulers = self.allSchedulers()
 780         removed = [s for s in oldschedulers if s not in newschedulers]
 781         added = [s for s in newschedulers if s not in oldschedulers]
 782         dl = [defer.maybeDeferred(s.disownServiceParent) for s in removed]
 783         def addNewOnes(res):
 784             for s in added:
 785                 s.setServiceParent(self)
 786         d = defer.DeferredList(dl, fireOnOneErrback=1)
 787         d.addCallback(addNewOnes)
 788         return d
 789
 790     def loadConfig_Builders(self, newBuilderData):
 791         somethingChanged = False
 792         newList = {}
 793         newBuilderNames = []
 794         allBuilders = self.botmaster.builders.copy()
 795         for data in newBuilderData:
 796             name = data['name']
 797             newList[name] = data
 798             newBuilderNames.append(name)
 799
 800         # identify all that were removed
 801         for oldname in self.botmaster.getBuildernames():
 802             if oldname not in newList:
 803                 log.msg("removing old builder %s" % oldname)
 804                 del allBuilders[oldname]
 805                 somethingChanged = True
 806                 # announce the change
 807                 self.status.builderRemoved(oldname)
 808
 809         # everything in newList is either unchanged, changed, or new
 810         for name, data in newList.items():
 811             old = self.botmaster.builders.get(name)
 812             basedir = data['builddir'] # used on both master and slave
 813             #name, slave, builddir, factory = data
 814             if not old: # new
 815                 # category added after 0.6.2
 816                 category = data.get('category', None)
 817                 log.msg("adding new builder %s for category %s" %
 818                         (name, category))
 819                 statusbag = self.status.builderAdded(name, basedir, category)
 820                 builder = Builder(data, statusbag)
 821                 allBuilders[name] = builder
 822                 somethingChanged = True
 823             elif old.compareToSetup(data):
 824                 # changed: try to minimize the disruption and only modify the
 825                 # pieces that really changed
 826                 diffs = old.compareToSetup(data)
 827                 log.msg("updating builder %s: %s" % (name, "\n".join(diffs)))
 828
 829                 statusbag = old.builder_status
 830                 statusbag.saveYourself() # seems like a good idea
 831                 # TODO: if the basedir was changed, we probably need to make
 832                 # a new statusbag
 833                 new_builder = Builder(data, statusbag)
 834                 new_builder.consumeTheSoulOfYourPredecessor(old)
 835                 # that migrates any retained slavebuilders too
 836
 837                 # point out that the builder was updated. On the Waterfall,
 838                 # this will appear just after any currently-running builds.
 839                 statusbag.addPointEvent(["config", "updated"])
 840
 841                 allBuilders[name] = new_builder
 842                 somethingChanged = True
 843             else:
 844                 # unchanged: leave it alone
 845                 log.msg("builder %s is unchanged" % name)
 846                 pass
 847
 848         if somethingChanged:
 849             sortedAllBuilders = [allBuilders[name] for name in newBuilderNames]
 850             d = self.botmaster.setBuilders(sortedAllBuilders)
 851             return d
 852         return None
 853
 854     def loadConfig_status(self, status):
 855         dl = []
 856
 857         # remove old ones
 858         for s in self.statusTargets[:]:
 859             if not s in status:
 860                 log.msg("removing IStatusReceiver", s)
 861                 d = defer.maybeDeferred(s.disownServiceParent)
 862                 dl.append(d)
 863                 self.statusTargets.remove(s)
 864         # after those are finished going away, add new ones
 865         def addNewOnes(res):
 866             for s in status:
 867                 if not s in self.statusTargets:
 868                     log.msg("adding IStatusReceiver", s)
 869                     s.setServiceParent(self)
 870                     self.statusTargets.append(s)
 871         d = defer.DeferredList(dl, fireOnOneErrback=1)
 872         d.addCallback(addNewOnes)
 873         return d
 874
 875
 876     def addChange(self, change):
 877         for s in self.allSchedulers():
 878             s.addChange(change)
 879
 880     def submitBuildSet(self, bs):
 881         # determine the set of Builders to use
 882         builders = []
 883         for name in bs.builderNames:
 884             b = self.botmaster.builders.get(name)
 885             if b:
 886                 if b not in builders:
 887                     builders.append(b)
 888                 continue
 889             # TODO: add aliases like 'all'
 890             raise KeyError("no such builder named '%s'" % name)
 891
 892         # now tell the BuildSet to create BuildRequests for all those
 893         # Builders and submit them
 894         bs.start(builders)
 895         self.status.buildsetSubmitted(bs.status)
 896
 897
 898 class Control:
 899     implements(interfaces.IControl)
 900
 901     def __init__(self, master):
 902         self.master = master
 903
 904     def addChange(self, change):
 905         self.master.change_svc.addChange(change)
 906
 907     def submitBuildSet(self, bs):
 908         self.master.submitBuildSet(bs)
 909
 910     def getBuilder(self, name):
 911         b = self.master.botmaster.builders[name]
 912         return interfaces.IBuilderControl(b)
 913
 914 components.registerAdapter(Control, BuildMaster, interfaces.IControl)
 915
 916 # so anybody who can get a handle on the BuildMaster can cause a build with:
 917 #  IControl(master).getBuilder("full-2.3").requestBuild(buildrequest)
 918