buildbot/master.py

   1 # -*- test-case-name: buildbot.test.test_run -*-
   2
   3 import os
   4 signal = None
   5 try:
   6     import signal
   7 except ImportError:
   8     pass
   9 from cPickle import load
  10 import warnings
  11
  12 from zope.interface import implements
  13 from twisted.python import log, components
  14 from twisted.internet import defer, reactor
  15 from twisted.spread import pb
  16 from twisted.cred import portal, checkers
  17 from twisted.application import service, strports
  18 from twisted.persisted import styles
  19
  20 import buildbot
  21 # sibling imports
  22 from buildbot.util import now
  23 from buildbot.pbutil import NewCredPerspective
  24 from buildbot.process.builder import Builder, IDLE
  25 from buildbot.process.base import BuildRequest
  26 from buildbot.status.builder import Status
  27 from buildbot.changes.changes import Change, ChangeMaster
  28 from buildbot.sourcestamp import SourceStamp
  29 from buildbot.buildslave import BuildSlave
  30 from buildbot import interfaces
  31 from buildbot.process.properties import Properties
  32
  33 ########################################
  34
  35 class BotMaster(service.MultiService):
  36
  37     """This is the master-side service which manages remote buildbot slaves.
  38     It provides them with BuildSlaves, and distributes file change
  39     notification messages to them.
  40     """
  41
  42     debug = 0
  43
  44     def __init__(self):
  45         service.MultiService.__init__(self)
  46         self.builders = {}
  47         self.builderNames = []
  48         # builders maps Builder names to instances of bb.p.builder.Builder,
  49         # which is the master-side object that defines and controls a build.
  50         # They are added by calling botmaster.addBuilder() from the startup
  51         # code.
  52
  53         # self.slaves contains a ready BuildSlave instance for each
  54         # potential buildslave, i.e. all the ones listed in the config file.
  55         # If the slave is connected, self.slaves[slavename].slave will
  56         # contain a RemoteReference to their Bot instance. If it is not
  57         # connected, that attribute will hold None.
  58         self.slaves = {} # maps slavename to BuildSlave
  59         self.statusClientService = None
  60         self.watchers = {}
  61
  62         # self.locks holds the real Lock instances
  63         self.locks = {}
  64
  65     # these four are convenience functions for testing
  66
  67     def waitUntilBuilderAttached(self, name):
  68         b = self.builders[name]
  69         #if b.slaves:
  70         #    return defer.succeed(None)
  71         d = defer.Deferred()
  72         b.watchers['attach'].append(d)
  73         return d
  74
  75     def waitUntilBuilderDetached(self, name):
  76         b = self.builders.get(name)
  77         if not b or not b.slaves:
  78             return defer.succeed(None)
  79         d = defer.Deferred()
  80         b.watchers['detach'].append(d)
  81         return d
  82
  83     def waitUntilBuilderFullyDetached(self, name):
  84         b = self.builders.get(name)
  85         # TODO: this looks too deeply inside the Builder object
  86         if not b or not b.slaves:
  87             return defer.succeed(None)
  88         d = defer.Deferred()
  89         b.watchers['detach_all'].append(d)
  90         return d
  91
  92     def waitUntilBuilderIdle(self, name):
  93         b = self.builders[name]
  94         # TODO: this looks way too deeply inside the Builder object
  95         for sb in b.slaves:
  96             if sb.state != IDLE:
  97                 d = defer.Deferred()
  98                 b.watchers['idle'].append(d)
  99                 return d
 100         return defer.succeed(None)
 101
 102     def loadConfig_Slaves(self, new_slaves):
 103         old_slaves = [c for c in list(self)
 104                       if interfaces.IBuildSlave.providedBy(c)]
 105
 106         # identify added/removed slaves. For each slave we construct a tuple
 107         # of (name, password, class), and we consider the slave to be already
 108         # present if the tuples match. (we include the class to make sure
 109         # that BuildSlave(name,pw) is different than
 110         # SubclassOfBuildSlave(name,pw) ). If the password or class has
 111         # changed, we will remove the old version of the slave and replace it
 112         # with a new one. If anything else has changed, we just update the
 113         # old BuildSlave instance in place. If the name has changed, of
 114         # course, it looks exactly the same as deleting one slave and adding
 115         # an unrelated one.
 116         old_t = {}
 117         for s in old_slaves:
 118             old_t[(s.slavename, s.password, s.__class__)] = s
 119         new_t = {}
 120         for s in new_slaves:
 121             new_t[(s.slavename, s.password, s.__class__)] = s
 122         removed = [old_t[t]
 123                    for t in old_t
 124                    if t not in new_t]
 125         added = [new_t[t]
 126                  for t in new_t
 127                  if t not in old_t]
 128         remaining_t = [t
 129                        for t in new_t
 130                        if t in old_t]
 131         # removeSlave will hang up on the old bot
 132         dl = []
 133         for s in removed:
 134             dl.append(self.removeSlave(s))
 135         d = defer.DeferredList(dl, fireOnOneErrback=True)
 136         def _add(res):
 137             for s in added:
 138                 self.addSlave(s)
 139             for t in remaining_t:
 140                 old_t[t].update(new_t[t])
 141         d.addCallback(_add)
 142         return d
 143
 144     def addSlave(self, s):
 145         s.setServiceParent(self)
 146         s.setBotmaster(self)
 147         self.slaves[s.slavename] = s
 148
 149     def removeSlave(self, s):
 150         # TODO: technically, disownServiceParent could return a Deferred
 151         s.disownServiceParent()
 152         d = self.slaves[s.slavename].disconnect()
 153         del self.slaves[s.slavename]
 154         return d
 155
 156     def slaveLost(self, bot):
 157         for name, b in self.builders.items():
 158             if bot.slavename in b.slavenames:
 159                 b.detached(bot)
 160
 161     def getBuildersForSlave(self, slavename):
 162         return [b
 163                 for b in self.builders.values()
 164                 if slavename in b.slavenames]
 165
 166     def getBuildernames(self):
 167         return self.builderNames
 168
 169     def getBuilders(self):
 170         allBuilders = [self.builders[name] for name in self.builderNames]
 171         return allBuilders
 172
 173     def setBuilders(self, builders):
 174         self.builders = {}
 175         self.builderNames = []
 176         for b in builders:
 177             for slavename in b.slavenames:
 178                 # this is actually validated earlier
 179                 assert slavename in self.slaves
 180             self.builders[b.name] = b
 181             self.builderNames.append(b.name)
 182             b.setBotmaster(self)
 183         d = self._updateAllSlaves()
 184         return d
 185
 186     def _updateAllSlaves(self):
 187         """Notify all buildslaves about changes in their Builders."""
 188         dl = [s.updateSlave() for s in self.slaves.values()]
 189         return defer.DeferredList(dl)
 190
 191     def maybeStartAllBuilds(self):
 192         for b in self.builders.values():
 193             b.maybeStartBuild()
 194
 195     def getPerspective(self, slavename):
 196         return self.slaves[slavename]
 197
 198     def shutdownSlaves(self):
 199         # TODO: make this into a bot method rather than a builder method
 200         for b in self.slaves.values():
 201             b.shutdownSlave()
 202
 203     def stopService(self):
 204         for b in self.builders.values():
 205             b.builder_status.addPointEvent(["master", "shutdown"])
 206             b.builder_status.saveYourself()
 207         return service.Service.stopService(self)
 208
 209     def getLockByID(self, lockid):
 210         """Convert a Lock identifier into an actual Lock instance.
 211         @param lockid: a locks.MasterLock or locks.SlaveLock instance
 212         @return: a locks.RealMasterLock or locks.RealSlaveLock instance
 213         """
 214         if not lockid in self.locks:
 215             self.locks[lockid] = lockid.lockClass(lockid)
 216         # if the master.cfg file has changed maxCount= on the lock, the next
 217         # time a build is started, they'll get a new RealLock instance. Note
 218         # that this requires that MasterLock and SlaveLock (marker) instances
 219         # be hashable and that they should compare properly.
 220         return self.locks[lockid]
 221
 222 ########################################
 223
 224
 225
 226 class DebugPerspective(NewCredPerspective):
 227     def attached(self, mind):
 228         return self
 229     def detached(self, mind):
 230         pass
 231
 232     def perspective_requestBuild(self, buildername, reason, branch, revision, properties={}):
 233         c = interfaces.IControl(self.master)
 234         bc = c.getBuilder(buildername)
 235         ss = SourceStamp(branch, revision)
 236         bpr = Properties()
 237         bpr.update(properties, "remote requestBuild")
 238         br = BuildRequest(reason, ss, builderName=buildername, properties=bpr)
 239         bc.requestBuild(br)
 240
 241     def perspective_pingBuilder(self, buildername):
 242         c = interfaces.IControl(self.master)
 243         bc = c.getBuilder(buildername)
 244         bc.ping()
 245
 246     def perspective_fakeChange(self, file, revision=None, who="fakeUser",
 247                                branch=None):
 248         change = Change(who, [file], "some fake comments\n",
 249                         branch=branch, revision=revision)
 250         c = interfaces.IControl(self.master)
 251         c.addChange(change)
 252
 253     def perspective_setCurrentState(self, buildername, state):
 254         builder = self.botmaster.builders.get(buildername)
 255         if not builder: return
 256         if state == "offline":
 257             builder.statusbag.currentlyOffline()
 258         if state == "idle":
 259             builder.statusbag.currentlyIdle()
 260         if state == "waiting":
 261             builder.statusbag.currentlyWaiting(now()+10)
 262         if state == "building":
 263             builder.statusbag.currentlyBuilding(None)
 264     def perspective_reload(self):
 265         print "doing reload of the config file"
 266         self.master.loadTheConfigFile()
 267     def perspective_pokeIRC(self):
 268         print "saying something on IRC"
 269         from buildbot.status import words
 270         for s in self.master:
 271             if isinstance(s, words.IRC):
 272                 bot = s.f
 273                 for channel in bot.channels:
 274                     print " channel", channel
 275                     bot.p.msg(channel, "Ow, quit it")
 276
 277     def perspective_print(self, msg):
 278         print "debug", msg
 279
 280 class Dispatcher(styles.Versioned):
 281     implements(portal.IRealm)
 282     persistenceVersion = 2
 283
 284     def __init__(self):
 285         self.names = {}
 286
 287     def upgradeToVersion1(self):
 288         self.master = self.botmaster.parent
 289     def upgradeToVersion2(self):
 290         self.names = {}
 291
 292     def register(self, name, afactory):
 293         self.names[name] = afactory
 294     def unregister(self, name):
 295         del self.names[name]
 296
 297     def requestAvatar(self, avatarID, mind, interface):
 298         assert interface == pb.IPerspective
 299         afactory = self.names.get(avatarID)
 300         if afactory:
 301             p = afactory.getPerspective()
 302         elif avatarID == "debug":
 303             p = DebugPerspective()
 304             p.master = self.master
 305             p.botmaster = self.botmaster
 306         elif avatarID == "statusClient":
 307             p = self.statusClientService.getPerspective()
 308         else:
 309             # it must be one of the buildslaves: no other names will make it
 310             # past the checker
 311             p = self.botmaster.getPerspective(avatarID)
 312
 313         if not p:
 314             raise ValueError("no perspective for '%s'" % avatarID)
 315
 316         d = defer.maybeDeferred(p.attached, mind)
 317         d.addCallback(self._avatarAttached, mind)
 318         return d
 319
 320     def _avatarAttached(self, p, mind):
 321         return (pb.IPerspective, p, lambda p=p,mind=mind: p.detached(mind))
 322
 323 ########################################
 324
 325 # service hierarchy:
 326 #  BuildMaster
 327 #   BotMaster
 328 #   ChangeMaster
 329 #    all IChangeSource objects
 330 #   StatusClientService
 331 #   TCPClient(self.ircFactory)
 332 #   TCPServer(self.slaveFactory) -> dispatcher.requestAvatar
 333 #   TCPServer(self.site)
 334 #   UNIXServer(ResourcePublisher(self.site))
 335
 336
 337 class BuildMaster(service.MultiService, styles.Versioned):
 338     debug = 0
 339     persistenceVersion = 3
 340     manhole = None
 341     debugPassword = None
 342     projectName = "(unspecified)"
 343     projectURL = None
 344     buildbotURL = None
 345     change_svc = None
 346     properties = Properties()
 347
 348     def __init__(self, basedir, configFileName="master.cfg"):
 349         service.MultiService.__init__(self)
 350         self.setName("buildmaster")
 351         self.basedir = basedir
 352         self.configFileName = configFileName
 353
 354         # the dispatcher is the realm in which all inbound connections are
 355         # looked up: slave builders, change notifications, status clients, and
 356         # the debug port
 357         dispatcher = Dispatcher()
 358         dispatcher.master = self
 359         self.dispatcher = dispatcher
 360         self.checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
 361         # the checker starts with no user/passwd pairs: they are added later
 362         p = portal.Portal(dispatcher)
 363         p.registerChecker(self.checker)
 364         self.slaveFactory = pb.PBServerFactory(p)
 365         self.slaveFactory.unsafeTracebacks = True # let them see exceptions
 366
 367         self.slavePortnum = None
 368         self.slavePort = None
 369
 370         self.botmaster = BotMaster()
 371         self.botmaster.setName("botmaster")
 372         self.botmaster.setServiceParent(self)
 373         dispatcher.botmaster = self.botmaster
 374
 375         self.status = Status(self.botmaster, self.basedir)
 376
 377         self.statusTargets = []
 378
 379         # this ChangeMaster is a dummy, only used by tests. In the real
 380         # buildmaster, where the BuildMaster instance is activated
 381         # (startService is called) by twistd, this attribute is overwritten.
 382         self.useChanges(ChangeMaster())
 383
 384         self.readConfig = False
 385
 386     def upgradeToVersion1(self):
 387         self.dispatcher = self.slaveFactory.root.portal.realm
 388
 389     def upgradeToVersion2(self): # post-0.4.3
 390         self.webServer = self.webTCPPort
 391         del self.webTCPPort
 392         self.webDistribServer = self.webUNIXPort
 393         del self.webUNIXPort
 394         self.configFileName = "master.cfg"
 395
 396     def upgradeToVersion3(self):
 397         # post 0.6.3, solely to deal with the 0.6.3 breakage. Starting with
 398         # 0.6.5 I intend to do away with .tap files altogether
 399         self.services = []
 400         self.namedServices = {}
 401         del self.change_svc
 402
 403     def startService(self):
 404         service.MultiService.startService(self)
 405         self.loadChanges() # must be done before loading the config file
 406         if not self.readConfig:
 407             # TODO: consider catching exceptions during this call to
 408             # loadTheConfigFile and bailing (reactor.stop) if it fails,
 409             # since without a config file we can't do anything except reload
 410             # the config file, and it would be nice for the user to discover
 411             # this quickly.
 412             self.loadTheConfigFile()
 413         if signal and hasattr(signal, "SIGHUP"):
 414             signal.signal(signal.SIGHUP, self._handleSIGHUP)
 415         for b in self.botmaster.builders.values():
 416             b.builder_status.addPointEvent(["master", "started"])
 417             b.builder_status.saveYourself()
 418
 419     def useChanges(self, changes):
 420         if self.change_svc:
 421             # TODO: can return a Deferred
 422             self.change_svc.disownServiceParent()
 423         self.change_svc = changes
 424         self.change_svc.basedir = self.basedir
 425         self.change_svc.setName("changemaster")
 426         self.dispatcher.changemaster = self.change_svc
 427         self.change_svc.setServiceParent(self)
 428
 429     def loadChanges(self):
 430         filename = os.path.join(self.basedir, "changes.pck")
 431         try:
 432             changes = load(open(filename, "rb"))
 433             styles.doUpgrade()
 434         except IOError:
 435             log.msg("changes.pck missing, using new one")
 436             changes = ChangeMaster()
 437         except EOFError:
 438             log.msg("corrupted changes.pck, using new one")
 439             changes = ChangeMaster()
 440         self.useChanges(changes)
 441
 442     def _handleSIGHUP(self, *args):
 443         reactor.callLater(0, self.loadTheConfigFile)
 444
 445     def getStatus(self):
 446         """
 447         @rtype: L{buildbot.status.builder.Status}
 448         """
 449         return self.status
 450
 451     def loadTheConfigFile(self, configFile=None):
 452         if not configFile:
 453             configFile = os.path.join(self.basedir, self.configFileName)
 454
 455         log.msg("Creating BuildMaster -- buildbot.version: %s" % buildbot.version)
 456         log.msg("loading configuration from %s" % configFile)
 457         configFile = os.path.expanduser(configFile)
 458
 459         try:
 460             f = open(configFile, "r")
 461         except IOError, e:
 462             log.msg("unable to open config file '%s'" % configFile)
 463             log.msg("leaving old configuration in place")
 464             log.err(e)
 465             return
 466
 467         try:
 468             self.loadConfig(f)
 469         except:
 470             log.msg("error during loadConfig")
 471             log.err()
 472             log.msg("The new config file is unusable, so I'll ignore it.")
 473             log.msg("I will keep using the previous config file instead.")
 474         f.close()
 475
 476     def loadConfig(self, f):
 477         """Internal function to load a specific configuration file. Any
 478         errors in the file will be signalled by raising an exception.
 479
 480         @return: a Deferred that will fire (with None) when the configuration
 481         changes have been completed. This may involve a round-trip to each
 482         buildslave that was involved."""
 483
 484         localDict = {'basedir': os.path.expanduser(self.basedir)}
 485         try:
 486             exec f in localDict
 487         except:
 488             log.msg("error while parsing config file")
 489             raise
 490
 491         try:
 492             config = localDict['BuildmasterConfig']
 493         except KeyError:
 494             log.err("missing config dictionary")
 495             log.err("config file must define BuildmasterConfig")
 496             raise
 497
 498         known_keys = ("bots", "slaves",
 499                       "sources", "change_source",
 500                       "schedulers", "builders",
 501                       "slavePortnum", "debugPassword", "manhole",
 502                       "status", "projectName", "projectURL", "buildbotURL",
 503                       "properties"
 504                       )
 505         for k in config.keys():
 506             if k not in known_keys:
 507                 log.msg("unknown key '%s' defined in config dictionary" % k)
 508
 509         try:
 510             # required
 511             schedulers = config['schedulers']
 512             builders = config['builders']
 513             for k in builders:
 514                 if k['name'].startswith("_"):
 515                     errmsg = ("builder names must not start with an "
 516                               "underscore: " + k['name'])
 517                     log.err(errmsg)
 518                     raise ValueError(errmsg)
 519
 520             slavePortnum = config['slavePortnum']
 521             #slaves = config['slaves']
 522             #change_source = config['change_source']
 523
 524             # optional
 525             debugPassword = config.get('debugPassword')
 526             manhole = config.get('manhole')
 527             status = config.get('status', [])
 528             projectName = config.get('projectName')
 529             projectURL = config.get('projectURL')
 530             buildbotURL = config.get('buildbotURL')
 531             properties = config.get('properties', {})
 532
 533         except KeyError, e:
 534             log.msg("config dictionary is missing a required parameter")
 535             log.msg("leaving old configuration in place")
 536             raise
 537
 538         #if "bots" in config:
 539         #    raise KeyError("c['bots'] is no longer accepted")
 540
 541         slaves = config.get('slaves', [])
 542         if "bots" in config:
 543             m = ("c['bots'] is deprecated as of 0.7.6 and will be "
 544                  "removed by 0.8.0 . Please use c['slaves'] instead.")
 545             log.msg(m)
 546             warnings.warn(m, DeprecationWarning)
 547             for name, passwd in config['bots']:
 548                 slaves.append(BuildSlave(name, passwd))
 549
 550         if "bots" not in config and "slaves" not in config:
 551             log.msg("config dictionary must have either 'bots' or 'slaves'")
 552             log.msg("leaving old configuration in place")
 553             raise KeyError("must have either 'bots' or 'slaves'")
 554
 555         #if "sources" in config:
 556         #    raise KeyError("c['sources'] is no longer accepted")
 557
 558         change_source = config.get('change_source', [])
 559         if isinstance(change_source, (list, tuple)):
 560             change_sources = change_source
 561         else:
 562             change_sources = [change_source]
 563         if "sources" in config:
 564             m = ("c['sources'] is deprecated as of 0.7.6 and will be "
 565                  "removed by 0.8.0 . Please use c['change_source'] instead.")
 566             log.msg(m)
 567             warnings.warn(m, DeprecationWarning)
 568             for s in config['sources']:
 569                 change_sources.append(s)
 570
 571         # do some validation first
 572         for s in slaves:
 573             assert isinstance(s, BuildSlave)
 574             if s.slavename in ("debug", "change", "status"):
 575                 raise KeyError, "reserved name '%s' used for a bot" % s.slavename
 576         if config.has_key('interlocks'):
 577             raise KeyError("c['interlocks'] is no longer accepted")
 578
 579         assert isinstance(change_sources, (list, tuple))
 580         for s in change_sources:
 581             assert interfaces.IChangeSource(s, None)
 582         # this assertion catches c['schedulers'] = Scheduler(), since
 583         # Schedulers are service.MultiServices and thus iterable.
 584         errmsg = "c['schedulers'] must be a list of Scheduler instances"
 585         assert isinstance(schedulers, (list, tuple)), errmsg
 586         for s in schedulers:
 587             assert interfaces.IScheduler(s, None), errmsg
 588         assert isinstance(status, (list, tuple))
 589         for s in status:
 590             assert interfaces.IStatusReceiver(s, None)
 591
 592         slavenames = [s.slavename for s in slaves]
 593         buildernames = []
 594         dirnames = []
 595         for b in builders:
 596             if type(b) is tuple:
 597                 raise ValueError("builder %s must be defined with a dict, "
 598                                  "not a tuple" % b[0])
 599             if b.has_key('slavename') and b['slavename'] not in slavenames:
 600                 raise ValueError("builder %s uses undefined slave %s" \
 601                                  % (b['name'], b['slavename']))
 602             for n in b.get('slavenames', []):
 603                 if n not in slavenames:
 604                     raise ValueError("builder %s uses undefined slave %s" \
 605                                      % (b['name'], n))
 606             if b['name'] in buildernames:
 607                 raise ValueError("duplicate builder name %s"
 608                                  % b['name'])
 609             buildernames.append(b['name'])
 610             if b['builddir'] in dirnames:
 611                 raise ValueError("builder %s reuses builddir %s"
 612                                  % (b['name'], b['builddir']))
 613             dirnames.append(b['builddir'])
 614
 615         unscheduled_buildernames = buildernames[:]
 616         schedulernames = []
 617         for s in schedulers:
 618             for b in s.listBuilderNames():
 619                 assert b in buildernames, \
 620                        "%s uses unknown builder %s" % (s, b)
 621                 if b in unscheduled_buildernames:
 622                     unscheduled_buildernames.remove(b)
 623
 624             if s.name in schedulernames:
 625                 # TODO: schedulers share a namespace with other Service
 626                 # children of the BuildMaster node, like status plugins, the
 627                 # Manhole, the ChangeMaster, and the BotMaster (although most
 628                 # of these don't have names)
 629                 msg = ("Schedulers must have unique names, but "
 630                        "'%s' was a duplicate" % (s.name,))
 631                 raise ValueError(msg)
 632             schedulernames.append(s.name)
 633
 634         if unscheduled_buildernames:
 635             log.msg("Warning: some Builders have no Schedulers to drive them:"
 636                     " %s" % (unscheduled_buildernames,))
 637
 638         # assert that all locks used by the Builds and their Steps are
 639         # uniquely named.
 640         locks = {}
 641         for b in builders:
 642             for l in b.get('locks', []):
 643                 if locks.has_key(l.name):
 644                     if locks[l.name] is not l:
 645                         raise ValueError("Two different locks (%s and %s) "
 646                                          "share the name %s"
 647                                          % (l, locks[l.name], l.name))
 648                 else:
 649                     locks[l.name] = l
 650             # TODO: this will break with any BuildFactory that doesn't use a
 651             # .steps list, but I think the verification step is more
 652             # important.
 653             for s in b['factory'].steps:
 654                 for l in s[1].get('locks', []):
 655                     if locks.has_key(l.name):
 656                         if locks[l.name] is not l:
 657                             raise ValueError("Two different locks (%s and %s)"
 658                                              " share the name %s"
 659                                              % (l, locks[l.name], l.name))
 660                     else:
 661                         locks[l.name] = l
 662
 663         if not isinstance(properties, dict):
 664             raise ValueError("c['properties'] must be a dictionary")
 665
 666         # slavePortnum supposed to be a strports specification
 667         if type(slavePortnum) is int:
 668             slavePortnum = "tcp:%d" % slavePortnum
 669
 670         # now we're committed to implementing the new configuration, so do
 671         # it atomically
 672         # TODO: actually, this is spread across a couple of Deferreds, so it
 673         # really isn't atomic.
 674
 675         d = defer.succeed(None)
 676
 677         self.projectName = projectName
 678         self.projectURL = projectURL
 679         self.buildbotURL = buildbotURL
 680
 681         self.properties = Properties()
 682         self.properties.update(properties, self.configFileName)
 683
 684         # self.slaves: Disconnect any that were attached and removed from the
 685         # list. Update self.checker with the new list of passwords, including
 686         # debug/change/status.
 687         d.addCallback(lambda res: self.loadConfig_Slaves(slaves))
 688
 689         # self.debugPassword
 690         if debugPassword:
 691             self.checker.addUser("debug", debugPassword)
 692             self.debugPassword = debugPassword
 693
 694         # self.manhole
 695         if manhole != self.manhole:
 696             # changing
 697             if self.manhole:
 698                 # disownServiceParent may return a Deferred
 699                 d.addCallback(lambda res: self.manhole.disownServiceParent())
 700                 def _remove(res):
 701                     self.manhole = None
 702                     return res
 703                 d.addCallback(_remove)
 704             if manhole:
 705                 def _add(res):
 706                     self.manhole = manhole
 707                     manhole.setServiceParent(self)
 708                 d.addCallback(_add)
 709
 710         # add/remove self.botmaster.builders to match builders. The
 711         # botmaster will handle startup/shutdown issues.
 712         d.addCallback(lambda res: self.loadConfig_Builders(builders))
 713
 714         d.addCallback(lambda res: self.loadConfig_status(status))
 715
 716         # Schedulers are added after Builders in case they start right away
 717         d.addCallback(lambda res: self.loadConfig_Schedulers(schedulers))
 718         # and Sources go after Schedulers for the same reason
 719         d.addCallback(lambda res: self.loadConfig_Sources(change_sources))
 720
 721         # self.slavePort
 722         if self.slavePortnum != slavePortnum:
 723             if self.slavePort:
 724                 def closeSlavePort(res):
 725                     d1 = self.slavePort.disownServiceParent()
 726                     self.slavePort = None
 727                     return d1
 728                 d.addCallback(closeSlavePort)
 729             if slavePortnum is not None:
 730                 def openSlavePort(res):
 731                     self.slavePort = strports.service(slavePortnum,
 732                                                       self.slaveFactory)
 733                     self.slavePort.setServiceParent(self)
 734                 d.addCallback(openSlavePort)
 735                 log.msg("BuildMaster listening on port %s" % slavePortnum)
 736             self.slavePortnum = slavePortnum
 737
 738         log.msg("configuration update started")
 739         def _done(res):
 740             self.readConfig = True
 741             log.msg("configuration update complete")
 742         d.addCallback(_done)
 743         d.addCallback(lambda res: self.botmaster.maybeStartAllBuilds())
 744         return d
 745
 746     def loadConfig_Slaves(self, new_slaves):
 747         # set up the Checker with the names and passwords of all valid bots
 748         self.checker.users = {} # violates abstraction, oh well
 749         for s in new_slaves:
 750             self.checker.addUser(s.slavename, s.password)
 751         self.checker.addUser("change", "changepw")
 752         # let the BotMaster take care of the rest
 753         return self.botmaster.loadConfig_Slaves(new_slaves)
 754
 755     def loadConfig_Sources(self, sources):
 756         if not sources:
 757             log.msg("warning: no ChangeSources specified in c['change_source']")
 758         # shut down any that were removed, start any that were added
 759         deleted_sources = [s for s in self.change_svc if s not in sources]
 760         added_sources = [s for s in sources if s not in self.change_svc]
 761         dl = [self.change_svc.removeSource(s) for s in deleted_sources]
 762         def addNewOnes(res):
 763             [self.change_svc.addSource(s) for s in added_sources]
 764         d = defer.DeferredList(dl, fireOnOneErrback=1, consumeErrors=0)
 765         d.addCallback(addNewOnes)
 766         return d
 767
 768     def allSchedulers(self):
 769         return [child for child in self
 770                 if interfaces.IScheduler.providedBy(child)]
 771
 772
 773     def loadConfig_Schedulers(self, newschedulers):
 774         oldschedulers = self.allSchedulers()
 775         removed = [s for s in oldschedulers if s not in newschedulers]
 776         added = [s for s in newschedulers if s not in oldschedulers]
 777         dl = [defer.maybeDeferred(s.disownServiceParent) for s in removed]
 778         def addNewOnes(res):
 779             for s in added:
 780                 s.setServiceParent(self)
 781         d = defer.DeferredList(dl, fireOnOneErrback=1)
 782         d.addCallback(addNewOnes)
 783         return d
 784
 785     def loadConfig_Builders(self, newBuilderData):
 786         somethingChanged = False
 787         newList = {}
 788         newBuilderNames = []
 789         allBuilders = self.botmaster.builders.copy()
 790         for data in newBuilderData:
 791             name = data['name']
 792             newList[name] = data
 793             newBuilderNames.append(name)
 794
 795         # identify all that were removed
 796         for oldname in self.botmaster.getBuildernames():
 797             if oldname not in newList:
 798                 log.msg("removing old builder %s" % oldname)
 799                 del allBuilders[oldname]
 800                 somethingChanged = True
 801                 # announce the change
 802                 self.status.builderRemoved(oldname)
 803
 804         # everything in newList is either unchanged, changed, or new
 805         for name, data in newList.items():
 806             old = self.botmaster.builders.get(name)
 807             basedir = data['builddir'] # used on both master and slave
 808             #name, slave, builddir, factory = data
 809             if not old: # new
 810                 # category added after 0.6.2
 811                 category = data.get('category', None)
 812                 log.msg("adding new builder %s for category %s" %
 813                         (name, category))
 814                 statusbag = self.status.builderAdded(name, basedir, category)
 815                 builder = Builder(data, statusbag)
 816                 allBuilders[name] = builder
 817                 somethingChanged = True
 818             elif old.compareToSetup(data):
 819                 # changed: try to minimize the disruption and only modify the
 820                 # pieces that really changed
 821                 diffs = old.compareToSetup(data)
 822                 log.msg("updating builder %s: %s" % (name, "\n".join(diffs)))
 823
 824                 statusbag = old.builder_status
 825                 statusbag.saveYourself() # seems like a good idea
 826                 # TODO: if the basedir was changed, we probably need to make
 827                 # a new statusbag
 828                 new_builder = Builder(data, statusbag)
 829                 new_builder.consumeTheSoulOfYourPredecessor(old)
 830                 # that migrates any retained slavebuilders too
 831
 832                 # point out that the builder was updated. On the Waterfall,
 833                 # this will appear just after any currently-running builds.
 834                 statusbag.addPointEvent(["config", "updated"])
 835
 836                 allBuilders[name] = new_builder
 837                 somethingChanged = True
 838             else:
 839                 # unchanged: leave it alone
 840                 log.msg("builder %s is unchanged" % name)
 841                 pass
 842
 843         if somethingChanged:
 844             sortedAllBuilders = [allBuilders[name] for name in newBuilderNames]
 845             d = self.botmaster.setBuilders(sortedAllBuilders)
 846             return d
 847         return None
 848
 849     def loadConfig_status(self, status):
 850         dl = []
 851
 852         # remove old ones
 853         for s in self.statusTargets[:]:
 854             if not s in status:
 855                 log.msg("removing IStatusReceiver", s)
 856                 d = defer.maybeDeferred(s.disownServiceParent)
 857                 dl.append(d)
 858                 self.statusTargets.remove(s)
 859         # after those are finished going away, add new ones
 860         def addNewOnes(res):
 861             for s in status:
 862                 if not s in self.statusTargets:
 863                     log.msg("adding IStatusReceiver", s)
 864                     s.setServiceParent(self)
 865                     self.statusTargets.append(s)
 866         d = defer.DeferredList(dl, fireOnOneErrback=1)
 867         d.addCallback(addNewOnes)
 868         return d
 869
 870
 871     def addChange(self, change):
 872         for s in self.allSchedulers():
 873             s.addChange(change)
 874
 875     def submitBuildSet(self, bs):
 876         # determine the set of Builders to use
 877         builders = []
 878         for name in bs.builderNames:
 879             b = self.botmaster.builders.get(name)
 880             if b:
 881                 if b not in builders:
 882                     builders.append(b)
 883                 continue
 884             # TODO: add aliases like 'all'
 885             raise KeyError("no such builder named '%s'" % name)
 886
 887         # now tell the BuildSet to create BuildRequests for all those
 888         # Builders and submit them
 889         bs.start(builders)
 890         self.status.buildsetSubmitted(bs.status)
 891
 892
 893 class Control:
 894     implements(interfaces.IControl)
 895
 896     def __init__(self, master):
 897         self.master = master
 898
 899     def addChange(self, change):
 900         self.master.change_svc.addChange(change)
 901
 902     def submitBuildSet(self, bs):
 903         self.master.submitBuildSet(bs)
 904
 905     def getBuilder(self, name):
 906         b = self.master.botmaster.builders[name]
 907         return interfaces.IBuilderControl(b)
 908
 909 components.registerAdapter(Control, BuildMaster, interfaces.IControl)
 910
 911 # so anybody who can get a handle on the BuildMaster can cause a build with:
 912 #  IControl(master).getBuilder("full-2.3").requestBuild(buildrequest)
 913