buildbot/test/test_slaves.py

   1 # -*- test-case-name: buildbot.test.test_slaves -*-
   2
   3 from twisted.trial import unittest
   4 from twisted.internet import defer, reactor
   5 from twisted.python import log, runtime, failure
   6
   7 from buildbot.buildslave import AbstractLatentBuildSlave
   8 from buildbot.test.runutils import RunMixin
   9 from buildbot.sourcestamp import SourceStamp
  10 from buildbot.process.base import BuildRequest
  11 from buildbot.status.builder import SUCCESS
  12 from buildbot.status import mail
  13 from buildbot.slave import bot
  14
  15 config_1 = """
  16 from buildbot.process import factory
  17 from buildbot.steps import dummy
  18 from buildbot.buildslave import BuildSlave
  19 s = factory.s
  20
  21 BuildmasterConfig = c = {}
  22 c['slaves'] = [BuildSlave('bot1', 'sekrit'), BuildSlave('bot2', 'sekrit'),
  23                BuildSlave('bot3', 'sekrit')]
  24 c['schedulers'] = []
  25 c['slavePortnum'] = 0
  26 c['schedulers'] = []
  27
  28 f1 = factory.BuildFactory([s(dummy.RemoteDummy, timeout=1)])
  29 f2 = factory.BuildFactory([s(dummy.RemoteDummy, timeout=2)])
  30 f3 = factory.BuildFactory([s(dummy.RemoteDummy, timeout=3)])
  31 f4 = factory.BuildFactory([s(dummy.RemoteDummy, timeout=5)])
  32
  33 c['builders'] = [
  34     {'name': 'b1', 'slavenames': ['bot1','bot2','bot3'],
  35      'builddir': 'b1', 'factory': f1},
  36     ]
  37 """
  38
  39 config_2 = config_1 + """
  40
  41 c['builders'] = [
  42     {'name': 'b1', 'slavenames': ['bot1','bot2','bot3'],
  43      'builddir': 'b1', 'factory': f2},
  44     ]
  45
  46 """
  47
  48 config_busyness = config_1 + """
  49 c['builders'] = [
  50     {'name': 'b1', 'slavenames': ['bot1'],
  51      'builddir': 'b1', 'factory': f3},
  52     {'name': 'b2', 'slavenames': ['bot1'],
  53      'builddir': 'b2', 'factory': f4},
  54     ]
  55 """
  56
  57 class Slave(RunMixin, unittest.TestCase):
  58
  59     def setUp(self):
  60         RunMixin.setUp(self)
  61         self.master.loadConfig(config_1)
  62         self.master.startService()
  63         d = self.connectSlave(["b1"])
  64         d.addCallback(lambda res: self.connectSlave(["b1"], "bot2"))
  65         return d
  66
  67     def doBuild(self, buildername):
  68         br = BuildRequest("forced", SourceStamp(), 'test_builder')
  69         d = br.waitUntilFinished()
  70         self.control.getBuilder(buildername).requestBuild(br)
  71         return d
  72
  73     def testSequence(self):
  74         # make sure both slaves appear in the list.
  75         attached_slaves = [c for c in self.master.botmaster.slaves.values()
  76                            if c.slave]
  77         self.failUnlessEqual(len(attached_slaves), 2)
  78         b = self.master.botmaster.builders["b1"]
  79         self.failUnlessEqual(len(b.slaves), 2)
  80
  81         # since the current scheduling algorithm is simple and does not
  82         # rotate or attempt any sort of load-balancing, two builds in
  83         # sequence should both use the first slave. This may change later if
  84         # we move to a more sophisticated scheme.
  85         b.CHOOSE_SLAVES_RANDOMLY = False
  86
  87         d = self.doBuild("b1")
  88         d.addCallback(self._testSequence_1)
  89         return d
  90     def _testSequence_1(self, res):
  91         self.failUnlessEqual(res.getResults(), SUCCESS)
  92         self.failUnlessEqual(res.getSlavename(), "bot1")
  93
  94         d = self.doBuild("b1")
  95         d.addCallback(self._testSequence_2)
  96         return d
  97     def _testSequence_2(self, res):
  98         self.failUnlessEqual(res.getSlavename(), "bot1")
  99
 100
 101     def testSimultaneous(self):
 102         # make sure we can actually run two builds at the same time
 103         d1 = self.doBuild("b1")
 104         d2 = self.doBuild("b1")
 105         d1.addCallback(self._testSimultaneous_1, d2)
 106         return d1
 107     def _testSimultaneous_1(self, res, d2):
 108         self.failUnlessEqual(res.getResults(), SUCCESS)
 109         b1_slavename = res.getSlavename()
 110         d2.addCallback(self._testSimultaneous_2, b1_slavename)
 111         return d2
 112     def _testSimultaneous_2(self, res, b1_slavename):
 113         self.failUnlessEqual(res.getResults(), SUCCESS)
 114         b2_slavename = res.getSlavename()
 115         # make sure the two builds were run by different slaves
 116         slavenames = [b1_slavename, b2_slavename]
 117         slavenames.sort()
 118         self.failUnlessEqual(slavenames, ["bot1", "bot2"])
 119
 120     def testFallback1(self):
 121         # detach the first slave, verify that a build is run using the second
 122         # slave instead
 123         d = self.shutdownSlave("bot1", "b1")
 124         d.addCallback(self._testFallback1_1)
 125         return d
 126     def _testFallback1_1(self, res):
 127         attached_slaves = [c for c in self.master.botmaster.slaves.values()
 128                            if c.slave]
 129         self.failUnlessEqual(len(attached_slaves), 1)
 130         self.failUnlessEqual(len(self.master.botmaster.builders["b1"].slaves),
 131                              1)
 132         d = self.doBuild("b1")
 133         d.addCallback(self._testFallback1_2)
 134         return d
 135     def _testFallback1_2(self, res):
 136         self.failUnlessEqual(res.getResults(), SUCCESS)
 137         self.failUnlessEqual(res.getSlavename(), "bot2")
 138
 139     def testFallback2(self):
 140         # Disable the first slave, so that a slaveping will timeout. Then
 141         # start a build, and verify that the non-failing (second) one is
 142         # claimed for the build, and that the failing one is removed from the
 143         # list.
 144
 145         b1 = self.master.botmaster.builders["b1"]
 146         # reduce the ping time so we'll failover faster
 147         b1.START_BUILD_TIMEOUT = 1
 148         assert b1.CHOOSE_SLAVES_RANDOMLY
 149         b1.CHOOSE_SLAVES_RANDOMLY = False
 150         self.disappearSlave("bot1", "b1", allowReconnect=False)
 151         d = self.doBuild("b1")
 152         d.addCallback(self._testFallback2_1)
 153         return d
 154     def _testFallback2_1(self, res):
 155         self.failUnlessEqual(res.getResults(), SUCCESS)
 156         self.failUnlessEqual(res.getSlavename(), "bot2")
 157         b1slaves = self.master.botmaster.builders["b1"].slaves
 158         self.failUnlessEqual(len(b1slaves), 1, "whoops: %s" % (b1slaves,))
 159         self.failUnlessEqual(b1slaves[0].slave.slavename, "bot2")
 160
 161
 162     def notFinished(self, brs):
 163         # utility method
 164         builds = brs.getBuilds()
 165         self.failIf(len(builds) > 1)
 166         if builds:
 167             self.failIf(builds[0].isFinished())
 168
 169     def testDontClaimPingingSlave(self):
 170         # have two slaves connect for the same builder. Do something to the
 171         # first one so that slavepings are delayed (but do not fail
 172         # outright).
 173         timers = []
 174         self.slaves['bot1'].debugOpts["stallPings"] = (10, timers)
 175         br = BuildRequest("forced", SourceStamp(), 'test_builder')
 176         d1 = br.waitUntilFinished()
 177         self.master.botmaster.builders["b1"].CHOOSE_SLAVES_RANDOMLY = False
 178         self.control.getBuilder("b1").requestBuild(br)
 179         s1 = br.status # this is a BuildRequestStatus
 180         # give it a chance to start pinging
 181         d2 = defer.Deferred()
 182         d2.addCallback(self._testDontClaimPingingSlave_1, d1, s1, timers)
 183         reactor.callLater(1, d2.callback, None)
 184         return d2
 185     def _testDontClaimPingingSlave_1(self, res, d1, s1, timers):
 186         # now the first build is running (waiting on the ping), so start the
 187         # second build. This should claim the second slave, not the first,
 188         # because the first is busy doing the ping.
 189         self.notFinished(s1)
 190         d3 = self.doBuild("b1")
 191         d3.addCallback(self._testDontClaimPingingSlave_2, d1, s1, timers)
 192         return d3
 193     def _testDontClaimPingingSlave_2(self, res, d1, s1, timers):
 194         self.failUnlessEqual(res.getSlavename(), "bot2")
 195         self.notFinished(s1)
 196         # now let the ping complete
 197         self.failUnlessEqual(len(timers), 1)
 198         timers[0].reset(0)
 199         d1.addCallback(self._testDontClaimPingingSlave_3)
 200         return d1
 201     def _testDontClaimPingingSlave_3(self, res):
 202         self.failUnlessEqual(res.getSlavename(), "bot1")
 203
 204 class FakeLatentBuildSlave(AbstractLatentBuildSlave):
 205
 206     testcase = None
 207     stop_wait = None
 208     start_message = None
 209     stopped = testing_substantiation_timeout = False
 210
 211     def start_instance(self):
 212         # responsible for starting instance that will try to connect with
 213         # this master
 214         # simulate having to do some work.
 215         d = defer.Deferred()
 216         if not self.testing_substantiation_timeout:
 217             reactor.callLater(0, self._start_instance, d)
 218         return d
 219
 220     def _start_instance(self, d):
 221         self.testcase.connectOneSlave(self.slavename)
 222         d.callback(self.start_message)
 223
 224     def stop_instance(self, fast=False):
 225         # responsible for shutting down instance
 226         # we're going to emulate dropping off the net.
 227
 228         # simulate this by replacing the slave Broker's .dataReceived method
 229         # with one that just throws away all data.
 230         self.fast_stop_request = fast
 231         if self.slavename not in self.testcase.slaves:
 232             assert self.testing_substantiation_timeout
 233             self.stopped = True
 234             return defer.succeed(None)
 235         d = defer.Deferred()
 236         if self.stop_wait is None:
 237             self._stop_instance(d)
 238         else:
 239             reactor.callLater(self.stop_wait, self._stop_instance, d)
 240         return d
 241
 242     def _stop_instance(self, d):
 243         try:
 244             s = self.testcase.slaves.pop(self.slavename)
 245         except KeyError:
 246             pass
 247         else:
 248             def discard(data):
 249                 pass
 250             bot = s.getServiceNamed("bot")
 251             for buildername in self.slavebuilders:
 252                 remote = bot.builders[buildername].remote
 253                 if remote is None:
 254                     continue
 255                 broker = remote.broker
 256                 broker.dataReceived = discard # seal its ears
 257                 broker.transport.write = discard # and take away its voice
 258             # also discourage it from reconnecting once the connection goes away
 259             s.bf.continueTrying = False
 260             # stop the service for cleanliness
 261             s.stopService()
 262         d.callback(None)
 263
 264 latent_config = """
 265 from buildbot.process import factory
 266 from buildbot.steps import dummy
 267 from buildbot.buildslave import BuildSlave
 268 from buildbot.test.test_slaves import FakeLatentBuildSlave
 269 s = factory.s
 270
 271 BuildmasterConfig = c = {}
 272 c['slaves'] = [FakeLatentBuildSlave('bot1', 'sekrit',
 273                                    ),
 274                FakeLatentBuildSlave('bot2', 'sekrit',
 275                                    ),
 276                BuildSlave('bot3', 'sekrit')]
 277 c['schedulers'] = []
 278 c['slavePortnum'] = 0
 279 c['schedulers'] = []
 280
 281 f1 = factory.BuildFactory([s(dummy.RemoteDummy, timeout=1)])
 282 f2 = factory.BuildFactory([s(dummy.RemoteDummy, timeout=2)])
 283 f3 = factory.BuildFactory([s(dummy.RemoteDummy, timeout=3)])
 284 f4 = factory.BuildFactory([s(dummy.RemoteDummy, timeout=5)])
 285
 286 c['builders'] = [
 287     {'name': 'b1', 'slavenames': ['bot1','bot2','bot3'],
 288      'builddir': 'b1', 'factory': f1},
 289     ]
 290 """
 291
 292
 293 class LatentSlave(RunMixin, unittest.TestCase):
 294
 295     def setUp(self):
 296         # debugging
 297         #import twisted.internet.base
 298         #twisted.internet.base.DelayedCall.debug = True
 299         # debugging
 300         RunMixin.setUp(self)
 301         self.master.loadConfig(latent_config)
 302         self.master.startService()
 303         self.bot1 = self.master.botmaster.slaves['bot1']
 304         self.bot2 = self.master.botmaster.slaves['bot2']
 305         self.bot3 = self.master.botmaster.slaves['bot3']
 306         self.bot1.testcase = self
 307         self.bot2.testcase = self
 308         self.b1 = self.master.botmaster.builders['b1']
 309
 310     def doBuild(self, buildername):
 311         br = BuildRequest("forced", SourceStamp())
 312         d = br.waitUntilFinished()
 313         self.control.getBuilder(buildername).requestBuild(br)
 314         return d
 315
 316     def testSequence(self):
 317         # make sure both slaves appear in the builder.  This is automatically,
 318         # without any attaching.
 319         self.assertEqual(len(self.b1.slaves), 2)
 320         self.assertEqual(sorted(sb.slave.slavename for sb in self.b1.slaves),
 321                          ['bot1', 'bot2'])
 322         # These have not substantiated
 323         self.assertEqual([sb.slave.substantiated for sb in self.b1.slaves],
 324                          [False, False])
 325         self.assertEqual([sb.slave.slave for sb in self.b1.slaves],
 326                          [None, None])
 327         # we can mix and match latent slaves and normal slaves.  ATM, they
 328         # are treated identically in terms of selecting slaves.
 329         d = self.connectSlave(builders=['b1'], slavename='bot3')
 330         d.addCallback(self._testSequence_1)
 331         return d
 332     def _testSequence_1(self, res):
 333         # now we have all three slaves.  Two are latent slaves, and one is a
 334         # standard slave.
 335         self.assertEqual(sorted(sb.slave.slavename for sb in self.b1.slaves),
 336                          ['bot1', 'bot2', 'bot3'])
 337         # Now it's time to try a build on one of the latent slaves,
 338         # substantiating it.
 339         # since the current scheduling algorithm is simple and does not
 340         # rotate or attempt any sort of load-balancing, two builds in
 341         # sequence should both use the first slave. This may change later if
 342         # we move to a more sophisticated scheme.
 343         self.b1.CHOOSE_SLAVES_RANDOMLY = False
 344
 345         self.build_deferred = self.doBuild("b1")
 346         # now there's an event waiting for the slave to substantiate.
 347         e = self.b1.builder_status.getEvent(-1)
 348         self.assertEqual(e.text, ['substantiating'])
 349         self.assertEqual(e.color, 'yellow')
 350         # the substantiation_deferred is an internal stash of a deferred
 351         # that we'll grab so we can find the point at which the slave is
 352         # substantiated but the build has not yet started.
 353         d = self.bot1.substantiation_deferred
 354         self.assertNotIdentical(d, None)
 355         d.addCallback(self._testSequence_2)
 356         return d
 357     def _testSequence_2(self, res):
 358         # bot 1 is substantiated.
 359         self.assertNotIdentical(self.bot1.slave, None)
 360         self.failUnless(self.bot1.substantiated)
 361         # the event has announced it's success
 362         e = self.b1.builder_status.getEvent(-1)
 363         self.assertEqual(e.text, ['substantiate', 'success'])
 364         self.assertEqual(e.color, 'green')
 365         self.assertNotIdentical(e.finished, None)
 366         # now we'll wait for the build to complete
 367         d = self.build_deferred
 368         del self.build_deferred
 369         d.addCallback(self._testSequence_3)
 370         return d
 371     def _testSequence_3(self, res):
 372         # build was a success!
 373         self.failUnlessEqual(res.getResults(), SUCCESS)
 374         self.failUnlessEqual(res.getSlavename(), "bot1")
 375         # bot1 is substantiated now.  bot2 has not.
 376         self.failUnless(self.bot1.substantiated)
 377         self.failIf(self.bot2.substantiated)
 378         # bot1 is waiting a bit to see if there will be another build before
 379         # it shuts down the instance ("insubstantiates")
 380         self.build_wait_timer = self.bot1.build_wait_timer
 381         self.assertNotIdentical(self.build_wait_timer, None)
 382         self.failUnless(self.build_wait_timer.active())
 383         self.assertApproximates(
 384             self.bot1.build_wait_timeout,
 385             self.build_wait_timer.time - runtime.seconds(),
 386             2)
 387         # now we'll do another build
 388         d = self.doBuild("b1")
 389         # the slave is already substantiated, so no event is created
 390         e = self.b1.builder_status.getEvent(-1)
 391         self.assertNotEqual(e.text, ['substantiating'])
 392         # wait for the next build
 393         d.addCallback(self._testSequence_4)
 394         return d
 395     def _testSequence_4(self, res):
 396         # build was a success!
 397         self.failUnlessEqual(res.getResults(), SUCCESS)
 398         self.failUnlessEqual(res.getSlavename(), "bot1")
 399         # bot1 is still waiting, but with a new timer
 400         self.assertNotIdentical(self.bot1.build_wait_timer, None)
 401         self.assertNotIdentical(self.build_wait_timer,
 402                                 self.bot1.build_wait_timer)
 403         self.assertApproximates(
 404             self.bot1.build_wait_timeout,
 405             self.bot1.build_wait_timer.time - runtime.seconds(),
 406             2)
 407         del self.build_wait_timer
 408         # We'll set the timer to fire sooner, and wait for it to fire.
 409         self.bot1.build_wait_timer.reset(0)
 410         d = defer.Deferred()
 411         reactor.callLater(1, d.callback, None)
 412         d.addCallback(self._testSequence_5)
 413         return d
 414     def _testSequence_5(self, res):
 415         # slave is insubstantiated
 416         self.assertIdentical(self.bot1.slave, None)
 417         self.failIf(self.bot1.substantiated)
 418         # Now we'll start up another build, to show that the shutdown left
 419         # things in such a state that we can restart.
 420         d = self.doBuild("b1")
 421         # the bot can return an informative message on success that the event
 422         # will render.  Let's use a mechanism of our test latent bot to
 423         # demonstrate that.
 424         self.bot1.start_message = ['[instance id]', '[start-up time]']
 425         # here's our event again:
 426         self.e = self.b1.builder_status.getEvent(-1)
 427         self.assertEqual(self.e.text, ['substantiating'])
 428         self.assertEqual(self.e.color, 'yellow')
 429         d.addCallback(self._testSequence_6)
 430         return d
 431     def _testSequence_6(self, res):
 432         # build was a success!
 433         self.failUnlessEqual(res.getResults(), SUCCESS)
 434         self.failUnlessEqual(res.getSlavename(), "bot1")
 435         # the event has announced it's success.  (Just imagine that
 436         # [instance id] and [start-up time] were actually valuable
 437         # information.)
 438         e = self.e
 439         del self.e
 440         self.assertEqual(
 441             e.text,
 442             ['substantiate', 'success', '[instance id]', '[start-up time]'])
 443         self.assertEqual(e.color, 'green')
 444         # Now we need to clean up the timer.  We could just cancel it, but
 445         # we'll go through the full dance once more time to show we can.
 446         # We'll set the timer to fire sooner, and wait for it to fire.
 447         # Also, we'll set the build_slave to take a little bit longer to shut
 448         # down, to see that it doesn't affect anything.
 449         self.bot1.stop_wait = 2
 450         self.bot1.build_wait_timer.reset(0)
 451         d = defer.Deferred()
 452         reactor.callLater(1, d.callback, None)
 453         d.addCallback(self._testSequence_7)
 454         return d
 455     def _testSequence_7(self, res):
 456         # slave is insubstantiated
 457         self.assertIdentical(self.bot1.slave, None)
 458         self.assertNot(self.bot1.substantiated)
 459         # the remote is still not cleaned out.  We'll wait for it.
 460         d = defer.Deferred()
 461         reactor.callLater(1, d.callback, None)
 462         return d
 463
 464     def testNeverSubstantiated(self):
 465         # When a substantiation is requested, the slave may never appear.
 466         # This is a serious problem, and recovering from it is not really
 467         # handled well right now (in part because a way to handle it is not
 468         # clear).  However, at the least, the status event will show a
 469         # failure, and the slave will be told to insubstantiate, and to be
 470         # removed from the botmaster as anavailable slave.
 471         # This tells our test bot to never start, and to not complain about
 472         # being told to stop without ever starting
 473         self.bot1.testing_substantiation_timeout = True
 474         # normally (by default) we have 20 minutes to try and connect to the
 475         # remote
 476         self.assertEqual(self.bot1.missing_timeout, 20*60)
 477         # for testing purposes, we'll put that down to a tenth of a second!
 478         self.bot1.missing_timeout = 0.1
 479         # since the current scheduling algorithm is simple and does not
 480         # rotate or attempt any sort of load-balancing, two builds in
 481         # sequence should both use the first slave. This may change later if
 482         # we move to a more sophisticated scheme.
 483         self.b1.CHOOSE_SLAVES_RANDOMLY = False
 484         # start a build
 485         self.build_deferred = self.doBuild('b1')
 486         # the event tells us we are instantiating, as usual
 487         e = self.b1.builder_status.getEvent(-1)
 488         self.assertEqual(e.text, ['substantiating'])
 489         self.assertEqual(e.color, 'yellow')
 490         # we'll see in a moment that the test flag we have to show that the
 491         # bot was told to insubstantiate has been fired.  Here, we just verify
 492         # that it is ready to be fired.
 493         self.failIf(self.bot1.stopped)
 494         # That substantiation is going to fail.  Let's wait for it.
 495         d = self.bot1.substantiation_deferred
 496         self.assertNotIdentical(d, None)
 497         d.addCallbacks(self._testNeverSubstantiated_BadSuccess,
 498                        self._testNeverSubstantiated_1)
 499         return d
 500     def _testNeverSubstantiated_BadSuccess(self, res):
 501         self.fail('we should not have succeeded here.')
 502     def _testNeverSubstantiated_1(self, res):
 503         # ok, we failed.
 504         self.assertIdentical(self.bot1.slave, None)
 505         self.failIf(self.bot1.substantiated)
 506         self.failUnless(isinstance(res, failure.Failure))
 507         self.assertIdentical(self.bot1.substantiation_deferred, None)
 508         # our event informs us of this
 509         e1 = self.b1.builder_status.getEvent(-3)
 510         self.assertEqual(e1.text, ['substantiate', 'failed'])
 511         self.assertEqual(e1.color, 'red')
 512         self.assertNotIdentical(e1.finished, None)
 513         # the slave is no longer available to build.  The events show it...
 514         e2 = self.b1.builder_status.getEvent(-2)
 515         self.assertEqual(e2.text, ['removing', 'latent', 'bot1'])
 516         e3 = self.b1.builder_status.getEvent(-1)
 517         self.assertEqual(e3.text, ['disconnect', 'bot1'])
 518         # ...and the builder shows it.
 519         self.assertEqual(['bot2'],
 520                          [sb.slave.slavename for sb in self.b1.slaves])
 521         # ideally, we would retry the build, but that infrastructure (which
 522         # would be used for other situations in the builder as well) does not
 523         # yet exist.  Therefore the build never completes one way or the
 524         # other, just as if a normal slave detached.
 525
 526     def testServiceStop(self):
 527         # if the slave has an instance when it is stopped, the slave should
 528         # be told to shut down.
 529         self.b1.CHOOSE_SLAVES_RANDOMLY = False
 530         d = self.doBuild("b1")
 531         d.addCallback(self._testServiceStop_1)
 532         return d
 533     def _testServiceStop_1(self, res):
 534         # build was a success!
 535         self.failUnlessEqual(res.getResults(), SUCCESS)
 536         self.failUnlessEqual(res.getSlavename(), "bot1")
 537         # bot 1 is substantiated.
 538         self.assertNotIdentical(self.bot1.slave, None)
 539         self.failUnless(self.bot1.substantiated)
 540         # now let's stop the bot.
 541         d = self.bot1.stopService()
 542         d.addCallback(self._testServiceStop_2)
 543         return d
 544     def _testServiceStop_2(self, res):
 545         # bot 1 is NOT substantiated.
 546         self.assertIdentical(self.bot1.slave, None)
 547         self.failIf(self.bot1.substantiated)
 548
 549     def testPing(self):
 550         # While a latent slave pings normally when it is substantiated, (as
 551         # happens behind the scene when a build is request), when
 552         # it is insubstantial, the ping is a no-op success.
 553         self.assertIdentical(self.bot1.slave, None)
 554         self.failIf(self.bot1.substantiated)
 555         d = self.connectSlave(builders=['b1'], slavename='bot3')
 556         d.addCallback(self._testPing_1)
 557         return d
 558     def _testPing_1(self, res):
 559         self.assertEqual(sorted(sb.slave.slavename for sb in self.b1.slaves),
 560                          ['bot1', 'bot2', 'bot3'])
 561         d = self.control.getBuilder('b1').ping()
 562         d.addCallback(self._testPing_2)
 563         return d
 564     def _testPing_2(self, res):
 565         # all three pings were successful
 566         self.assert_(res)
 567         # but neither bot1 not bot2 substantiated.
 568         self.assertIdentical(self.bot1.slave, None)
 569         self.failIf(self.bot1.substantiated)
 570         self.assertIdentical(self.bot2.slave, None)
 571         self.failIf(self.bot2.substantiated)
 572
 573
 574 class SlaveBusyness(RunMixin, unittest.TestCase):
 575
 576     def setUp(self):
 577         RunMixin.setUp(self)
 578         self.master.loadConfig(config_busyness)
 579         self.master.startService()
 580         d = self.connectSlave(["b1", "b2"])
 581         return d
 582
 583     def doBuild(self, buildername):
 584         br = BuildRequest("forced", SourceStamp(), 'test_builder')
 585         d = br.waitUntilFinished()
 586         self.control.getBuilder(buildername).requestBuild(br)
 587         return d
 588
 589     def getRunningBuilds(self):
 590         return len(self.status.getSlave("bot1").getRunningBuilds())
 591
 592     def testSlaveNotBusy(self):
 593         self.failUnlessEqual(self.getRunningBuilds(), 0)
 594         # now kick a build, wait for it to finish, then check again
 595         d = self.doBuild("b1")
 596         d.addCallback(self._testSlaveNotBusy_1)
 597         return d
 598
 599     def _testSlaveNotBusy_1(self, res):
 600         self.failUnlessEqual(self.getRunningBuilds(), 0)
 601
 602     def testSlaveBusyOneBuild(self):
 603         d1 = self.doBuild("b1")
 604         d2 = defer.Deferred()
 605         reactor.callLater(.5, d2.callback, None)
 606         d2.addCallback(self._testSlaveBusyOneBuild_1)
 607         d1.addCallback(self._testSlaveBusyOneBuild_finished_1)
 608         return defer.DeferredList([d1,d2])
 609
 610     def _testSlaveBusyOneBuild_1(self, res):
 611         self.failUnlessEqual(self.getRunningBuilds(), 1)
 612
 613     def _testSlaveBusyOneBuild_finished_1(self, res):
 614         self.failUnlessEqual(self.getRunningBuilds(), 0)
 615
 616     def testSlaveBusyTwoBuilds(self):
 617         d1 = self.doBuild("b1")
 618         d2 = self.doBuild("b2")
 619         d3 = defer.Deferred()
 620         reactor.callLater(.5, d3.callback, None)
 621         d3.addCallback(self._testSlaveBusyTwoBuilds_1)
 622         d1.addCallback(self._testSlaveBusyTwoBuilds_finished_1, d2)
 623         return defer.DeferredList([d1,d3])
 624
 625     def _testSlaveBusyTwoBuilds_1(self, res):
 626         self.failUnlessEqual(self.getRunningBuilds(), 2)
 627
 628     def _testSlaveBusyTwoBuilds_finished_1(self, res, d2):
 629         self.failUnlessEqual(self.getRunningBuilds(), 1)
 630         d2.addCallback(self._testSlaveBusyTwoBuilds_finished_2)
 631         return d2
 632
 633     def _testSlaveBusyTwoBuilds_finished_2(self, res):
 634         self.failUnlessEqual(self.getRunningBuilds(), 0)
 635
 636     def testSlaveDisconnect(self):
 637         d1 = self.doBuild("b1")
 638         d2 = defer.Deferred()
 639         reactor.callLater(.5, d2.callback, None)
 640         d2.addCallback(self._testSlaveDisconnect_1)
 641         d1.addCallback(self._testSlaveDisconnect_finished_1)
 642         return defer.DeferredList([d1, d2])
 643
 644     def _testSlaveDisconnect_1(self, res):
 645         self.failUnlessEqual(self.getRunningBuilds(), 1)
 646         return self.shutdownAllSlaves()
 647
 648     def _testSlaveDisconnect_finished_1(self, res):
 649         self.failUnlessEqual(self.getRunningBuilds(), 0)
 650
 651 config_3 = """
 652 from buildbot.process import factory
 653 from buildbot.steps import dummy
 654 from buildbot.buildslave import BuildSlave
 655 s = factory.s
 656
 657 BuildmasterConfig = c = {}
 658 c['slaves'] = [BuildSlave('bot1', 'sekrit')]
 659 c['schedulers'] = []
 660 c['slavePortnum'] = 0
 661 c['schedulers'] = []
 662
 663 f1 = factory.BuildFactory([s(dummy.Wait, handle='one')])
 664 f2 = factory.BuildFactory([s(dummy.Wait, handle='two')])
 665 f3 = factory.BuildFactory([s(dummy.Wait, handle='three')])
 666
 667 c['builders'] = [
 668     {'name': 'b1', 'slavenames': ['bot1'],
 669      'builddir': 'b1', 'factory': f1},
 670     ]
 671 """
 672
 673 config_4 = config_3 + """
 674 c['builders'] = [
 675     {'name': 'b1', 'slavenames': ['bot1'],
 676      'builddir': 'b1', 'factory': f2},
 677     ]
 678 """
 679
 680 config_5 = config_3 + """
 681 c['builders'] = [
 682     {'name': 'b1', 'slavenames': ['bot1'],
 683      'builddir': 'b1', 'factory': f3},
 684     ]
 685 """
 686
 687 from buildbot.slave.commands import waitCommandRegistry
 688
 689 class Reconfig(RunMixin, unittest.TestCase):
 690
 691     def setUp(self):
 692         RunMixin.setUp(self)
 693         self.master.loadConfig(config_3)
 694         self.master.startService()
 695         d = self.connectSlave(["b1"])
 696         return d
 697
 698     def _one_started(self):
 699         log.msg("testReconfig._one_started")
 700         self.build1_started = True
 701         self.d1.callback(None)
 702         return self.d2
 703
 704     def _two_started(self):
 705         log.msg("testReconfig._two_started")
 706         self.build2_started = True
 707         self.d3.callback(None)
 708         return self.d4
 709
 710     def _three_started(self):
 711         log.msg("testReconfig._three_started")
 712         self.build3_started = True
 713         self.d5.callback(None)
 714         return self.d6
 715
 716     def testReconfig(self):
 717         # reconfiguring a Builder should not interrupt any running Builds. No
 718         # queued BuildRequests should be lost. The next Build started should
 719         # use the new process.
 720         slave1 = self.slaves['bot1']
 721         bot1 = slave1.getServiceNamed('bot')
 722         sb1 = bot1.builders['b1']
 723         self.failUnless(isinstance(sb1, bot.SlaveBuilder))
 724         self.failUnless(sb1.running)
 725         b1 = self.master.botmaster.builders['b1']
 726         self.orig_b1 = b1
 727
 728         self.d1 = d1 = defer.Deferred()
 729         self.d2 = d2 = defer.Deferred()
 730         self.d3, self.d4 = defer.Deferred(), defer.Deferred()
 731         self.d5, self.d6 = defer.Deferred(), defer.Deferred()
 732         self.build1_started = False
 733         self.build2_started = False
 734         self.build3_started = False
 735         waitCommandRegistry[("one","build1")] = self._one_started
 736         waitCommandRegistry[("two","build2")] = self._two_started
 737         waitCommandRegistry[("three","build3")] = self._three_started
 738
 739         # use different branches to make sure these cannot be merged
 740         br1 = BuildRequest("build1", SourceStamp(branch="1"), 'test_builder')
 741         b1.submitBuildRequest(br1)
 742         br2 = BuildRequest("build2", SourceStamp(branch="2"), 'test_builder')
 743         b1.submitBuildRequest(br2)
 744         br3 = BuildRequest("build3", SourceStamp(branch="3"), 'test_builder')
 745         b1.submitBuildRequest(br3)
 746         self.requests = (br1, br2, br3)
 747         # all three are now in the queue
 748
 749         # wait until the first one has started
 750         d1.addCallback(self._testReconfig_2)
 751         return d1
 752
 753     def _testReconfig_2(self, res):
 754         log.msg("_testReconfig_2")
 755         # confirm that it is building
 756         brs = self.requests[0].status.getBuilds()
 757         self.failUnlessEqual(len(brs), 1)
 758         self.build1 = brs[0]
 759         self.failUnlessEqual(self.build1.getCurrentStep().getName(), "wait")
 760         # br1 is building, br2 and br3 are in the queue (in that order). Now
 761         # we reconfigure the Builder.
 762         self.failUnless(self.build1_started)
 763         d = self.master.loadConfig(config_4)
 764         d.addCallback(self._testReconfig_3)
 765         return d
 766
 767     def _testReconfig_3(self, res):
 768         log.msg("_testReconfig_3")
 769         # now check to see that br1 is still building, and that br2 and br3
 770         # are in the queue of the new builder
 771         b1 = self.master.botmaster.builders['b1']
 772         self.failIfIdentical(b1, self.orig_b1)
 773         self.failIf(self.build1.isFinished())
 774         self.failUnlessEqual(self.build1.getCurrentStep().getName(), "wait")
 775         self.failUnlessEqual(len(b1.buildable), 2)
 776         self.failUnless(self.requests[1] in b1.buildable)
 777         self.failUnless(self.requests[2] in b1.buildable)
 778
 779         # allow br1 to finish, and make sure its status is delivered normally
 780         d = self.requests[0].waitUntilFinished()
 781         d.addCallback(self._testReconfig_4)
 782         self.d2.callback(None)
 783         return d
 784
 785     def _testReconfig_4(self, bs):
 786         log.msg("_testReconfig_4")
 787         self.failUnlessEqual(bs.getReason(), "build1")
 788         self.failUnless(bs.isFinished())
 789         self.failUnlessEqual(bs.getResults(), SUCCESS)
 790
 791         # at this point, the first build has finished, and there is a pending
 792         # call to start the second build. Once that pending call fires, there
 793         # is a network roundtrip before the 'wait' RemoteCommand is delivered
 794         # to the slave. We need to wait for both events to happen before we
 795         # can check to make sure it is using the correct process. Just wait a
 796         # full second.
 797         d = defer.Deferred()
 798         d.addCallback(self._testReconfig_5)
 799         reactor.callLater(1, d.callback, None)
 800         return d
 801
 802     def _testReconfig_5(self, res):
 803         log.msg("_testReconfig_5")
 804         # at this point the next build ought to be running
 805         b1 = self.master.botmaster.builders['b1']
 806         self.failUnlessEqual(len(b1.buildable), 1)
 807         self.failUnless(self.requests[2] in b1.buildable)
 808         self.failUnlessEqual(len(b1.building), 1)
 809         # and it ought to be using the new process
 810         self.failUnless(self.build2_started)
 811
 812         # now, while the second build is running, change the config multiple
 813         # times.
 814
 815         d = self.master.loadConfig(config_3)
 816         d.addCallback(lambda res: self.master.loadConfig(config_4))
 817         d.addCallback(lambda res: self.master.loadConfig(config_5))
 818         def _done(res):
 819             # then once that's done, allow the second build to finish and
 820             # wait for it to complete
 821             da = self.requests[1].waitUntilFinished()
 822             self.d4.callback(None)
 823             return da
 824         d.addCallback(_done)
 825         def _done2(res):
 826             # and once *that*'s done, wait another second to let the third
 827             # build start
 828             db = defer.Deferred()
 829             reactor.callLater(1, db.callback, None)
 830             return db
 831         d.addCallback(_done2)
 832         d.addCallback(self._testReconfig_6)
 833         return d
 834
 835     def _testReconfig_6(self, res):
 836         log.msg("_testReconfig_6")
 837         # now check to see that the third build is running
 838         self.failUnless(self.build3_started)
 839
 840         # we're done
 841
 842
 843
 844 class Slave2(RunMixin, unittest.TestCase):
 845
 846     revision = 0
 847
 848     def setUp(self):
 849         RunMixin.setUp(self)
 850         self.master.loadConfig(config_1)
 851         self.master.startService()
 852
 853     def doBuild(self, buildername, reason="forced"):
 854         # we need to prevent these builds from being merged, so we create
 855         # each of them with a different revision specifier. The revision is
 856         # ignored because our build process does not have a source checkout
 857         # step.
 858         self.revision += 1
 859         br = BuildRequest(reason, SourceStamp(revision=self.revision),
 860                                                             'test_builder')
 861         d = br.waitUntilFinished()
 862         self.control.getBuilder(buildername).requestBuild(br)
 863         return d
 864
 865     def testFirstComeFirstServed(self):
 866         # submit three builds, then connect a slave which fails the
 867         # slaveping. The first build will claim the slave, do the slaveping,
 868         # give up, and re-queue the build. Verify that the build gets
 869         # re-queued in front of all other builds. This may be tricky, because
 870         # the other builds may attempt to claim the just-failed slave.
 871
 872         d1 = self.doBuild("b1", "first")
 873         d2 = self.doBuild("b1", "second")
 874         #buildable = self.master.botmaster.builders["b1"].buildable
 875         #print [b.reason for b in buildable]
 876
 877         # specifically, I want the poor build to get precedence over any
 878         # others that were waiting. To test this, we need more builds than
 879         # slaves.
 880
 881         # now connect a broken slave. The first build started as soon as it
 882         # connects, so by the time we get to our _1 method, the ill-fated
 883         # build has already started.
 884         d = self.connectSlave(["b1"], opts={"failPingOnce": True})
 885         d.addCallback(self._testFirstComeFirstServed_1, d1, d2)
 886         return d
 887     def _testFirstComeFirstServed_1(self, res, d1, d2):
 888         # the master has send the slaveping. When this is received, it will
 889         # fail, causing the master to hang up on the slave. When it
 890         # reconnects, it should find the first build at the front of the
 891         # queue. If we simply wait for both builds to complete, then look at
 892         # the status logs, we should see that the builds ran in the correct
 893         # order.
 894
 895         d = defer.DeferredList([d1,d2])
 896         d.addCallback(self._testFirstComeFirstServed_2)
 897         return d
 898     def _testFirstComeFirstServed_2(self, res):
 899         b = self.status.getBuilder("b1")
 900         builds = b.getBuild(0), b.getBuild(1)
 901         reasons = [build.getReason() for build in builds]
 902         self.failUnlessEqual(reasons, ["first", "second"])
 903
 904 config_multi_builders = config_1 + """
 905 c['builders'] = [
 906     {'name': 'dummy', 'slavenames': ['bot1','bot2','bot3'],
 907      'builddir': 'b1', 'factory': f2},
 908     {'name': 'dummy2', 'slavenames': ['bot1','bot2','bot3'],
 909      'builddir': 'b2', 'factory': f2},
 910     {'name': 'dummy3', 'slavenames': ['bot1','bot2','bot3'],
 911      'builddir': 'b3', 'factory': f2},
 912     ]
 913
 914 """
 915
 916 config_mail_missing = config_1 + """
 917 c['slaves'] = [BuildSlave('bot1', 'sekrit', notify_on_missing='admin',
 918                           missing_timeout=1)]
 919 c['builders'] = [
 920     {'name': 'dummy', 'slavenames': ['bot1'],
 921      'builddir': 'b1', 'factory': f1},
 922     ]
 923 c['projectName'] = 'myproject'
 924 c['projectURL'] = 'myURL'
 925 """
 926
 927 class FakeMailer(mail.MailNotifier):
 928     def sendMessage(self, m, recipients):
 929         self.messages.append((m,recipients))
 930         return defer.succeed(None)
 931
 932 class BuildSlave(RunMixin, unittest.TestCase):
 933     def test_track_builders(self):
 934         self.master.loadConfig(config_multi_builders)
 935         self.master.readConfig = True
 936         self.master.startService()
 937         d = self.connectSlave()
 938
 939         def _check(res):
 940             b = self.master.botmaster.builders['dummy']
 941             self.failUnless(len(b.slaves) == 1) # just bot1
 942
 943             bs = b.slaves[0].slave
 944             self.failUnless(len(bs.slavebuilders) == 3)
 945             self.failUnless(b in [sb.builder for sb in
 946                                   bs.slavebuilders.values()])
 947
 948         d.addCallback(_check)
 949         return d
 950
 951     def test_mail_on_missing(self):
 952         self.master.loadConfig(config_mail_missing)
 953         self.master.readConfig = True
 954         self.master.startService()
 955         fm = FakeMailer("buildbot@example.org")
 956         fm.messages = []
 957         fm.setServiceParent(self.master)
 958         self.master.statusTargets.append(fm)
 959
 960         d = self.connectSlave()
 961         d.addCallback(self.stall, 1)
 962         d.addCallback(lambda res: self.shutdownSlave("bot1", "dummy"))
 963         def _not_yet(res):
 964             self.failIf(fm.messages)
 965         d.addCallback(_not_yet)
 966         # we reconnect right away, so the timer shouldn't fire
 967         d.addCallback(lambda res: self.connectSlave())
 968         d.addCallback(self.stall, 3)
 969         d.addCallback(_not_yet)
 970         d.addCallback(lambda res: self.shutdownSlave("bot1", "dummy"))
 971         d.addCallback(_not_yet)
 972         # now we let it sit disconnected for long enough for the timer to
 973         # fire
 974         d.addCallback(self.stall, 3)
 975         def _check(res):
 976             self.failUnlessEqual(len(fm.messages), 1)
 977             msg,recips = fm.messages[0]
 978             self.failUnlessEqual(recips, ["admin"])
 979             body = msg.as_string()
 980             self.failUnlessIn("To: admin", body)
 981             self.failUnlessIn("Subject: Buildbot: buildslave bot1 was lost",
 982                               body)
 983             self.failUnlessIn("From: buildbot@example.org", body)
 984             self.failUnlessIn("working for 'myproject'", body)
 985             self.failUnlessIn("has noticed that the buildslave named bot1 went away",
 986                               body)
 987             self.failUnlessIn("was 'one'", body)
 988             self.failUnlessIn("myURL", body)
 989         d.addCallback(_check)
 990         return d
 991
 992     def stall(self, result, delay=1):
 993         d = defer.Deferred()
 994         reactor.callLater(delay, d.callback, result)
 995         return d