buildbot/test/test_slaves.py

   1 # -*- test-case-name: buildbot.test.test_slaves -*-
   2
   3 from twisted.trial import unittest
   4 from twisted.internet import defer, reactor
   5 from twisted.python import log
   6
   7 from buildbot.test.runutils import RunMixin
   8 from buildbot.sourcestamp import SourceStamp
   9 from buildbot.process.base import BuildRequest
  10 from buildbot.status.builder import SUCCESS
  11 from buildbot.slave import bot
  12
  13 config_1 = """
  14 from buildbot.process import factory
  15 from buildbot.steps import dummy
  16 s = factory.s
  17
  18 BuildmasterConfig = c = {}
  19 c['bots'] = [('bot1', 'sekrit'), ('bot2', 'sekrit'), ('bot3', 'sekrit')]
  20 c['sources'] = []
  21 c['schedulers'] = []
  22 c['slavePortnum'] = 0
  23 c['schedulers'] = []
  24
  25 f1 = factory.BuildFactory([s(dummy.RemoteDummy, timeout=1)])
  26 f2 = factory.BuildFactory([s(dummy.RemoteDummy, timeout=2)])
  27
  28 c['builders'] = [
  29     {'name': 'b1', 'slavenames': ['bot1','bot2','bot3'],
  30      'builddir': 'b1', 'factory': f1},
  31     ]
  32 """
  33
  34 config_2 = config_1 + """
  35
  36 c['builders'] = [
  37     {'name': 'b1', 'slavenames': ['bot1','bot2','bot3'],
  38      'builddir': 'b1', 'factory': f2},
  39     ]
  40
  41 """
  42
  43 class Slave(RunMixin, unittest.TestCase):
  44
  45     def setUp(self):
  46         RunMixin.setUp(self)
  47         self.master.loadConfig(config_1)
  48         self.master.startService()
  49         d = self.connectSlave(["b1"])
  50         d.addCallback(lambda res: self.connectSlave(["b1"], "bot2"))
  51         return d
  52
  53     def doBuild(self, buildername):
  54         br = BuildRequest("forced", SourceStamp())
  55         d = br.waitUntilFinished()
  56         self.control.getBuilder(buildername).requestBuild(br)
  57         return d
  58
  59     def testSequence(self):
  60         # make sure both slaves appear in the list.
  61         attached_slaves = [c for c in self.master.botmaster.slaves.values()
  62                            if c.slave]
  63         self.failUnlessEqual(len(attached_slaves), 2)
  64         b = self.master.botmaster.builders["b1"]
  65         self.failUnlessEqual(len(b.slaves), 2)
  66
  67         # since the current scheduling algorithm is simple and does not
  68         # rotate or attempt any sort of load-balancing, two builds in
  69         # sequence should both use the first slave. This may change later if
  70         # we move to a more sophisticated scheme.
  71
  72         d = self.doBuild("b1")
  73         d.addCallback(self._testSequence_1)
  74         return d
  75     def _testSequence_1(self, res):
  76         self.failUnlessEqual(res.getResults(), SUCCESS)
  77         self.failUnlessEqual(res.getSlavename(), "bot1")
  78
  79         d = self.doBuild("b1")
  80         d.addCallback(self._testSequence_2)
  81         return d
  82     def _testSequence_2(self, res):
  83         self.failUnlessEqual(res.getSlavename(), "bot1")
  84
  85
  86     def testSimultaneous(self):
  87         # make sure we can actually run two builds at the same time
  88         d1 = self.doBuild("b1")
  89         d2 = self.doBuild("b1")
  90         d1.addCallback(self._testSimultaneous_1, d2)
  91         return d1
  92     def _testSimultaneous_1(self, res, d2):
  93         self.failUnlessEqual(res.getResults(), SUCCESS)
  94         self.failUnlessEqual(res.getSlavename(), "bot1")
  95         d2.addCallback(self._testSimultaneous_2)
  96         return d2
  97     def _testSimultaneous_2(self, res):
  98         self.failUnlessEqual(res.getResults(), SUCCESS)
  99         self.failUnlessEqual(res.getSlavename(), "bot2")
 100
 101     def testFallback1(self):
 102         # detach the first slave, verify that a build is run using the second
 103         # slave instead
 104         d = self.shutdownSlave("bot1", "b1")
 105         d.addCallback(self._testFallback1_1)
 106         return d
 107     def _testFallback1_1(self, res):
 108         attached_slaves = [c for c in self.master.botmaster.slaves.values()
 109                            if c.slave]
 110         self.failUnlessEqual(len(attached_slaves), 1)
 111         self.failUnlessEqual(len(self.master.botmaster.builders["b1"].slaves),
 112                              1)
 113         d = self.doBuild("b1")
 114         d.addCallback(self._testFallback1_2)
 115         return d
 116     def _testFallback1_2(self, res):
 117         self.failUnlessEqual(res.getResults(), SUCCESS)
 118         self.failUnlessEqual(res.getSlavename(), "bot2")
 119
 120     def testFallback2(self):
 121         # Disable the first slave, so that a slaveping will timeout. Then
 122         # start a build, and verify that the non-failing (second) one is
 123         # claimed for the build, and that the failing one is removed from the
 124         # list.
 125
 126         # reduce the ping time so we'll failover faster
 127         self.master.botmaster.builders["b1"].START_BUILD_TIMEOUT = 1
 128         self.disappearSlave("bot1", "b1")
 129         d = self.doBuild("b1")
 130         d.addCallback(self._testFallback2_1)
 131         return d
 132     def _testFallback2_1(self, res):
 133         self.failUnlessEqual(res.getResults(), SUCCESS)
 134         self.failUnlessEqual(res.getSlavename(), "bot2")
 135         b1slaves = self.master.botmaster.builders["b1"].slaves
 136         self.failUnlessEqual(len(b1slaves), 1)
 137         self.failUnlessEqual(b1slaves[0].slave.slavename, "bot2")
 138
 139
 140     def notFinished(self, brs):
 141         # utility method
 142         builds = brs.getBuilds()
 143         self.failIf(len(builds) > 1)
 144         if builds:
 145             self.failIf(builds[0].isFinished())
 146
 147     def testDontClaimPingingSlave(self):
 148         # have two slaves connect for the same builder. Do something to the
 149         # first one so that slavepings are delayed (but do not fail
 150         # outright).
 151         timers = []
 152         self.slaves['bot1'].debugOpts["stallPings"] = (10, timers)
 153         br = BuildRequest("forced", SourceStamp())
 154         d1 = br.waitUntilFinished()
 155         self.control.getBuilder("b1").requestBuild(br)
 156         s1 = br.status # this is a BuildRequestStatus
 157         # give it a chance to start pinging
 158         d2 = defer.Deferred()
 159         d2.addCallback(self._testDontClaimPingingSlave_1, d1, s1, timers)
 160         reactor.callLater(1, d2.callback, None)
 161         return d2
 162     def _testDontClaimPingingSlave_1(self, res, d1, s1, timers):
 163         # now the first build is running (waiting on the ping), so start the
 164         # second build. This should claim the second slave, not the first,
 165         # because the first is busy doing the ping.
 166         self.notFinished(s1)
 167         d3 = self.doBuild("b1")
 168         d3.addCallback(self._testDontClaimPingingSlave_2, d1, s1, timers)
 169         return d3
 170     def _testDontClaimPingingSlave_2(self, res, d1, s1, timers):
 171         self.failUnlessEqual(res.getSlavename(), "bot2")
 172         self.notFinished(s1)
 173         # now let the ping complete
 174         self.failUnlessEqual(len(timers), 1)
 175         timers[0].reset(0)
 176         d1.addCallback(self._testDontClaimPingingSlave_3)
 177         return d1
 178     def _testDontClaimPingingSlave_3(self, res):
 179         self.failUnlessEqual(res.getSlavename(), "bot1")
 180
 181 config_3 = """
 182 from buildbot.process import factory
 183 from buildbot.steps import dummy
 184 s = factory.s
 185
 186 BuildmasterConfig = c = {}
 187 c['bots'] = [('bot1', 'sekrit')]
 188 c['sources'] = []
 189 c['schedulers'] = []
 190 c['slavePortnum'] = 0
 191 c['schedulers'] = []
 192
 193 f1 = factory.BuildFactory([s(dummy.Wait, handle='one')])
 194 f2 = factory.BuildFactory([s(dummy.Wait, handle='two')])
 195 f3 = factory.BuildFactory([s(dummy.Wait, handle='three')])
 196
 197 c['builders'] = [
 198     {'name': 'b1', 'slavenames': ['bot1'],
 199      'builddir': 'b1', 'factory': f1},
 200     ]
 201 """
 202
 203 config_4 = config_3 + """
 204 c['builders'] = [
 205     {'name': 'b1', 'slavenames': ['bot1'],
 206      'builddir': 'b1', 'factory': f2},
 207     ]
 208 """
 209
 210 config_5 = config_3 + """
 211 c['builders'] = [
 212     {'name': 'b1', 'slavenames': ['bot1'],
 213      'builddir': 'b1', 'factory': f3},
 214     ]
 215 """
 216
 217 from buildbot.slave.commands import waitCommandRegistry
 218
 219 class Reconfig(RunMixin, unittest.TestCase):
 220
 221     def setUp(self):
 222         RunMixin.setUp(self)
 223         self.master.loadConfig(config_3)
 224         self.master.startService()
 225         d = self.connectSlave(["b1"])
 226         return d
 227
 228     def _one_started(self):
 229         log.msg("testReconfig._one_started")
 230         self.build1_started = True
 231         self.d1.callback(None)
 232         return self.d2
 233
 234     def _two_started(self):
 235         log.msg("testReconfig._two_started")
 236         self.build2_started = True
 237         self.d3.callback(None)
 238         return self.d4
 239
 240     def _three_started(self):
 241         log.msg("testReconfig._three_started")
 242         self.build3_started = True
 243         self.d5.callback(None)
 244         return self.d6
 245
 246     def testReconfig(self):
 247         # reconfiguring a Builder should not interrupt any running Builds. No
 248         # queued BuildRequests should be lost. The next Build started should
 249         # use the new process.
 250         slave1 = self.slaves['bot1']
 251         bot1 = slave1.getServiceNamed('bot')
 252         sb1 = bot1.builders['b1']
 253         self.failUnless(isinstance(sb1, bot.SlaveBuilder))
 254         self.failUnless(sb1.running)
 255         b1 = self.master.botmaster.builders['b1']
 256         self.orig_b1 = b1
 257
 258         self.d1 = d1 = defer.Deferred()
 259         self.d2 = d2 = defer.Deferred()
 260         self.d3, self.d4 = defer.Deferred(), defer.Deferred()
 261         self.d5, self.d6 = defer.Deferred(), defer.Deferred()
 262         self.build1_started = False
 263         self.build2_started = False
 264         self.build3_started = False
 265         waitCommandRegistry[("one","build1")] = self._one_started
 266         waitCommandRegistry[("two","build2")] = self._two_started
 267         waitCommandRegistry[("three","build3")] = self._three_started
 268
 269         # use different branches to make sure these cannot be merged
 270         br1 = BuildRequest("build1", SourceStamp(branch="1"))
 271         b1.submitBuildRequest(br1)
 272         br2 = BuildRequest("build2", SourceStamp(branch="2"))
 273         b1.submitBuildRequest(br2)
 274         br3 = BuildRequest("build3", SourceStamp(branch="3"))
 275         b1.submitBuildRequest(br3)
 276         self.requests = (br1, br2, br3)
 277         # all three are now in the queue
 278
 279         # wait until the first one has started
 280         d1.addCallback(self._testReconfig_2)
 281         return d1
 282
 283     def _testReconfig_2(self, res):
 284         log.msg("_testReconfig_2")
 285         # confirm that it is building
 286         brs = self.requests[0].status.getBuilds()
 287         self.failUnlessEqual(len(brs), 1)
 288         self.build1 = brs[0]
 289         self.failUnlessEqual(self.build1.getCurrentStep().getName(), "wait")
 290         # br1 is building, br2 and br3 are in the queue (in that order). Now
 291         # we reconfigure the Builder.
 292         self.failUnless(self.build1_started)
 293         d = self.master.loadConfig(config_4)
 294         d.addCallback(self._testReconfig_3)
 295         return d
 296
 297     def _testReconfig_3(self, res):
 298         log.msg("_testReconfig_3")
 299         # now check to see that br1 is still building, and that br2 and br3
 300         # are in the queue of the new builder
 301         b1 = self.master.botmaster.builders['b1']
 302         self.failIfIdentical(b1, self.orig_b1)
 303         self.failIf(self.build1.isFinished())
 304         self.failUnlessEqual(self.build1.getCurrentStep().getName(), "wait")
 305         self.failUnlessEqual(len(b1.buildable), 2)
 306         self.failUnless(self.requests[1] in b1.buildable)
 307         self.failUnless(self.requests[2] in b1.buildable)
 308
 309         # allow br1 to finish, and make sure its status is delivered normally
 310         d = self.requests[0].waitUntilFinished()
 311         d.addCallback(self._testReconfig_4)
 312         self.d2.callback(None)
 313         return d
 314
 315     def _testReconfig_4(self, bs):
 316         log.msg("_testReconfig_4")
 317         self.failUnlessEqual(bs.getReason(), "build1")
 318         self.failUnless(bs.isFinished())
 319         self.failUnlessEqual(bs.getResults(), SUCCESS)
 320
 321         # at this point, the first build has finished, and there is a pending
 322         # call to start the second build. Once that pending call fires, there
 323         # is a network roundtrip before the 'wait' RemoteCommand is delivered
 324         # to the slave. We need to wait for both events to happen before we
 325         # can check to make sure it is using the correct process. Just wait a
 326         # full second.
 327         d = defer.Deferred()
 328         d.addCallback(self._testReconfig_5)
 329         reactor.callLater(1, d.callback, None)
 330         return d
 331
 332     def _testReconfig_5(self, res):
 333         log.msg("_testReconfig_5")
 334         # at this point the next build ought to be running
 335         b1 = self.master.botmaster.builders['b1']
 336         self.failUnlessEqual(len(b1.buildable), 1)
 337         self.failUnless(self.requests[2] in b1.buildable)
 338         self.failUnlessEqual(len(b1.building), 1)
 339         # and it ought to be using the new process
 340         self.failUnless(self.build2_started)
 341
 342         # now, while the second build is running, change the config multiple
 343         # times.
 344
 345         d = self.master.loadConfig(config_3)
 346         d.addCallback(lambda res: self.master.loadConfig(config_4))
 347         d.addCallback(lambda res: self.master.loadConfig(config_5))
 348         def _done(res):
 349             # then once that's done, allow the second build to finish and
 350             # wait for it to complete
 351             da = self.requests[1].waitUntilFinished()
 352             self.d4.callback(None)
 353             return da
 354         d.addCallback(_done)
 355         def _done2(res):
 356             # and once *that*'s done, wait another second to let the third
 357             # build start
 358             db = defer.Deferred()
 359             reactor.callLater(1, db.callback, None)
 360             return db
 361         d.addCallback(_done2)
 362         d.addCallback(self._testReconfig_6)
 363         return d
 364
 365     def _testReconfig_6(self, res):
 366         log.msg("_testReconfig_6")
 367         # now check to see that the third build is running
 368         self.failUnless(self.build3_started)
 369
 370         # we're done
 371
 372
 373
 374 class Slave2(RunMixin, unittest.TestCase):
 375
 376     revision = 0
 377
 378     def setUp(self):
 379         RunMixin.setUp(self)
 380         self.master.loadConfig(config_1)
 381         self.master.startService()
 382
 383     def doBuild(self, buildername, reason="forced"):
 384         # we need to prevent these builds from being merged, so we create
 385         # each of them with a different revision specifier. The revision is
 386         # ignored because our build process does not have a source checkout
 387         # step.
 388         self.revision += 1
 389         br = BuildRequest(reason, SourceStamp(revision=self.revision))
 390         d = br.waitUntilFinished()
 391         self.control.getBuilder(buildername).requestBuild(br)
 392         return d
 393
 394     def testFirstComeFirstServed(self):
 395         # submit three builds, then connect a slave which fails the
 396         # slaveping. The first build will claim the slave, do the slaveping,
 397         # give up, and re-queue the build. Verify that the build gets
 398         # re-queued in front of all other builds. This may be tricky, because
 399         # the other builds may attempt to claim the just-failed slave.
 400
 401         d1 = self.doBuild("b1", "first")
 402         d2 = self.doBuild("b1", "second")
 403         #buildable = self.master.botmaster.builders["b1"].buildable
 404         #print [b.reason for b in buildable]
 405
 406         # specifically, I want the poor build to get precedence over any
 407         # others that were waiting. To test this, we need more builds than
 408         # slaves.
 409
 410         # now connect a broken slave. The first build started as soon as it
 411         # connects, so by the time we get to our _1 method, the ill-fated
 412         # build has already started.
 413         d = self.connectSlave(["b1"], opts={"failPingOnce": True})
 414         d.addCallback(self._testFirstComeFirstServed_1, d1, d2)
 415         return d
 416     def _testFirstComeFirstServed_1(self, res, d1, d2):
 417         # the master has send the slaveping. When this is received, it will
 418         # fail, causing the master to hang up on the slave. When it
 419         # reconnects, it should find the first build at the front of the
 420         # queue. If we simply wait for both builds to complete, then look at
 421         # the status logs, we should see that the builds ran in the correct
 422         # order.
 423
 424         d = defer.DeferredList([d1,d2])
 425         d.addCallback(self._testFirstComeFirstServed_2)
 426         return d
 427     def _testFirstComeFirstServed_2(self, res):
 428         b = self.status.getBuilder("b1")
 429         builds = b.getBuild(0), b.getBuild(1)
 430         reasons = [build.getReason() for build in builds]
 431         self.failUnlessEqual(reasons, ["first", "second"])
 432